Add bus_space and busdma support for sparc64.

5 files changed, 2150 insertions, 20 deletions

diff --git a/sys/sparc64/include/bus.h b/sys/sparc64/include/bus.h
index 5f14bcf..3ef2e60 100644
--- a/sys/sparc64/include/bus.h
+++ b/sys/sparc64/include/bus.h
@@ -1,7 +1,11 @@
 /*-
- * Copyright (c) 2001 Jake Burkholder.
+ * Copyright (c) 1996, 1997, 1998, 2001 The NetBSD Foundation, Inc.
  * All rights reserved.
  *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -10,18 +14,60 @@
  * 2. Redistributions in binary form must reproduce the above copyright
  *    notice, this list of conditions and the following disclaimer in the
  *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the NetBSD
+ *	Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * Copyright (c) 1997-1999 Eduardo E. Horvath. All rights reserved.
+ * Copyright (c) 1996 Charles M. Hannum.  All rights reserved.
+ * Copyright (c) 1996 Christopher G. Demetriou.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *      This product includes software developed by Christopher G. Demetriou
+ *	for the NetBSD Project.
+ * 4. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
  *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
+ * 	from: NetBSD: bus.h,v 1.28 2001/07/19 15:32:19 thorpej Exp
+ *	and
+ *	from: FreeBSD: src/sys/alpha/include/bus.h,v 1.9 2001/01/09
  *
  * $FreeBSD$
  */
@@ -29,7 +75,1368 @@
 #ifndef	_MACHINE_BUS_H_
 #define	_MACHINE_BUS_H_
 
-typedef	int bus_space_tag_t;
-typedef int bus_space_handle_t;
+#include <machine/types.h>
+#include <machine/cpufunc.h>
+
+/*
+ * Debug hooks
+ */
+
+#define	BSDB_ACCESS	0x01
+
+extern int bus_space_debug;
+
+/*
+ * UPA and SBUS spaces are non-cached and big endian
+ * (except for RAM and PROM)
+ *
+ * PCI spaces are non-cached and little endian
+ */
+
+#define	UPA_BUS_SPACE		0
+#define	SBUS_BUS_SPACE		1
+#define	PCI_CONFIG_BUS_SPACE	2
+#define	PCI_IO_BUS_SPACE	3
+#define	PCI_MEMORY_BUS_SPACE	4
+#define	LAST_BUS_SPACE		5
+
+extern int bus_type_asi[];
+extern int bus_stream_asi[];
+
+#define __BUS_SPACE_HAS_STREAM_METHODS	1
+
+/*
+ * Bus address and size types
+ */
+typedef	u_long		bus_space_handle_t;
+typedef int		bus_type_t;
+typedef u_long		bus_addr_t;
+typedef u_long		bus_size_t;
+
+#define BUS_SPACE_MAXSIZE_24BIT	0xFFFFFF
+#define BUS_SPACE_MAXSIZE_32BIT 0xFFFFFFFF
+#define BUS_SPACE_MAXSIZE	(128 * 1024) /* Maximum supported size */
+#define BUS_SPACE_MAXADDR_24BIT	0xFFFFFF
+#define BUS_SPACE_MAXADDR_32BIT 0xFFFFFFFF
+#define BUS_SPACE_MAXADDR	0xFFFFFFFF
+
+#define BUS_SPACE_UNRESTRICTED	(~0UL)
+
+/*
+ * Access methods for bus resources and address space.
+ */
+typedef struct bus_space_tag	*bus_space_tag_t;
+
+struct bus_space_tag {
+	void		*cookie;
+	bus_space_tag_t	parent;
+	int		type;
+
+	void	(*bus_barrier) __P((
+				bus_space_tag_t,
+				bus_space_handle_t,
+				bus_size_t,		/*offset*/
+				bus_size_t,		/*size*/
+				int));			/*flags*/
+};
+
+/*
+ * Helpers
+ */
+int		sparc64_bus_mem_map __P((
+				bus_type_t,
+				bus_addr_t,
+				bus_size_t,
+				int,			/*flags*/
+				vm_offset_t,		/*preferred vaddr*/
+				void **));
+int		sparc64_bus_mem_unmap __P((
+				void *,
+				bus_size_t));
+bus_space_handle_t	sparc64_fake_bustag __P((
+				int,
+				bus_addr_t,
+				struct bus_space_tag *));
+    
+/*
+ * Bus space function prototypes.
+ */
+static void	bus_space_barrier __P((
+				bus_space_tag_t,
+				bus_space_handle_t,
+				bus_size_t,
+				bus_size_t,
+				int));
+
+/* This macro finds the first "upstream" implementation of method `f' */
+#define _BS_CALL(t,f)			\
+	while (t->f == NULL)		\
+		t = t->parent;		\
+	return (*(t)->f)
+
+__inline__ void
+bus_space_barrier(t, h, o, s, f)
+	bus_space_tag_t t;
+	bus_space_handle_t h;
+	bus_size_t o;
+	bus_size_t s;
+	int f;
+{
+	_BS_CALL(t, bus_barrier)(t, h, o, s, f);
+}
+
+/* flags for bus space map functions */
+#define BUS_SPACE_MAP_CACHEABLE		0x0001
+#define BUS_SPACE_MAP_LINEAR		0x0002
+#define BUS_SPACE_MAP_READONLY		0x0004
+#define BUS_SPACE_MAP_PREFETCHABLE	0x0008
+/* placeholders for bus functions... */
+#define BUS_SPACE_MAP_BUS1		0x0100
+#define BUS_SPACE_MAP_BUS2		0x0200
+#define BUS_SPACE_MAP_BUS3		0x0400
+#define BUS_SPACE_MAP_BUS4		0x0800
+
+/* flags for bus_space_barrier() */
+#define	BUS_SPACE_BARRIER_READ		0x01	/* force read barrier */
+#define	BUS_SPACE_BARRIER_WRITE		0x02	/* force write barrier */
+
+/*
+ *	u_intN_t bus_space_read_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset));
+ *
+ * Read a 1, 2, 4, or 8 byte quantity from bus space
+ * described by tag/handle/offset.
+ */
+#ifndef BUS_SPACE_DEBUG
+#define	bus_space_read_1(t, h, o)					\
+	    lduba_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type])
+
+#define	bus_space_read_2(t, h, o)					\
+	    lduha_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type])
+
+#define	bus_space_read_4(t, h, o)					\
+	    lduwa_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type])
+
+#define	bus_space_read_8(t, h, o)					\
+	    ldxa_nc((caddr_t)(h) + (o), bus_type_asi[(t)->type])
+#else
+#define	bus_space_read_1(t, h, o) ({					\
+	unsigned char __bv =				      		\
+	    lduba_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type]);	\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsr1(%llx + %llx, %x) -> %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_type_asi[(t)->type], (unsigned int) __bv);		\
+	__bv; })
+
+#define	bus_space_read_2(t, h, o) ({					\
+	unsigned short __bv =				      		\
+	    lduha_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type]);	\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsr2(%llx + %llx, %x) -> %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_type_asi[(t)->type], (unsigned int)__bv);		\
+	__bv; })
+
+#define	bus_space_read_4(t, h, o) ({					\
+	unsigned int __bv =				      		\
+	    lduwa_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type]);	\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsr4(%llx + %llx, %x) -> %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_type_asi[(t)->type], __bv);				\
+	__bv; })
+
+#define	bus_space_read_8(t, h, o) ({					\
+	u_int64_t __bv =				      		\
+	    ldxa_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type]);	\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsr8(%llx + %llx, %x) -> %llx\n", (u_int64_t)(h),	\
+		(u_int64_t)(o),						\
+		bus_type_asi[(t)->type], __bv);				\
+	__bv; })
+#endif
+
+/*
+ *	void bus_space_read_multi_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset,
+ *	    u_intN_t *addr, size_t count));
+ *
+ * Read `count' 1, 2, 4, or 8 byte quantities from bus space
+ * described by tag/handle/offset and copy into buffer provided.
+ */
+#define	bus_space_read_multi_1(t, h, o, a, c) do {			\
+	int i = c;							\
+	u_int8_t *p = (u_int8_t *)a;					\
+	while (i-- > 0)							\
+		*p++ = bus_space_read_1(t, h, o);			\
+} while (0)
+
+#define	bus_space_read_multi_2(t, h, o, a, c) do {			\
+	int i = c;							\
+	u_int16_t *p = (u_int16_t *)a;					\
+	while (i-- > 0)							\
+		*p++ = bus_space_read_2(t, h, o);			\
+} while (0)
+
+#define	bus_space_read_multi_4(t, h, o, a, c) do {			\
+	int i = c;							\
+	u_int32_t *p = (u_int32_t *)a;					\
+	while (i-- > 0)							\
+		*p++ = bus_space_read_4(t, h, o);			\
+} while (0)
+
+#define	bus_space_read_multi_8(t, h, o, a, c) do {			\
+	int i = c;							\
+	u_int64_t *p = (u_int64_t *)a;					\
+	while (i-- > 0)							\
+		*p++ = bus_space_read_8(t, h, o);			\
+} while (0)
+
+/*
+ *	void bus_space_write_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset,
+ *	    u_intN_t value));
+ *
+ * Write the 1, 2, 4, or 8 byte value `value' to bus space
+ * described by tag/handle/offset.
+ */
+#ifndef BUS_SPACE_DEBUG
+#define	bus_space_write_1(t, h, o, v)					\
+	stba_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type], (v))
+
+#define	bus_space_write_2(t, h, o, v)					\
+	stha_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type], (v))
+
+#define	bus_space_write_4(t, h, o, v)					\
+	stwa_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type], (v))
+
+#define	bus_space_write_8(t, h, o, v)					\
+	stxa_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type], (v))
+#else
+#define	bus_space_write_1(t, h, o, v) do {				\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsw1(%llx + %llx, %x) <- %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_type_asi[(t)->type], (unsigned int) v);		\
+	stba_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type], (v));	\
+} while (0)
+
+#define	bus_space_write_2(t, h, o, v) do {				\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsw2(%llx + %llx, %x) <- %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_type_asi[(t)->type], (unsigned int) v);		\
+	stha_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type], (v));	\
+} while (0)
+
+#define	bus_space_write_4(t, h, o, v) do {				\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsw4(%llx + %llx, %x) <- %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_type_asi[(t)->type], (unsigned int) v);		\
+	stwa_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type], (v));	\
+} while (0)
+
+#define	bus_space_write_8(t, h, o, v) do {				\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsw8(%llx + %llx, %x) <- %llx\n", (u_int64_t)(h),	\
+		(u_int64_t)(o),						\
+		bus_type_asi[(t)->type], (u_int64_t) v);		\
+	stxa_nc((caddr_t)((h) + (o)), bus_type_asi[(t)->type], (v));	\
+} while (0)
+#endif
+
+/*
+ *	void bus_space_write_multi_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset,
+ *	    const u_intN_t *addr, size_t count));
+ *
+ * Write `count' 1, 2, 4, or 8 byte quantities from the buffer
+ * provided to bus space described by tag/handle/offset.
+ */
+#define	bus_space_write_multi_1(t, h, o, a, c) do {			\
+	int i = c;							\
+	u_int8_t *p = (u_int8_t *)a;					\
+	while (i-- > 0)							\
+		bus_space_write_1(t, h, o, *p++);			\
+} while (0)
+
+#define bus_space_write_multi_2(t, h, o, a, c) do {			\
+	int i = c;							\
+	u_int16_t *p = (u_int16_t *)a;					\
+	while (i-- > 0)							\
+		bus_space_write_2(t, h, o, *p++);			\
+} while (0)
+
+#define bus_space_write_multi_4(t, h, o, a, c) do {			\
+	int i = c;							\
+	u_int32_t *p = (u_int32_t *)a;					\
+	while (i-- > 0)							\
+		bus_space_write_4(t, h, o, *p++);			\
+} while (0)
+
+#define bus_space_write_multi_8(t, h, o, a, c) do {			\
+	int i = c;							\
+	u_int64_t *p = (u_int64_t *)a;					\
+	while (i-- > 0)							\
+		bus_space_write_8(t, h, o, *p++);			\
+} while (0)
+
+/*
+ *	void bus_space_set_multi_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset, u_intN_t val,
+ *	    size_t count));
+ *
+ * Write the 1, 2, 4, or 8 byte value `val' to bus space described
+ * by tag/handle/offset `count' times.
+ */
+#define bus_space_set_multi_1(t, h, o, v, c) do {			\
+	int i = c;							\
+	while (i-- > 0)							\
+		bus_space_write_1(t, h, o, v);				\
+} while (0)
+
+#define bus_space_set_multi_2(t, h, o, v, c) do {			\
+	int i = c;							\
+	while (i-- > 0)							\
+		bus_space_write_2(t, h, o, v);				\
+} while (0)
+
+#define bus_space_set_multi_4(t, h, o, v, c) do {			\
+	int i = c;							\
+	while (i-- > 0)							\
+		bus_space_write_4(t, h, o, v);				\
+} while (0)
+
+#define bus_space_set_multi_8(t, h, o, v, c) do {			\
+	int i = c;							\
+	while (i-- > 0)							\
+		bus_space_write_8(t, h, o, v);				\
+} while (0)
+
+/*
+ *	void bus_space_read_region_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t off,
+ *	    u_intN_t *addr, bus_size_t count));
+ *
+ */
+static void bus_space_read_region_1 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	u_int8_t *,
+	bus_size_t));
+static void bus_space_read_region_2 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	u_int16_t *,
+	bus_size_t));
+static void bus_space_read_region_4 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	u_int32_t *,
+	bus_size_t));
+static void bus_space_read_region_8 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	u_int64_t *,
+	bus_size_t));
+
+static __inline__ void
+bus_space_read_region_1(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	u_int8_t		*a;
+{
+	for (; c; a++, c--, o++)
+		*a = bus_space_read_1(t, h, o);
+}
+
+static __inline__ void
+bus_space_read_region_2(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	u_int16_t		*a;
+{
+	for (; c; a++, c--, o+=2)
+		*a = bus_space_read_2(t, h, o);
+}
+
+static __inline__ void
+bus_space_read_region_4(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	u_int32_t		*a;
+{
+	for (; c; a++, c--, o+=4)
+		*a = bus_space_read_4(t, h, o);
+}
+
+static __inline__ void
+bus_space_read_region_8(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	u_int64_t		*a;
+{
+	for (; c; a++, c--, o+=8)
+		*a = bus_space_read_8(t, h, o);
+}
+
+/*
+ *	void bus_space_write_region_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t off,
+ *	    u_intN_t *addr, bus_size_t count));
+ *
+ */
+static void bus_space_write_region_1 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int8_t *,
+	bus_size_t));
+static void bus_space_write_region_2 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int16_t *,
+	bus_size_t));
+static void bus_space_write_region_4 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int32_t *,
+	bus_size_t));
+static void bus_space_write_region_8 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int64_t *,
+	bus_size_t));
+
+static __inline__ void
+bus_space_write_region_1(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int8_t		*a;
+{
+	for (; c; a++, c--, o++)
+		bus_space_write_1(t, h, o, *a);
+}
+
+static __inline__ void
+bus_space_write_region_2(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int16_t		*a;
+{
+	for (; c; a++, c--, o+=2)
+		bus_space_write_2(t, h, o, *a);
+}
+
+static __inline__ void
+bus_space_write_region_4(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int32_t		*a;
+{
+	for (; c; a++, c--, o+=4)
+		bus_space_write_4(t, h, o, *a);
+}
+
+static __inline__ void
+bus_space_write_region_8(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int64_t		*a;
+{
+	for (; c; a++, c--, o+=8)
+		bus_space_write_8(t, h, o, *a);
+}
+
+/*
+ *	void bus_space_set_region_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t off,
+ *	    u_intN_t *addr, bus_size_t count));
+ *
+ */
+static void bus_space_set_region_1 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int8_t,
+	bus_size_t));
+static void bus_space_set_region_2 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int16_t,
+	bus_size_t));
+static void bus_space_set_region_4 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int32_t,
+	bus_size_t));
+static void bus_space_set_region_8 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int64_t,
+	bus_size_t));
+
+static __inline__ void
+bus_space_set_region_1(t, h, o, v, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int8_t		v;
+{
+	for (; c; c--, o++)
+		bus_space_write_1(t, h, o, v);
+}
+
+static __inline__ void
+bus_space_set_region_2(t, h, o, v, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int16_t		v;
+{
+	for (; c; c--, o+=2)
+		bus_space_write_2(t, h, o, v);
+}
+
+static __inline__ void
+bus_space_set_region_4(t, h, o, v, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int32_t		v;
+{
+	for (; c; c--, o+=4)
+		bus_space_write_4(t, h, o, v);
+}
+
+static __inline__ void
+bus_space_set_region_8(t, h, o, v, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int64_t		v;
+{
+	for (; c; c--, o+=8)
+		bus_space_write_8(t, h, o, v);
+}
+
+/*
+ *	void bus_space_copy_region_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh1, bus_size_t off1,
+ *	    bus_space_handle_t bsh2, bus_size_t off2,
+ *	    bus_size_t count));
+ *
+ * Copy `count' 1, 2, 4, or 8 byte values from bus space starting
+ * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2.
+ */
+static void bus_space_copy_region_1 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_size_t));
+static void bus_space_copy_region_2 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_size_t));
+static void bus_space_copy_region_4 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_size_t));
+static void bus_space_copy_region_8 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_size_t));
+
+static __inline__ void
+bus_space_copy_region_1(t, h1, o1, h2, o2, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h1, h2;
+	bus_size_t		o1, o2;
+	bus_size_t		c;
+{
+	for (; c; c--, o1++, o2++)
+	    bus_space_write_1(t, h1, o1, bus_space_read_1(t, h2, o2));
+}
+
+static __inline__ void
+bus_space_copy_region_2(t, h1, o1, h2, o2, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h1, h2;
+	bus_size_t		o1, o2;
+	bus_size_t		c;
+{
+	for (; c; c--, o1+=2, o2+=2)
+	    bus_space_write_2(t, h1, o1, bus_space_read_2(t, h2, o2));
+}
+
+static __inline__ void
+bus_space_copy_region_4(t, h1, o1, h2, o2, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h1, h2;
+	bus_size_t		o1, o2;
+	bus_size_t		c;
+{
+	for (; c; c--, o1+=4, o2+=4)
+	    bus_space_write_4(t, h1, o1, bus_space_read_4(t, h2, o2));
+}
+
+static __inline__ void
+bus_space_copy_region_8(t, h1, o1, h2, o2, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h1, h2;
+	bus_size_t		o1, o2;
+	bus_size_t		c;
+{
+	for (; c; c--, o1+=8, o2+=8)
+	    bus_space_write_8(t, h1, o1, bus_space_read_8(t, h2, o2));
+}
+
+/*
+ *	u_intN_t bus_space_read_stream_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset));
+ *
+ * Read a 1, 2, 4, or 8 byte quantity from bus space
+ * described by tag/handle/offset.
+ */
+#ifndef BUS_SPACE_DEBUG
+#define	bus_space_read_stream_1(t, h, o)				\
+	    lduba_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type])
+
+#define	bus_space_read_stream_2(t, h, o)				\
+	    lduha_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type])
+
+#define	bus_space_read_stream_4(t, h, o)				\
+	    lduwa_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type])
+
+#define	bus_space_read_stream_8(t, h, o)				\
+	    ldxa_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type])
+#else
+#define	bus_space_read_stream_1(t, h, o) ({				\
+	unsigned char __bv =				      		\
+	    lduba_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type]);	\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsr1(%llx + %llx, %x) -> %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_stream_asi[(t)->type], (unsigned int) __bv);	\
+	__bv; })
+
+#define	bus_space_read_stream_2(t, h, o) ({				\
+	unsigned short __bv =				      		\
+	    lduha_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type]);	\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsr2(%llx + %llx, %x) -> %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_stream_asi[(t)->type], (unsigned int)__bv);		\
+	__bv; })
+
+#define	bus_space_read_stream_4(t, h, o) ({				\
+	unsigned int __bv =				      		\
+	    lduwa_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type]);	\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsr4(%llx + %llx, %x) -> %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_stream_asi[(t)->type], __bv);			\
+	__bv; })
+
+#define	bus_space_read_stream_8(t, h, o) ({				\
+	u_int64_t __bv =				      		\
+	    ldxa_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type]);	\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsr8(%llx + %llx, %x) -> %llx\n", (u_int64_t)(h),	\
+		(u_int64_t)(o),						\
+		bus_stream_asi[(t)->type], __bv);			\
+	__bv; })
+#endif
+
+/*
+ *	void bus_space_read_multi_stream_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset,
+ *	    u_intN_t *addr, size_t count));
+ *
+ * Read `count' 1, 2, 4, or 8 byte quantities from bus space
+ * described by tag/handle/offset and copy into buffer provided.
+ */
+#define	bus_space_read_multi_stream_1(t, h, o, a, c) do {		\
+	int i = c;							\
+	u_int8_t *p = (u_int8_t *)a;					\
+	while (i-- > 0)							\
+		*p++ = bus_space_read_stream_1(t, h, o);		\
+} while (0)
+
+#define	bus_space_read_multi_stream_2(t, h, o, a, c) do {		\
+	int i = c;							\
+	u_int16_t *p = (u_int16_t *)a;					\
+	while (i-- > 0)							\
+		*p++ = bus_space_read_stream_2(t, h, o);		\
+} while (0)
+
+#define	bus_space_read_multi_stream_4(t, h, o, a, c) do {		\
+	int i = c;							\
+	u_int32_t *p = (u_int32_t *)a;					\
+	while (i-- > 0)							\
+		*p++ = bus_space_read_stream_4(t, h, o);		\
+} while (0)
+
+#define	bus_space_read_multi_stream_8(t, h, o, a, c) do {		\
+	int i = c;							\
+	u_int64_t *p = (u_int64_t *)a;					\
+	while (i-- > 0)							\
+		*p++ = bus_space_read_stream_8(t, h, o);		\
+} while (0)
+
+/*
+ *	void bus_space_write_stream_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset,
+ *	    u_intN_t value));
+ *
+ * Write the 1, 2, 4, or 8 byte value `value' to bus space
+ * described by tag/handle/offset.
+ */
+#ifndef BUS_SPACE_DEBUG
+#define	bus_space_write_stream_1(t, h, o, v)				\
+	stba_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type], (v))
+
+#define	bus_space_write_stream_2(t, h, o, v)				\
+	stha_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type], (v))
+
+#define	bus_space_write_stream_4(t, h, o, v)				\
+	stwa_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type], (v))
+
+#define	bus_space_write_stream_8(t, h, o, v)				\
+	stxa_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type], (v))
+#else
+#define	bus_space_write_stream_1(t, h, o, v) do {			\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsw1(%llx + %llx, %x) <- %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_stream_asi[(t)->type], (unsigned int) v);		\
+	stba_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type], (v));	\
+} while (0)
+
+#define	bus_space_write_stream_2(t, h, o, v) do {			\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsw2(%llx + %llx, %x) <- %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_stream_asi[(t)->type], (unsigned int) v);		\
+	stha_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type], (v));	\
+} while (0)
+
+#define	bus_space_write_stream_4(t, h, o, v) ({				\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsw4(%llx + %llx, %x) <- %x\n", (u_int64_t)(h),		\
+		(u_int64_t)(o),						\
+		bus_stream_asi[(t)->type], (unsigned int) v);		\
+	stwa_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type], (v));	\
+} while (0)
+
+#define	bus_space_write_stream_8(t, h, o, v) ({				\
+	if (bus_space_debug & BSDB_ACCESS)				\
+	printf("bsw8(%llx + %llx, %x) <- %llx\n", (u_int64_t)(h),	\
+		(u_int64_t)(o),						\
+		bus_stream_asi[(t)->type], (u_int64_t) v);		\
+	stxa_nc((caddr_t)((h) + (o)), bus_stream_asi[(t)->type], (v));	\
+} while (0)
+#endif
+
+/*
+ *	void bus_space_write_multi_stream_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset,
+ *	    const u_intN_t *addr, size_t count));
+ *
+ * Write `count' 1, 2, 4, or 8 byte quantities from the buffer
+ * provided to bus space described by tag/handle/offset.
+ */
+#define	bus_space_write_multi_stream_1(t, h, o, a, c) do {		\
+	int i = c;							\
+	u_int8_t *p = (u_int8_t *)a;					\
+	while (i-- > 0)							\
+		bus_space_write_stream_1(t, h, o, *p++);		\
+} while (0)
+
+#define bus_space_write_multi_stream_2(t, h, o, a, c) do {		\
+	int i = c;							\
+	u_int16_t *p = (u_int16_t *)a;					\
+	while (i-- > 0)							\
+		bus_space_write_stream_2(t, h, o, *p++);		\
+} while (0)
+
+#define bus_space_write_multi_stream_4(t, h, o, a, c) do {		\
+	int i = c;							\
+	u_int32_t *p = (u_int32_t *)a;					\
+	while (i-- > 0)							\
+		bus_space_write_stream_4(t, h, o, *p++);		\
+} while (0)
+
+#define bus_space_write_multi_stream_8(t, h, o, a, c) do {		\
+	int i = c;							\
+	u_int64_t *p = (u_int64_t *)a;					\
+	while (i-- > 0)							\
+		bus_space_write_stream_8(t, h, o, *p++);		\
+} while (0)
+
+/*
+ *	void bus_space_set_multi_stream_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t offset, u_intN_t val,
+ *	    size_t count));
+ *
+ * Write the 1, 2, 4, or 8 byte value `val' to bus space described
+ * by tag/handle/offset `count' times.
+ */
+#define bus_space_set_multi_stream_1(t, h, o, v, c) do {		\
+	int i = c;							\
+	while (i-- > 0)							\
+		bus_space_write_stream_1(t, h, o, v);			\
+} while (0)
+
+#define bus_space_set_multi_stream_2(t, h, o, v, c) do {		\
+	int i = c;							\
+	while (i-- > 0)							\
+		bus_space_write_stream_2(t, h, o, v);			\
+} while (0)
+
+#define bus_space_set_multi_stream_4(t, h, o, v, c) do {		\
+	int i = c;							\
+	while (i-- > 0)							\
+		bus_space_write_stream_4(t, h, o, v);			\
+} while (0)
+
+#define bus_space_set_multi_stream_8(t, h, o, v, c) do {		\
+	int i = c;							\
+	while (i-- > 0)							\
+		bus_space_write_stream_8(t, h, o, v);			\
+} while (0)
+
+/*
+ *	void bus_space_read_region_stream_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t off,
+ *	    u_intN_t *addr, bus_size_t count));
+ *
+ */
+static void bus_space_read_region_stream_1 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	u_int8_t *,
+	bus_size_t));
+static void bus_space_read_region_stream_2 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	u_int16_t *,
+	bus_size_t));
+static void bus_space_read_region_stream_4 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	u_int32_t *,
+	bus_size_t));
+static void bus_space_read_region_stream_8 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	u_int64_t *,
+	bus_size_t));
+
+static __inline__ void
+bus_space_read_region_stream_1(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	u_int8_t		*a;
+{
+	for (; c; a++, c--, o++)
+		*a = bus_space_read_stream_1(t, h, o);
+}
+
+static __inline__ void
+bus_space_read_region_stream_2(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	u_int16_t		*a;
+{
+	for (; c; a++, c--, o+=2)
+		*a = bus_space_read_stream_2(t, h, o);
+}
+
+static __inline__ void
+bus_space_read_region_stream_4(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	u_int32_t		*a;
+{
+	for (; c; a++, c--, o+=4)
+		*a = bus_space_read_stream_4(t, h, o);
+}
+
+static __inline__ void
+bus_space_read_region_stream_8(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	u_int64_t		*a;
+{
+	for (; c; a++, c--, o+=8)
+		*a = bus_space_read_stream_8(t, h, o);
+}
+
+/*
+ *	void bus_space_write_region_stream_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t off,
+ *	    u_intN_t *addr, bus_size_t count));
+ *
+ */
+static void bus_space_write_region_stream_1 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int8_t *,
+	bus_size_t));
+static void bus_space_write_region_stream_2 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int16_t *,
+	bus_size_t));
+static void bus_space_write_region_stream_4 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int32_t *,
+	bus_size_t));
+static void bus_space_write_region_stream_8 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int64_t *,
+	bus_size_t));
+
+static __inline__ void
+bus_space_write_region_stream_1(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int8_t		*a;
+{
+	for (; c; a++, c--, o++)
+		bus_space_write_stream_1(t, h, o, *a);
+}
+
+static __inline__ void
+bus_space_write_region_stream_2(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int16_t		*a;
+{
+	for (; c; a++, c--, o+=2)
+		bus_space_write_stream_2(t, h, o, *a);
+}
+
+static __inline__ void
+bus_space_write_region_stream_4(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int32_t		*a;
+{
+	for (; c; a++, c--, o+=4)
+		bus_space_write_stream_4(t, h, o, *a);
+}
+
+static __inline__ void
+bus_space_write_region_stream_8(t, h, o, a, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int64_t		*a;
+{
+	for (; c; a++, c--, o+=8)
+		bus_space_write_stream_8(t, h, o, *a);
+}
+
+/*
+ *	void bus_space_set_region_stream_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh, bus_size_t off,
+ *	    u_intN_t *addr, bus_size_t count));
+ *
+ */
+static void bus_space_set_region_stream_1 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int8_t,
+	bus_size_t));
+static void bus_space_set_region_stream_2 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int16_t,
+	bus_size_t));
+static void bus_space_set_region_stream_4 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int32_t,
+	bus_size_t));
+static void bus_space_set_region_stream_8 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	const u_int64_t,
+	bus_size_t));
+
+static __inline__ void
+bus_space_set_region_stream_1(t, h, o, v, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int8_t		v;
+{
+	for (; c; c--, o++)
+		bus_space_write_stream_1(t, h, o, v);
+}
+
+static __inline__ void
+bus_space_set_region_stream_2(t, h, o, v, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int16_t		v;
+{
+	for (; c; c--, o+=2)
+		bus_space_write_stream_2(t, h, o, v);
+}
+
+static __inline__ void
+bus_space_set_region_stream_4(t, h, o, v, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int32_t		v;
+{
+	for (; c; c--, o+=4)
+		bus_space_write_stream_4(t, h, o, v);
+}
+
+static __inline__ void
+bus_space_set_region_stream_8(t, h, o, v, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h;
+	bus_size_t		o, c;
+	const u_int64_t		v;
+{
+	for (; c; c--, o+=8)
+		bus_space_write_stream_8(t, h, o, v);
+}
+
+/*
+ *	void bus_space_copy_region_stream_N __P((bus_space_tag_t tag,
+ *	    bus_space_handle_t bsh1, bus_size_t off1,
+ *	    bus_space_handle_t bsh2, bus_size_t off2,
+ *	    bus_size_t count));
+ *
+ * Copy `count' 1, 2, 4, or 8 byte values from bus space starting
+ * at tag/bsh1/off1 to bus space starting at tag/bsh2/off2.
+ */
+static void bus_space_copy_region_stream_1 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_size_t));
+static void bus_space_copy_region_stream_2 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_size_t));
+static void bus_space_copy_region_stream_4 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_size_t));
+static void bus_space_copy_region_stream_8 __P((bus_space_tag_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_space_handle_t,
+	bus_size_t,
+	bus_size_t));
+
+
+static __inline__ void
+bus_space_copy_region_stream_1(t, h1, o1, h2, o2, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h1, h2;
+	bus_size_t		o1, o2;
+	bus_size_t		c;
+{
+	for (; c; c--, o1++, o2++)
+	    bus_space_write_stream_1(t, h1, o1, bus_space_read_stream_1(t, h2,
+		o2));
+}
+
+static __inline__ void
+bus_space_copy_region_stream_2(t, h1, o1, h2, o2, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h1, h2;
+	bus_size_t		o1, o2;
+	bus_size_t		c;
+{
+	for (; c; c--, o1+=2, o2+=2)
+	    bus_space_write_stream_2(t, h1, o1, bus_space_read_stream_2(t, h2,
+		o2));
+}
+
+static __inline__ void
+bus_space_copy_region_stream_4(t, h1, o1, h2, o2, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h1, h2;
+	bus_size_t		o1, o2;
+	bus_size_t		c;
+{
+	for (; c; c--, o1+=4, o2+=4)
+	    bus_space_write_stream_4(t, h1, o1, bus_space_read_stream_4(t, h2,
+		o2));
+}
+
+static __inline__ void
+bus_space_copy_region_stream_8(t, h1, o1, h2, o2, c)
+	bus_space_tag_t		t;
+	bus_space_handle_t	h1, h2;
+	bus_size_t		o1, o2;
+	bus_size_t		c;
+{
+	for (; c; c--, o1+=8, o2+=8)
+	    bus_space_write_stream_8(t, h1, o1, bus_space_read_8(t, h2, o2));
+}
+
+#define BUS_SPACE_ALIGNED_POINTER(p, t) ALIGNED_POINTER(p, t)
+
+/* Back-compat functions for old ISA drivers */
+
+extern bus_space_tag_t isa_io_bt;
+extern bus_space_handle_t isa_io_hdl;
+extern bus_space_tag_t isa_mem_bt;
+extern bus_space_handle_t isa_mem_hdl;
+
+#define inb(o)		bus_space_read_1(isa_io_bt, isa_io_hdl, o)
+#define inw(o)		bus_space_read_2(isa_io_bt, isa_io_hdl, o)
+#define inl(o)		bus_space_read_4(isa_io_bt, isa_io_hdl, o)
+#if 0
+#define outb(o, v) do {							\
+	printf("outb used at %s:%d, address 0x%x -> 0x%lx\n",		\
+	    __func__, __LINE__, o, (unsigned long)isa_io_hdl + o);	\
+	bus_space_write_1(isa_io_bt, isa_io_hdl, o, v);			\
+} while (0)
+#else
+#define outb(o, v) bus_space_write_1(isa_io_bt, isa_io_hdl, o, v)
+#endif
+#define outw(o, v)	bus_space_write_2(isa_io_bt, isa_io_hdl, o, v)
+#define outl(o, v)	bus_space_write_4(isa_io_bt, isa_io_hdl, o, v)
+
+#define readb(o)	bus_space_read_1(isa_mem_bt, isa_mem_hdl, o)
+#define readw(o)	bus_space_read_2(isa_mem_bt, isa_mem_hdl, o)
+#define readl(o)	bus_space_read_4(isa_mem_bt, isa_mem_hdl, o)
+#define writeb(o, v)	bus_space_write_1(isa_mem_bt, isa_mem_hdl, o, v)
+#define writew(o, v)	bus_space_write_2(isa_mem_bt, isa_mem_hdl, o, v)
+#define writel(o, v)	bus_space_write_4(isa_mem_bt, isa_mem_hdl, o, v)
+
+#define insb(o, a, c) \
+	bus_space_read_multi_1(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define insw(o, a, c) \
+	bus_space_read_multi_2(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define insl(o, a, c) \
+	bus_space_read_multi_4(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define outsb(o, a, c) \
+	bus_space_write_multi_1(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define outsw(o, a, c) \
+	bus_space_write_multi_2(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+#define outsl(o, a, c) \
+	bus_space_write_multi_4(isa_io_bt, isa_io_hdl, o, (void*)a, c)
+
+#define memcpy_fromio(d, s, c) \
+	bus_space_read_region_1(isa_mem_bt, isa_mem_hdl, s, d, c)
+#define memcpy_toio(d, s, c) \
+	bus_space_write_region_1(isa_mem_bt, isa_mem_hdl, d, s, c)
+#define memcpy_io(d, s, c) \
+	bus_space_copy_region_1(isa_mem_bt, isa_mem_hdl, s, isa_mem_hdl, d, c)
+#define memset_io(d, v, c) \
+	bus_space_set_region_1(isa_mem_bt, isa_mem_hdl, d, v, c)
+#define memsetw_io(d, v, c) \
+	bus_space_set_region_2(isa_mem_bt, isa_mem_hdl, d, v, c)
+
+static __inline void
+memsetw(void *d, int val, size_t size)
+{
+    u_int16_t *sp = d;
+
+    while (size--)
+	*sp++ = val;
+}
+
+/* DMA support */
+
+/*
+ * Flags used in various bus DMA methods.
+ */
+#define	BUS_DMA_WAITOK		0x000	/* safe to sleep (pseudo-flag) */
+#define	BUS_DMA_NOWAIT		0x001	/* not safe to sleep */
+#define	BUS_DMA_ALLOCNOW	0x002	/* perform resource allocation now */
+#define	BUS_DMAMEM_NOSYNC	0x004	/* map memory to not require sync */
+#define	BUS_DMA_NOWRITE		0x008
+#define	BUS_DMA_BUS1		0x010	
+#define	BUS_DMA_BUS2		0x020
+#define	BUS_DMA_BUS3		0x040
+#define	BUS_DMA_BUS4		0x080
+/*
+ * The following flags are from NetBSD, but are not implemented for all
+ * architetures, and should therefore not be used in MI code.
+ * Some have different values than under NetBSD.
+ */
+#define	BUS_DMA_STREAMING	0x100	/* hint: sequential, unidirectional */
+#define	BUS_DMA_READ		0x200	/* mapping is device -> memory only */
+#define	BUS_DMA_WRITE		0x400	/* mapping is memory -> device only */
+#define	BUS_DMA_COHERENT	0x800	/* hint: map memory DMA coherent */
+
+#define	BUS_DMA_NOCACHE		BUS_DMA_BUS1
+/* Don't bother with alignment */
+#define	BUS_DMA_DVMA		BUS_DMA_BUS2
+
+/* Forwards needed by prototypes below. */
+struct mbuf;
+struct uio;
+
+/*
+ *	bus_dmasync_op_t
+ *
+ *	Operations performed by bus_dmamap_sync().
+ */
+typedef enum {
+	BUS_DMASYNC_PREREAD,
+	BUS_DMASYNC_POSTREAD,
+	BUS_DMASYNC_PREWRITE,
+	BUS_DMASYNC_POSTWRITE,
+} bus_dmasync_op_t;
+
+/*
+ * A function that returns 1 if the address cannot be accessed by
+ * a device and 0 if it can be.
+ */
+typedef int bus_dma_filter_t(void *, bus_addr_t);
+
+typedef struct bus_dma_tag	*bus_dma_tag_t;
+typedef struct bus_dmamap	*bus_dmamap_t;
+
+/*
+ *	bus_dma_segment_t
+ *
+ *	Describes a single contiguous DMA transaction.  Values
+ *	are suitable for programming into DMA registers.
+ */
+struct bus_dma_segment {
+	bus_addr_t	ds_addr;	/* DVMA address */
+	bus_size_t	ds_len;		/* length of transfer */
+};
+typedef struct bus_dma_segment	bus_dma_segment_t;
+
+/*
+ * A function that processes a successfully loaded dma map or an error
+ * from a delayed load map.
+ */
+typedef void bus_dmamap_callback_t(void *, bus_dma_segment_t *, int, int);
+
+/*
+ *	bus_dma_tag_t
+ *
+ *	A machine-dependent opaque type describing the implementation of
+ *	DMA for a given bus.
+ */
+struct bus_dma_tag {
+	void		*cookie;		/* cookie used in the guts */
+	bus_dma_tag_t	parent;
+	bus_size_t	alignment;
+	bus_size_t	boundary;
+	bus_addr_t	lowaddr;
+	bus_addr_t	highaddr;
+	bus_dma_filter_t	*filter;
+	void		*filterarg;
+	bus_size_t	maxsize;
+	u_int		nsegments;
+	bus_size_t	maxsegsz;
+	int		flags;
+	int		ref_count;
+	int		map_count;
+
+	/*
+	 * DMA mapping methods.
+	 */
+	int	(*dmamap_create) __P((bus_dma_tag_t, int, bus_dmamap_t *));
+	int	(*dmamap_destroy) __P((bus_dma_tag_t, bus_dmamap_t));
+	int	(*dmamap_load) __P((bus_dma_tag_t, bus_dmamap_t, void *,
+		    bus_size_t, bus_dmamap_callback_t *, void *, int));
+	void	(*dmamap_unload) __P((bus_dma_tag_t, bus_dmamap_t));
+	void	(*dmamap_sync) __P((bus_dma_tag_t, bus_dmamap_t,
+		    bus_dmasync_op_t));
+
+	/*
+	 * DMA memory utility functions.
+	 */
+	int	(*dmamem_alloc) __P((bus_dma_tag_t, void **, int,
+	    	    bus_dmamap_t *));
+	void	(*dmamem_free) __P((bus_dma_tag_t, void *, bus_dmamap_t));
+};
+
+/*
+ * XXX: This is a kluge. It would be better to handle dma tags in a hierarchical
+ * way, and have a BUS_GET_DMA_TAG(); however, since this is not currently the
+ * case, save a root tag in the relevant bus attach function and use that.
+ * Keep the hierarchical structure, it might become needed in the future.
+ */
+extern bus_dma_tag_t sparc64_root_dma_tag;
+
+int bus_dma_tag_create(bus_dma_tag_t, bus_size_t, bus_size_t, bus_addr_t,
+	bus_addr_t, bus_dma_filter_t *, void *, bus_size_t, int, bus_size_t,
+	int, bus_dma_tag_t *);
+
+int bus_dma_tag_destroy(bus_dma_tag_t);
+
+int sparc64_dmamem_alloc_map(bus_dma_tag_t dmat, bus_dmamap_t *mapp);
+void sparc64_dmamem_free_map(bus_dma_tag_t dmat, bus_dmamap_t map);
+
+#define	bus_dmamap_create(t, f, p)					\
+	(*(t)->dmamap_create)((t), (f), (p))
+#define	bus_dmamap_destroy(t, p)					\
+	(*(t)->dmamap_destroy)((t), (p))
+#define	bus_dmamap_load(t, m, p, s, cb, cba, f)				\
+	(*(t)->dmamap_load)((t), (m), (p), (s), (cb), (cba), (f))
+#define	bus_dmamap_unload(t, p)						\
+	(*(t)->dmamap_unload)((t), (p))
+#define	bus_dmamap_sync(t, m, op)					\
+	(void)((t)->dmamap_sync ?					\
+	    (*(t)->dmamap_sync)((t), (m), (op)) : (void)0)
+
+#define	bus_dmamem_alloc(t, v, f, m)					\
+	(*(t)->dmamem_alloc)((t), (v), (f), (m))
+#define	bus_dmamem_free(t, v, m)					\
+	(*(t)->dmamem_free)((t), (v), (m))
+
+/*
+ *	bus_dmamap_t
+ *
+ *	Describes a DMA mapping.
+ */
+struct bus_dmamap {
+	bus_dma_tag_t	dmat;
+	void		*buf;		/* unmapped buffer pointer */
+	bus_size_t	buflen;		/* unmapped buffer length */
+	bus_addr_t	start;		/* start of mapped region */
+	struct resource *res;		/* associated resource */
+};
 
 #endif /* !_MACHINE_BUS_H_ */
diff --git a/sys/sparc64/include/bus_memio.h b/sys/sparc64/include/bus_memio.h
new file mode 100644
index 0000000..f27ce00
--- /dev/null
+++ b/sys/sparc64/include/bus_memio.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 1997 Justin Gibbs.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * 	from: FreeBSD: src/sys/i386/include/bus_memio.h,v 1.2 1999/08/28
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_BUS_MEMIO_H_
+#define _MACHINE_BUS_MEMIO_H_
+#endif /* _MACHINE_BUS_MEMIO_H_ */
diff --git a/sys/sparc64/include/bus_pio.h b/sys/sparc64/include/bus_pio.h
new file mode 100644
index 0000000..06a996b
--- /dev/null
+++ b/sys/sparc64/include/bus_pio.h
@@ -0,0 +1,33 @@
+/*
+ * Copyright (c) 1997 Justin Gibbs.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * 	from: FreeBSD: src/sys/i386/include/bus_pio.h,v 1.2 1999/08/28
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_BUS_PIO_H_
+#define _MACHINE_BUS_PIO_H_
+#endif /* _MACHINE_BUS_PIO_H_ */
diff --git a/sys/sparc64/include/cpufunc.h b/sys/sparc64/include/cpufunc.h
index 054b8df..8ce80db 100644
--- a/sys/sparc64/include/cpufunc.h
+++ b/sys/sparc64/include/cpufunc.h
@@ -80,17 +80,55 @@
 	__asm __volatile("flushw" : :);					\
 } while (0)
 
-#define	ldxa(va, asi) ({						\
-	u_long __r;							\
-	__asm __volatile("ldxa [%1] %2, %0"				\
+/* Generate ld*a/st*a functions for non-constant ASI's. */
+#define LDNC_GEN(tp, o)							\
+	static __inline tp						\
+	o ## _nc(caddr_t va, int asi)					\
+	{								\
+		tp r;							\
+		__asm __volatile("wr %2, 0, %%asi;" #o " [%1] %%asi, %0"\
+		    : "=r" (r) : "r" (va), "r" (asi));			\
+		return (r);						\
+	}
+
+LDNC_GEN(u_char, lduba);
+LDNC_GEN(u_short, lduha);
+LDNC_GEN(u_int, lduwa);
+LDNC_GEN(u_long, ldxa);
+
+#define	LD_GENERIC(va, asi, op, type) ({				\
+	type __r;							\
+	__asm __volatile(#op " [%1] %2, %0"				\
 	    : "=r" (__r) : "r" (va), "n" (asi));			\
 	__r;								\
 })
 
-#define	stxa(va, asi, val) do {						\
-	__asm __volatile("stxa %0, [%1] %2"				\
+#define	lduba(va, asi)	LD_GENERIC(va, asi, lduba, u_char)
+#define	lduha(va, asi)	LD_GENERIC(va, asi, lduha, u_short)
+#define	lduwa(va, asi)	LD_GENERIC(va, asi, lduwa, u_int)
+#define	ldxa(va, asi)	LD_GENERIC(va, asi, ldxa, u_long)
+
+#define STNC_GEN(tp, o)							\
+	static __inline void						\
+	o ## _nc(caddr_t va, int asi, tp val)				\
+	{								\
+		__asm __volatile("wr %2, 0, %%asi;" #o " %0, [%1] %%asi"\
+		    : : "r" (val), "r" (va), "r" (asi));		\
+	}
+
+STNC_GEN(u_char, stba);
+STNC_GEN(u_short, stha);
+STNC_GEN(u_int, stwa);
+STNC_GEN(u_long, stxa);
+
+#define	ST_GENERIC(va, asi, val, op)					\
+	__asm __volatile(#op " %0, [%1] %2"				\
 	    : : "r" (val), "r" (va), "n" (asi));			\
-} while (0)
+
+#define	stba(va, asi, val)	ST_GENERIC(va, asi, val, stba)
+#define	stha(va, asi, val)	ST_GENERIC(va, asi, val, stha)
+#define	stwa(va, asi, val)	ST_GENERIC(va, asi, val, stwa)
+#define	stxa(va, asi, val)	ST_GENERIC(va, asi, val, stxa)
 
 #define	membar(mask) do {						\
 	__asm __volatile("membar %0" : : "n" (mask));			\
@@ -140,6 +178,9 @@ critical_exit(critical_t pil)
 	wrpr(pil, pil, 0);
 }
 
+void ascopyfrom(u_long sasi, vm_offset_t src, caddr_t dst, size_t len);
+void ascopyto(caddr_t src, u_long dasi, vm_offset_t dst, size_t len);
+
 /*
  * Ultrasparc II doesn't implement popc in hardware.  Suck.
  */
@@ -165,4 +206,7 @@ ffs(int mask)
 }
 #endif
 
+#undef LDNC_GEN
+#undef STNC_GEN
+
 #endif /* !_MACHINE_CPUFUNC_H_ */
diff --git a/sys/sparc64/sparc64/bus_machdep.c b/sys/sparc64/sparc64/bus_machdep.c
new file mode 100644
index 0000000..91c8057
--- /dev/null
+++ b/sys/sparc64/sparc64/bus_machdep.c
@@ -0,0 +1,613 @@
+/*-
+ * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
+ * NASA Ames Research Center.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the NetBSD
+ *	Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Lawrence Berkeley Laboratory.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*
+ * Copyright (c) 1997, 1998 Justin T. Gibbs.
+ * All rights reserved.
+ * Copyright 2001 by Thomas Moestl <tmm@FreeBSD.org>.  All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions, and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE FOR
+ * ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)machdep.c	8.6 (Berkeley) 1/14/94
+ *	from: NetBSD: machdep.c,v 1.111 2001/09/15 07:13:40 eeh Exp
+ *	and
+ * 	from: FreeBSD: src/sys/i386/i386/busdma_machdep.c,v 1.24 2001/08/15
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/bus.h>
+#include <sys/malloc.h>
+#include <sys/proc.h>
+#include <sys/systm.h>
+
+#include <vm/vm.h>
+#include <vm/vm_extern.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_page.h>
+#include <vm/vm_param.h>
+
+#include <machine/asi.h>
+#include <machine/bus.h>
+#include <machine/cache.h>
+#include <machine/pmap.h>
+
+/* ASI's for bus access. */
+int bus_type_asi[] = {
+	ASI_PHYS_BYPASS_EC_WITH_EBIT,		/* UPA */
+	ASI_PHYS_BYPASS_EC_WITH_EBIT,		/* SBUS */
+	ASI_PHYS_BYPASS_EC_WITH_EBIT_L,		/* PCI configuration space */
+	ASI_PHYS_BYPASS_EC_WITH_EBIT_L,		/* PCI memory space */
+	ASI_PHYS_BYPASS_EC_WITH_EBIT_L,		/* PCI I/O space */
+	0
+};
+
+int bus_stream_asi[] = {
+	ASI_PHYS_BYPASS_EC_WITH_EBIT,		/* UPA */
+	ASI_PHYS_BYPASS_EC_WITH_EBIT,		/* SBUS */
+	ASI_PHYS_BYPASS_EC_WITH_EBIT,		/* PCI configuration space */
+	ASI_PHYS_BYPASS_EC_WITH_EBIT,		/* PCI memory space */
+	ASI_PHYS_BYPASS_EC_WITH_EBIT,		/* PCI I/O space */
+	0
+};
+
+/*
+ * busdma support code.
+ * Note: there is no support for bounce buffers yet.
+ */
+
+static int nexus_dmamap_create(bus_dma_tag_t, int, bus_dmamap_t *);
+static int nexus_dmamap_destroy(bus_dma_tag_t, bus_dmamap_t);
+static int nexus_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *, bus_size_t,
+    bus_dmamap_callback_t *, void *, int);
+static void nexus_dmamap_unload(bus_dma_tag_t, bus_dmamap_t);
+static void nexus_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_dmasync_op_t);
+static int nexus_dmamem_alloc(bus_dma_tag_t, void **, int, bus_dmamap_t *);
+static void nexus_dmamem_free(bus_dma_tag_t, void *, bus_dmamap_t);
+
+
+bus_dma_tag_t sparc64_root_dma_tag;
+
+/*
+ * Allocate a device specific dma_tag.
+ */
+int
+bus_dma_tag_create(bus_dma_tag_t parent, bus_size_t alignment,
+    bus_size_t boundary, bus_addr_t lowaddr, bus_addr_t highaddr,
+    bus_dma_filter_t *filter, void *filterarg, bus_size_t maxsize,
+    int nsegments, bus_size_t maxsegsz, int flags, bus_dma_tag_t *dmat)
+{
+
+	bus_dma_tag_t newtag, eparent;
+
+	/* Return a NULL tag on failure */
+	*dmat = NULL;
+
+	newtag = (bus_dma_tag_t)malloc(sizeof(*newtag), M_DEVBUF, M_NOWAIT);
+	if (newtag == NULL)
+		return (ENOMEM);
+
+	/* Ugh... */
+	eparent = parent != NULL ? parent : sparc64_root_dma_tag;
+	memcpy(newtag, eparent, sizeof(*newtag));
+	if (parent != NULL)
+		newtag->parent = parent;
+	newtag->alignment = alignment;
+	newtag->boundary = boundary;
+	newtag->lowaddr = trunc_page((vm_offset_t)lowaddr) + (PAGE_SIZE - 1);
+	newtag->highaddr = trunc_page((vm_offset_t)highaddr) + (PAGE_SIZE - 1);
+	newtag->filter = filter;
+	newtag->filterarg = filterarg;
+	newtag->maxsize = maxsize;
+	newtag->nsegments = nsegments;
+	newtag->maxsegsz = maxsegsz;
+	newtag->flags = flags;
+	newtag->ref_count = 1; /* Count ourself */
+	newtag->map_count = 0;
+	
+	/* Take into account any restrictions imposed by our parent tag */
+	if (parent != NULL) {
+		newtag->lowaddr = ulmin(parent->lowaddr, newtag->lowaddr);
+		newtag->highaddr = ulmax(parent->highaddr, newtag->highaddr);
+		/*
+		 * XXX Not really correct??? Probably need to honor boundary
+		 *     all the way up the inheritence chain.
+		 */
+		newtag->boundary = ulmax(parent->boundary, newtag->boundary);
+		if (parent != NULL)
+			parent->ref_count++;
+	}
+	
+	*dmat = newtag;
+	return (0);
+}
+
+int
+bus_dma_tag_destroy(bus_dma_tag_t dmat)
+{
+
+	if (dmat != NULL) {
+		if (dmat->map_count != 0)
+			return (EBUSY);
+
+		while (dmat != NULL) {
+			bus_dma_tag_t parent;
+
+			parent = dmat->parent;
+			dmat->ref_count--;
+			if (dmat->ref_count == 0) {
+				free(dmat, M_DEVBUF);
+				/*
+				 * Last reference count, so
+				 * release our reference
+				 * count on our parent.
+				 */
+				dmat = parent;
+			} else
+				dmat = NULL;
+		}
+	}
+	return (0);
+}
+
+/*
+ * Common function for DMA map creation.  May be called by bus-specific
+ * DMA map creation functions.
+ */
+static int
+nexus_dmamap_create(bus_dma_tag_t dmat, int flags, bus_dmamap_t *mapp)
+{
+
+	/* Not much to do...? */
+	*mapp = malloc(sizeof(**mapp), M_DEVBUF, M_WAITOK | M_ZERO);
+	dmat->map_count++;
+	return (0);
+}
+
+/*
+ * Common function for DMA map destruction.  May be called by bus-specific
+ * DMA map destruction functions.
+ */
+static int
+nexus_dmamap_destroy(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+
+	free(map, M_DEVBUF);
+	dmat->map_count--;
+	return (0);
+}
+
+#define BUS_DMAMAP_NSEGS ((BUS_SPACE_MAXSIZE / PAGE_SIZE) + 1)
+
+/*
+ * Common function for loading a DMA map with a linear buffer.  May
+ * be called by bus-specific DMA map load functions.
+ *
+ * Most SPARCs have IOMMUs in the bus controllers.  In those cases
+ * they only need one segment and will use virtual addresses for DVMA.
+ * Those bus controllers should intercept these vectors and should
+ * *NEVER* call nexus_dmamap_load() which is used only by devices that
+ * bypass DVMA.
+ */
+static int
+nexus_dmamap_load(bus_dma_tag_t dmat, bus_dmamap_t map, void *buf,
+    bus_size_t buflen, bus_dmamap_callback_t *callback, void *callback_arg,
+    int flags)
+{
+	vm_offset_t vaddr;
+	vm_offset_t paddr;
+#ifdef __GNUC__
+	bus_dma_segment_t dm_segments[dmat->nsegments];
+#else
+	bus_dma_segment_t dm_segments[BUS_DMAMAP_NSEGS];
+#endif
+	bus_dma_segment_t *sg;
+	int seg;
+	int error;
+	vm_offset_t nextpaddr;
+	bus_size_t size;
+
+	error = 0;
+
+	vaddr = (vm_offset_t)buf;
+	sg = &dm_segments[0];
+	seg = 1;
+	sg->ds_len = 0;
+
+	map->buf = buf;
+	map->buflen = buflen;
+	map->start = (bus_addr_t)buf;
+
+	nextpaddr = 0;
+	do {
+		paddr = pmap_kextract(vaddr);
+		size = PAGE_SIZE - (paddr & PAGE_MASK);
+		if (size > buflen)
+			size = buflen;
+
+		if (sg->ds_len == 0) {
+			sg->ds_addr = paddr;
+			sg->ds_len = size;
+		} else if (paddr == nextpaddr) {
+			sg->ds_len += size;
+		} else {
+			/* Go to the next segment */
+			sg++;
+			seg++;
+			if (seg > dmat->nsegments)
+				break;
+			sg->ds_addr = paddr;
+			sg->ds_len = size;
+		}
+		vaddr += size;
+		nextpaddr = paddr + size;
+		buflen -= size;
+	} while (buflen > 0);
+
+	if (buflen != 0) {
+		printf("bus_dmamap_load: Too many segs! buf_len = 0x%lx\n",
+		       (u_long)buflen);
+		error = EFBIG;
+	}
+
+	(*callback)(callback_arg, dm_segments, seg, error);
+
+	return (0);
+}
+
+/*
+ * Common function for unloading a DMA map.  May be called by
+ * bus-specific DMA map unload functions.
+ */
+static void
+nexus_dmamap_unload(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+
+	/* Nothing to do...? */
+}
+
+/*
+ * Common function for DMA map synchronization.  May be called
+ * by bus-specific DMA map synchronization functions.
+ */
+static void
+nexus_dmamap_sync(bus_dma_tag_t dmat, bus_dmamap_t map, bus_dmasync_op_t op)
+{
+
+	/*
+	 * We sync out our caches, but the bus must do the same.
+	 *
+	 * Actually a #Sync is expensive.  We should optimize.
+	 */
+	if ((op == BUS_DMASYNC_PREREAD) || (op == BUS_DMASYNC_PREWRITE)) {
+		/* 
+		 * Don't really need to do anything, but flush any pending
+		 * writes anyway. 
+		 */
+		membar(Sync);
+	}
+	if (op == BUS_DMASYNC_POSTREAD) {
+		/*
+		 * Invalidate the caches (it is unclear whether that is really
+		 * needed. The manual only mentions that PCI transactions are
+		 * cache coherent).
+		 */
+		ecache_flush((vm_offset_t)map->buf,
+		    (vm_offset_t)map->buf + map->buflen - 1);
+	}
+	if (op == BUS_DMASYNC_POSTWRITE) {
+		/* Nothing to do.  Handled by the bus controller. */
+	}
+}
+
+/*
+ * Helper functions for buses that use their private dmamem_alloc/dmamem_free
+ * versions.
+ * These differ from the dmamap_alloc() functions in that they create a tag
+ * that is specifically for use with dmamem_alloc'ed memory.
+ * These are primitive now, but I expect that some fields of the map will need
+ * to be filled soon.
+ */
+int
+sparc64_dmamem_alloc_map(bus_dma_tag_t dmat, bus_dmamap_t *mapp)
+{
+
+	*mapp = malloc(sizeof(**mapp), M_DEVBUF, M_WAITOK | M_ZERO);
+	if (*mapp == NULL)
+		return (ENOMEM);
+	
+	dmat->map_count++;
+	return (0);
+}
+
+void
+sparc64_dmamem_free_map(bus_dma_tag_t dmat, bus_dmamap_t map)
+{
+
+	free(map, M_DEVBUF);
+	dmat->map_count--;
+}
+
+/*
+ * Common function for DMA-safe memory allocation.  May be called
+ * by bus-specific DMA memory allocation functions.
+ */
+static int
+nexus_dmamem_alloc(bus_dma_tag_t dmat, void **vaddr, int flags,
+    bus_dmamap_t *mapp)
+{
+	
+	if ((dmat->maxsize <= PAGE_SIZE)) {
+		*vaddr = malloc(dmat->maxsize, M_DEVBUF,
+		    (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK);
+	} else {
+		/*
+		 * XXX: Use contigmalloc until it is merged into this facility
+		 * and handles multi-seg allocations.  Nobody is doing multi-seg
+		 * allocations yet though.
+		 */
+		*vaddr = contigmalloc(dmat->maxsize, M_DEVBUF,
+		    (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK,
+		    0ul, dmat->lowaddr, dmat->alignment ? dmat->alignment : 1UL,
+		    dmat->boundary);
+	}
+	if (*vaddr == NULL) {
+		free(*mapp, M_DEVBUF);
+		return (ENOMEM);
+	}
+	return (0);
+}
+
+/*
+ * Common function for freeing DMA-safe memory.  May be called by
+ * bus-specific DMA memory free functions.
+ */
+static void
+nexus_dmamem_free(bus_dma_tag_t dmat, void *vaddr, bus_dmamap_t map)
+{
+
+	sparc64_dmamem_free_map(dmat, map);
+	if ((dmat->maxsize <= PAGE_SIZE))
+		free(vaddr, M_DEVBUF);
+	else
+		contigfree(vaddr, dmat->maxsize, M_DEVBUF);
+}
+
+struct bus_dma_tag nexus_dmatag = {
+	NULL,
+	NULL,
+	8,
+	0,
+	0,
+	0x3ffffffff,
+	NULL,		/* XXX */
+	NULL,
+	0x3ffffffff,	/* XXX */
+	0xff,		/* XXX */
+	0xffffffff,	/* XXX */
+	0,
+	0,
+	0,
+	nexus_dmamap_create,
+	nexus_dmamap_destroy,
+	nexus_dmamap_load,
+	nexus_dmamap_unload,
+	nexus_dmamap_sync,
+
+	nexus_dmamem_alloc,
+	nexus_dmamem_free,
+};
+
+/*
+ * Helpers to map/unmap bus memory
+ */
+int
+sparc64_bus_mem_map(bus_type_t iospace, bus_addr_t addr,
+    bus_size_t size, int flags, vm_offset_t vaddr, void **hp)
+{
+	vm_offset_t v;
+	vm_offset_t pa;
+	u_long pm_flags;
+
+	size = round_page(size);
+	if (size == 0) {
+		printf("sparc64_bus_map: zero size\n");
+		return (EINVAL);
+	}
+	switch (iospace) {
+	case PCI_CONFIG_BUS_SPACE:
+	case PCI_IO_BUS_SPACE:
+	case PCI_MEMORY_BUS_SPACE:
+		pm_flags = TD_IE;
+		break;
+	default:
+		pm_flags = 0;
+		break;
+	}
+
+	if (!(flags & BUS_SPACE_MAP_CACHEABLE))
+		pm_flags |= TD_E;
+
+	if (vaddr != NULL)
+		v = trunc_page(vaddr);
+	else {
+		if ((v = kmem_alloc_nofault(kernel_map, size)) == NULL)
+			panic("sparc64_bus_map: cannot allocate virtual "
+			    "memory");
+	}
+
+	/* note: preserve page offset */
+	*hp = (void *)(v | ((u_long)addr & PAGE_MASK));
+
+	pa = trunc_page(addr);
+	if ((flags & BUS_SPACE_MAP_READONLY) == 0)
+		pm_flags |= TD_W;
+
+	do {
+		pmap_kenter_flags(v, pa, pm_flags);
+		v += PAGE_SIZE;
+		pa += PAGE_SIZE;
+	} while ((size -= PAGE_SIZE) > 0);
+	return (0);
+}
+
+int
+sparc64_bus_mem_unmap(void *bh, bus_size_t size)
+{
+	vm_offset_t va ;
+	vm_offset_t endva;
+
+	va = trunc_page((vm_offset_t)bh);
+	endva = va + round_page(size);
+	for (; va < endva; va += PAGE_SIZE)
+		pmap_kremove(va);
+	kmem_free(kernel_map, va, size);
+	return (0);
+}
+
+/*
+ * Fake up a bus tag, for use by console drivers in early boot when the regular
+ * means to allocate resources are not yet available.
+ * Note that these tags are not eligible for bus_space_barrier operations.
+ * Addr is the physical address of the desired start of the handle.
+ */
+bus_space_handle_t
+sparc64_fake_bustag(int space, bus_addr_t addr, struct bus_space_tag *ptag)
+{
+
+	ptag->cookie = NULL;
+	ptag->parent = NULL;
+	ptag->type = space;
+	ptag->bus_barrier = NULL;
+	return (addr);
+}
+
+/*
+ * Base bus space handlers.
+ */
+static void nexus_bus_barrier(bus_space_tag_t, bus_space_handle_t,
+    bus_size_t, bus_size_t, int);
+
+static void
+nexus_bus_barrier(bus_space_tag_t t, bus_space_handle_t h, bus_size_t offset,
+    bus_size_t size, int flags)
+{
+
+	/* 
+	 * We have lots of alternatives depending on whether we're
+	 * synchronizing loads with loads, loads with stores, stores
+	 * with loads, or stores with stores.  The only ones that seem
+	 * generic are #Sync and #MemIssue.  I'll use #Sync for safety.
+	 */
+	switch(flags) {
+	case BUS_SPACE_BARRIER_READ | BUS_SPACE_BARRIER_WRITE:
+	case BUS_SPACE_BARRIER_READ:
+	case BUS_SPACE_BARRIER_WRITE:
+		membar(Sync);
+		break;
+	default:
+		panic("sparc64_bus_barrier: unknown flags");
+	}
+	return;
+}
+
+struct bus_space_tag nexus_bustag = {
+	NULL,				/* cookie */
+	NULL,				/* parent bus tag */
+	UPA_BUS_SPACE,			/* type */
+	nexus_bus_barrier,		/* bus_space_barrier */
+};