Support for the UltraSpac DVMA MMU (IOMMU), ported from NetBSD.

3 files changed, 965 insertions, 0 deletions

diff --git a/sys/sparc64/include/iommureg.h b/sys/sparc64/include/iommureg.h
new file mode 100644
index 0000000..10b5cea
--- /dev/null
+++ b/sys/sparc64/include/iommureg.h
@@ -0,0 +1,156 @@
+/*
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Lawrence Berkeley Laboratory.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: @(#)sbusreg.h	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: iommureg.h,v 1.6 2001/07/20 00:07:13 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_IOMMUREG_H_
+#define _MACHINE_IOMMUREG_H_
+
+/*
+ * UltraSPARC IOMMU registers, common to both the sbus and PCI
+ * controllers.
+ */
+
+/* iommmu registers */
+struct iommureg {
+	u_int64_t	iommu_cr;	/* IOMMU control register */
+	u_int64_t	iommu_tsb;	/* IOMMU TSB base register */
+	u_int64_t	iommu_flush;	/* IOMMU flush register */
+};
+
+/* streaming buffer registers */
+struct iommu_strbuf {
+	u_int64_t	strbuf_ctl;	/* streaming buffer control reg */
+	u_int64_t	strbuf_pgflush;	/* streaming buffer page flush */
+	u_int64_t	strbuf_flushsync;/* streaming buffer flush sync */
+};
+
+/* streaming buffer control register */
+#define STRBUF_EN		0x0000000000000001UL
+#define STRBUF_D		0x0000000000000002UL
+
+#define	IOMMU_BITS		34
+#define	IOMMU_MAXADDR		(1UL << IOMMU_BITS)
+
+/*
+ * control register bits
+ */
+/* Nummber of entries in IOTSB */
+#define IOMMUCR_TSB1K		0x0000000000000000UL
+#define IOMMUCR_TSB2K		0x0000000000010000UL
+#define IOMMUCR_TSB4K		0x0000000000020000UL
+#define IOMMUCR_TSB8K		0x0000000000030000UL
+#define IOMMUCR_TSB16K		0x0000000000040000UL
+#define IOMMUCR_TSB32K		0x0000000000050000UL
+#define IOMMUCR_TSB64K		0x0000000000060000UL
+#define IOMMUCR_TSB128K		0x0000000000070000UL
+/* Mask for above */
+#define IOMMUCR_TSBMASK		0xfffffffffff8ffffUL
+/* 8K iommu page size */
+#define IOMMUCR_8KPG		0x0000000000000000UL
+/* 64K iommu page size */
+#define IOMMUCR_64KPG		0x0000000000000004UL
+/* Diag enable */
+#define IOMMUCR_DE		0x0000000000000002UL
+/* Enable IOMMU */
+#define IOMMUCR_EN		0x0000000000000001UL
+
+/*
+ * IOMMU stuff
+ */
+/* Entry valid */
+#define	IOTTE_V			0x8000000000000000UL
+/* 8K or 64K page? */
+#define IOTTE_64K		0x2000000000000000UL
+#define IOTTE_8K		0x0000000000000000UL
+/* Is page streamable? */
+#define IOTTE_STREAM		0x1000000000000000UL
+/* Accesses to same bus segment? */
+#define	IOTTE_LOCAL		0x0800000000000000UL
+/* Let's assume this is correct */
+#define IOTTE_PAMASK		0x000001ffffffe000UL
+/* Accesses to cacheable space */
+#define IOTTE_C			0x0000000000000010UL
+/* Writeable */
+#define IOTTE_W			0x0000000000000002UL
+
+/*
+ * On sun4u each bus controller has a separate IOMMU.  The IOMMU has 
+ * a TSB which must be page aligned and physically contiguous.  Mappings
+ * can be of 8K IOMMU pages or 64K IOMMU pages.  We use 8K for compatibility
+ * with the CPU's MMU.
+ *
+ * On sysio, psycho, and psycho+, IOMMU TSBs using 8K pages can map the
+ * following size segments:
+ *
+ *	VA size		VA base		TSB size	tsbsize
+ *	--------	--------	---------	-------
+ *	8MB		ff800000	8K		0
+ *	16MB		ff000000	16K		1
+ *	32MB		fe000000	32K		2
+ *	64MB		fc000000	64K		3
+ *	128MB		f8000000	128K		4
+ *	256MB		f0000000	256K		5
+ *	512MB		e0000000	512K		6
+ *	1GB		c0000000	1MB		7
+ *
+ * Unfortunately, sabres on UltraSPARC IIi and IIe processors does not use
+ * this scheme to determine the IOVA base address.  Instead, bits 31-29 are
+ * used to check against the Target Address Space register in the IIi and
+ * the the IOMMU is used if they hit.  God knows what goes on in the IIe.
+ *
+ */
+
+#define IOTSB_VEND		(~PAGE_MASK)
+#define IOTSB_VSTART(sz)	(u_int)(IOTSB_VEND << ((sz) + 10)) 
+
+#define MAKEIOTTE(pa,w,c,s)						\
+	(((pa) & IOTTE_PAMASK) | ((w) ? IOTTE_W : 0) |			\
+	((c) ? IOTTE_C : 0) | ((s) ? IOTTE_STREAM : 0) |		\
+	(IOTTE_V | IOTTE_8K))
+#define IOTSBSLOT(va,sz)						\
+	((u_int)(((vm_offset_t)(va)) - (is->is_dvmabase)) >> PAGE_SHIFT)
+
+#endif /* !_MACHINE_IOMMUREG_H_ */
diff --git a/sys/sparc64/include/iommuvar.h b/sys/sparc64/include/iommuvar.h
new file mode 100644
index 0000000..81fe19d
--- /dev/null
+++ b/sys/sparc64/include/iommuvar.h
@@ -0,0 +1,79 @@
+/*
+ * Copyright (c) 1999 Matthew R. Green
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: iommuvar.h,v 1.7 2001/07/20 00:07:13 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _MACHINE_IOMMUVAR_H_
+#define _MACHINE_IOMMUVAR_H_
+
+/*
+ * per-IOMMU state
+ */
+struct iommu_state {
+	vm_offset_t		is_ptsb;	/* TSB physical address */
+	u_int64_t		*is_tsb;	/* TSB virtual address */
+	int			is_tsbsize;	/* 0 = 8K, ... */
+	u_int64_t		is_dvmabase;
+	int64_t			is_cr;		/* IOMMU control regiter value */
+	struct rman		is_dvma_rman;	/* DVMA map for this instance */
+
+	vm_offset_t		is_flushpa;	/* used to flush the SBUS */
+	/* Needs to be volatile or egcs optimizes away loads */
+	volatile int64_t	is_flush;
+
+	/* copies of our parents state, to allow us to be self contained */
+	bus_space_tag_t		is_bustag;	/* our bus tag */
+	struct iommureg		*is_iommu;	/* IOMMU registers */
+	struct iommu_strbuf	*is_sb;		/* streaming buffer */
+	u_int64_t		*is_dtag;	/* tag diagnostics access */
+	u_int64_t		*is_ddram;	/* data ram diag. access */
+	u_int64_t		*is_dqueue;	/* LRU queue diag. access */
+	u_int64_t		*is_dva;	/* VA diag. register */
+	u_int64_t		*is_dtcmp;	/* tag compare diag. access */
+};
+
+/* interfaces for PCI/SBUS code */
+void iommu_init __P((char *, struct iommu_state *, int, u_int32_t));
+void iommu_reset __P((struct iommu_state *));
+void iommu_enter __P((struct iommu_state *, vm_offset_t, vm_offset_t, int));
+void iommu_remove __P((struct iommu_state *, vm_offset_t, size_t));
+
+int iommu_dvmamem_alloc __P((bus_dma_tag_t, struct iommu_state *, void **, int,
+    bus_dmamap_t *));
+void iommu_dvmamem_free __P((bus_dma_tag_t, struct iommu_state *, void *,
+    bus_dmamap_t));
+int iommu_dvmamap_load __P((bus_dma_tag_t, struct iommu_state *, bus_dmamap_t,
+    void *, bus_size_t, bus_dmamap_callback_t *, void *, int));
+void iommu_dvmamap_unload __P((bus_dma_tag_t, struct iommu_state *,
+    bus_dmamap_t));
+void iommu_dvmamap_sync __P((bus_dma_tag_t, struct iommu_state *, bus_dmamap_t,
+    bus_dmasync_op_t));
+
+#endif /* !_MACHINE_IOMMUVAR_H_ */
diff --git a/sys/sparc64/sparc64/iommu.c b/sys/sparc64/sparc64/iommu.c
new file mode 100644
index 0000000..6111da9
--- /dev/null
+++ b/sys/sparc64/sparc64/iommu.c
@@ -0,0 +1,730 @@
+/*
+ * Copyright (c) 1999, 2000 Matthew R. Green
+ * Copyright (c) 2001 Thomas Moestl
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. The name of the author may not be used to endorse or promote products
+ *    derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
+ * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
+ * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
+ * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+ * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+/*-
+ * Copyright (c) 1998 The NetBSD Foundation, Inc.
+ * All rights reserved.
+ *
+ * This code is derived from software contributed to The NetBSD Foundation
+ * by Paul Kranenburg.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *        This product includes software developed by the NetBSD
+ *        Foundation, Inc. and its contributors.
+ * 4. Neither the name of The NetBSD Foundation nor the names of its
+ *    contributors may be used to endorse or promote products derived
+ *    from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
+ * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
+ * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
+ * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
+ * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
+ * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
+ * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
+ * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
+ * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
+ * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
+ * POSSIBILITY OF SUCH DAMAGE.
+ */
+/*
+ * Copyright (c) 1992, 1993
+ *	The Regents of the University of California.  All rights reserved.
+ *
+ * This software was developed by the Computer Systems Engineering group
+ * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and
+ * contributed to Berkeley.
+ *
+ * All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Lawrence Berkeley Laboratory.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ *    must display the following acknowledgement:
+ *	This product includes software developed by the University of
+ *	California, Berkeley and its contributors.
+ * 4. Neither the name of the University nor the names of its contributors
+ *    may be used to endorse or promote products derived from this software
+ *    without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	from: NetBSD: sbus.c,v 1.13 1999/05/23 07:24:02 mrg Exp
+ *	from: @(#)sbus.c	8.1 (Berkeley) 6/11/93
+ *	from: NetBSD: iommu.c,v 1.37 2001/08/06 22:02:58 eeh Exp
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * UltraSPARC IOMMU support; used by both the sbus and pci code.
+ */
+#include "opt_iommu.h"
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <machine/bus.h>
+#include <machine/cache.h>
+#include <machine/iommureg.h>
+#include <machine/pmap.h>
+#include <machine/resource.h>
+
+#include <sys/rman.h>
+
+#include <machine/iommuvar.h>
+
+#ifdef IOMMU_DEBUG
+#define IDB_BUSDMA	0x1
+#define IDB_IOMMU	0x2
+#define IDB_INFO	0x4
+#define	IDB_SYNC	0x8
+int iommudebug = 0xff;
+#define DPRINTF(l, s)   do { if (iommudebug & l) printf s; } while (0)
+#else
+#define DPRINTF(l, s)
+#endif
+
+MALLOC_DEFINE(M_IOMMU, "dvmamem", "IOMMU DVMA Buffers");
+
+#define iommu_strbuf_flush(i,v)		(i)->is_sb->strbuf_pgflush = (v)
+
+static	int iommu_strbuf_flush_done(struct iommu_state *);
+#ifdef IOMMU_DIAG
+static 	void iommu_diag(struct iommu_state *, vm_offset_t va);
+#endif
+
+#define	IO_PAGE_SIZE		PAGE_SIZE
+#define	IO_PAGE_MASK		PAGE_MASK
+#define	round_io_page(x)	round_page(x)
+#define	trunc_io_page(x)	trunc_page(x)
+
+/*
+ * initialise the UltraSPARC IOMMU (SBUS or PCI):
+ *	- allocate and setup the iotsb.
+ *	- enable the IOMMU
+ *	- initialise the streaming buffers (if they exist)
+ *	- create a private DVMA map.
+ */
+void
+iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase)
+{
+	vm_size_t size;
+
+	/*
+	 * Setup the iommu.
+	 *
+	 * The sun4u iommu is part of the SBUS or PCI controller so we
+	 * will deal with it here..
+	 *
+	 * The IOMMU address space always ends at 0xffffe000, but the starting
+	 * address depends on the size of the map.  The map size is 1024 * 2 ^
+	 * is->is_tsbsize entries, where each entry is 8 bytes.  The start of
+	 * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)).
+	 */
+	is->is_cr = (tsbsize << 16) | IOMMUCR_EN;
+	is->is_tsbsize = tsbsize;
+	is->is_dvmabase = iovabase;
+	if (iovabase == -1)
+		is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
+
+	/*
+	 * Allocate memory for I/O pagetables.  They need to be physically
+	 * contiguous.
+	 */
+	size = PAGE_SIZE << is->is_tsbsize;
+	is->is_tsb = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, ~0UL, PAGE_SIZE,
+	    0);
+	if (is->is_tsb == 0)
+		panic("iommu_init: no memory");
+	is->is_ptsb = pmap_kextract((vm_offset_t)is->is_tsb);
+
+	bzero(is->is_tsb, size);
+
+#ifdef IOMMU_DEBUG
+	if (iommudebug & IDB_INFO) {
+		/* Probe the iommu */
+		struct iommureg *regs = is->is_iommu;
+
+		printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n",
+		    (u_long)&regs->iommu_cr,
+		    (u_long)&regs->iommu_tsb,
+		    (u_long)&regs->iommu_flush);
+		printf("iommu cr=%lx tsb=%lx\n",
+		    (unsigned long)regs->iommu_cr,
+		    (unsigned long)regs->iommu_tsb);
+		printf("TSB base %p phys %lx\n", (void *)is->is_tsb,
+		    (unsigned long)is->is_ptsb);
+		DELAY(1000000); /* 1 s */
+	}
+#endif
+
+	/*
+	 * Initialize streaming buffer, if it is there.
+	 */
+	if (is->is_sb)
+		is->is_flushpa = pmap_kextract((vm_offset_t)&is->is_flush);
+
+	/*
+	 * now actually start up the IOMMU
+	 */
+	iommu_reset(is);
+
+	/*
+	 * Now all the hardware's working we need to setup dvma resource
+	 * management.
+	 */
+	printf("DVMA map: %lx to %lx\n",
+	    is->is_dvmabase, is->is_dvmabase + (size << 10) - 1);
+
+	is->is_dvma_rman.rm_type = RMAN_ARRAY;
+	is->is_dvma_rman.rm_descr = "DVMA Memory";
+	if (rman_init(&is->is_dvma_rman) != 0 ||
+	    rman_manage_region(&is->is_dvma_rman,
+	    is->is_dvmabase / IO_PAGE_SIZE,
+	    (is->is_dvmabase + (size << 10)) / IO_PAGE_SIZE))
+		panic("iommu_init: can't initialize dvma rman");
+}
+
+/*
+ * Streaming buffers don't exist on the UltraSPARC IIi; we should have
+ * detected that already and disabled them.  If not, we will notice that
+ * they aren't there when the STRBUF_EN bit does not remain.
+ */
+void
+iommu_reset(struct iommu_state *is)
+{
+
+	is->is_iommu->iommu_tsb = is->is_ptsb;
+	/* Enable IOMMU in diagnostic mode */
+	is->is_iommu->iommu_cr = is->is_cr | IOMMUCR_DE;
+
+	if (!is->is_sb)
+		return;
+
+	/* Enable diagnostics mode? */
+	is->is_sb->strbuf_ctl = STRBUF_EN;
+
+	/* No streaming buffers? Disable them */
+	if (is->is_sb->strbuf_ctl == 0)
+		is->is_sb = 0;
+}
+
+/*
+ * Here are the iommu control routines.
+ */
+void
+iommu_enter(struct iommu_state *is, vm_offset_t va, vm_offset_t pa, int flags)
+{
+	int64_t tte;
+
+#ifdef DIAGNOSTIC
+	if (va < is->is_dvmabase)
+		panic("iommu_enter: va %#lx not in DVMA space", va);
+#endif
+
+	tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE),
+	    !(flags & BUS_DMA_NOCACHE), (flags & BUS_DMA_STREAMING));
+
+	/* Is the streamcache flush really needed? */
+	if (is->is_sb) {
+		iommu_strbuf_flush(is, va);
+		iommu_strbuf_flush_done(is);
+	}
+	DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n",
+	    (int)IOTSBSLOT(va, is->is_tsbsize), (void *)(u_long)va));
+	is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] = tte;
+	is->is_iommu->iommu_flush = va;
+	DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n",
+	    va, (long)pa, (u_long)IOTSBSLOT(va, is->is_tsbsize),
+	    (void *)(u_long)&is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)],
+	    (u_long)tte));
+#ifdef IOMMU_DIAGA
+	iommu_diag(is, va);
+#endif
+}
+
+/*
+ * iommu_remove: removes mappings created by iommu_enter
+ * Only demap from IOMMU if flag is set.
+ *
+ * XXX: this function needs better internal error checking.
+ */
+void
+iommu_remove(struct iommu_state *is, vm_offset_t va, vm_size_t len)
+{
+
+#ifdef IOMMU_DIAG
+	iommu_diag(is, va);
+#endif
+#ifdef DIAGNOSTIC
+	if (va < is->is_dvmabase)
+		panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va);
+	if ((long)(va + len) < (long)va) {
+		panic("iommu_remove: va 0x%lx + len 0x%lx wraps",
+		    (long)va, (long)len);
+	}
+	if (len & ~0xfffffff)
+		panic("iommu_remove: ridiculous len 0x%lx", (u_long)len);
+#endif
+
+	va = trunc_io_page(va);
+	DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n",
+	    va, (u_long)IOTSBSLOT(va, is->is_tsbsize),
+	    &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]));
+	while (len > 0) {
+		DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d for va "
+		    "%p size %lx\n", (int)IOTSBSLOT(va, is->is_tsbsize),
+		    (void *)(u_long)va, (u_long)len));
+		if (is->is_sb) {
+			DPRINTF(IDB_IOMMU, ("iommu_remove: flushing va %p "
+			    "TSB[%lx]@%p=%lx, %lu bytes left\n",
+			    (void *)(u_long)va,
+			    (long)IOTSBSLOT(va, is->is_tsbsize),
+			    (void *)(u_long)&is->is_tsb[
+				    IOTSBSLOT(va, is->is_tsbsize)],
+			    (long)(is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]),
+			    (u_long)len));
+			iommu_strbuf_flush(is, va);
+			if (len <= IO_PAGE_SIZE)
+				iommu_strbuf_flush_done(is);
+			DPRINTF(IDB_IOMMU, ("iommu_remove: flushed va %p "
+			    "TSB[%lx]@%p=%lx, %lu bytes left\n",
+			    (void *)(u_long)va,
+			    (long)IOTSBSLOT(va, is->is_tsbsize),
+			    (void *)(u_long)&is->is_tsb[
+				    IOTSBSLOT(va,is->is_tsbsize)],
+			    (long)(is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]),
+			    (u_long)len));
+		}
+
+		if (len <= IO_PAGE_SIZE)
+			len = 0;
+		else
+			len -= IO_PAGE_SIZE;
+
+		is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] = 0;
+		is->is_iommu->iommu_flush = va;
+		va += IO_PAGE_SIZE;
+	}
+}
+
+static int
+iommu_strbuf_flush_done(struct iommu_state *is)
+{
+	struct timeval cur, end;
+
+	if (!is->is_sb)
+		return (0);
+
+	/*
+	 * Streaming buffer flushes:
+	 *
+	 *   1 Tell strbuf to flush by storing va to strbuf_pgflush.  If
+	 *     we're not on a cache line boundary (64-bits):
+	 *   2 Store 0 in flag
+	 *   3 Store pointer to flag in flushsync
+	 *   4 wait till flushsync becomes 0x1
+	 *
+	 * If it takes more than .5 sec, something
+	 * went wrong.
+	 */
+	is->is_flush = 0;
+	membar(StoreStore);
+	is->is_sb->strbuf_flushsync = is->is_flushpa;
+	membar(Sync);	/* Prolly not needed at all. */
+
+	microtime(&cur);
+	end.tv_sec = 0;
+	end.tv_usec = 500000;
+	timevaladd(&end, &cur);
+
+	DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx at va = %lx "
+	    "pa = %lx now=%lx:%lx\n",   (long)is->is_flush, (long)&is->is_flush,
+	    (long)is->is_flushpa, cur.tv_sec, cur.tv_usec));
+	while (!is->is_flush && timevalcmp(&cur, &end, <=))
+		microtime(&cur);
+
+#ifdef DIAGNOSTIC
+	if (!is->is_flush) {
+		panic("iommu_strbuf_flush_done: flush timeout %p at %p",
+		    (void *)(u_long)is->is_flush,
+		    (void *)(u_long)is->is_flushpa);
+	}
+#endif
+	DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n"));
+	return (is->is_flush);
+}
+
+int
+iommu_dvmamem_alloc(bus_dma_tag_t t, struct iommu_state *is, void **vaddr,
+    int flags, bus_dmamap_t *mapp)
+{
+	int error;
+
+	/*
+	 * XXX: This will break for 32 bit transfers on machines with more than
+	 * 16G (2 << 34 bytes) of memory.
+	 */
+	if ((error = sparc64_dmamem_alloc_map(t, mapp)) != 0)
+		return (error);
+	if ((*vaddr = malloc(t->maxsize, M_IOMMU,
+	    (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK)) == NULL)
+		return (ENOMEM);
+	return (0);
+}
+
+void
+iommu_dvmamem_free(bus_dma_tag_t t, struct iommu_state *is, void *vaddr,
+    bus_dmamap_t map)
+{
+
+	sparc64_dmamem_free_map(t, map);
+	free(vaddr, M_IOMMU);
+}
+
+#define BUS_DMAMAP_NSEGS ((BUS_SPACE_MAXSIZE / PAGE_SIZE) + 1)
+
+/*
+ * IOMMU DVMA operations, common to SBUS and PCI.
+ */
+int
+iommu_dvmamap_load(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map,
+    void *buf, bus_size_t buflen, bus_dmamap_callback_t *cb, void *cba,
+    int flags)
+{
+#ifdef __GNUC__
+	bus_dma_segment_t sgs[t->nsegments];
+#else
+	bus_dma_segment_t sgs[BUS_DMAMAP_NSEGS];
+#endif
+	bus_size_t sgsize;
+	vm_offset_t curaddr;
+	u_long dvmaddr;
+	bus_size_t align, bound;
+	vm_offset_t vaddr;
+	u_long astart, aoffs;
+	int error, sgcnt;
+
+	if (map->res) {
+		/* Already in use?? */
+#ifdef DIAGNOSTIC
+		printf("iommu_dvmamap_load: map still in use\n");
+#endif
+		bus_dmamap_unload(t, map);
+	}
+	if (buflen > t->maxsize) {
+		DPRINTF(IDB_BUSDMA,
+		    ("iommu_dvmamap_load(): error %d > %d -- "
+		     "map size exceeded!\n", (int)buflen, (int)map->buflen));
+		return (EINVAL);
+	}
+
+	vaddr = (vm_offset_t)buf;
+	map->buf = buf;
+	map->buflen = buflen;
+
+	/*
+	 * Allocate a virtual region in the dvma map. Alignment to a page
+	 * boundary is always enforced.
+	 */
+	align = t->alignment / IO_PAGE_SIZE;
+	sgsize = round_io_page(buflen + ((int)vaddr & PAGE_MASK)) /
+	    IO_PAGE_SIZE;
+	/*
+	 * XXX: This is a bad kluge: the resource manager does not support
+	 * boundaries for allocations. To support this feature anyway, we
+	 * allocate at size + lcm(boundary, align) bytes (which is an upper
+	 * boundary to avoid more complex calculations), select the right
+	 * subregion, free and reallocate. There is a race in this, too...
+	 *
+	 * Should smaller boundaries be inforced using bounce buffers?
+	 */
+	if (t->boundary > 0 && (buflen > t->boundary ||
+	    t->boundary < IO_PAGE_SIZE))
+		panic("immu_dvmamap_load: illegal boundary specified");
+	bound = ulmax(t->boundary / IO_PAGE_SIZE, 1);
+	map->res = rman_reserve_resource(&is->is_dvma_rman, 0L, t->lowaddr,
+	    sgsize + align * bound,
+	    RF_ACTIVE | rman_make_alignment_flags(align), NULL);
+	if (map->res == NULL)
+		return (ENOMEM);
+	astart = rman_get_start(map->res);
+	/*
+	 * Move astart to the start of a boundary region. The alignment is
+	 * maintained by rman_reserve resource().
+	 */
+	if (t->boundary > 0)
+		aoffs = (bound - (astart & (bound - 1))) % bound;
+	else
+		aoffs = 0;
+	astart += aoffs;
+	if (astart + sgsize - 1 > rman_get_end(map->res)) {
+		/* This only should happen with bad a boundary.	 */
+		panic("iommo_dvmamap_load: boundary could not be "
+		    "maintained: boundary %lu (%lu), offset %lu, start %lu, "
+		    "end %lu\n", t->boundary, bound, aoffs,
+		    rman_get_start(map->res), rman_get_end(map->res));
+	}
+	rman_release_resource(map->res);
+	map->res = rman_reserve_resource(&is->is_dvma_rman, astart,
+	    astart + sgsize + align, sgsize, RF_ACTIVE |
+	    rman_make_alignment_flags(align), NULL);
+	if (map->res == NULL)
+		panic("iommo_dvmamap_load: reallocation failed");
+
+	dvmaddr = (rman_get_start(map->res) * IO_PAGE_SIZE) |
+	    (vaddr & IO_PAGE_MASK);
+	map->start = dvmaddr;
+	sgcnt = -1;
+	error = 0;
+	for (; buflen > 0; ) {
+		/*
+		 * Get the physical address for this page.
+		 */
+		curaddr = pmap_kextract((vm_offset_t)vaddr);
+
+		/*
+		 * Compute the segment size, and adjust counts.
+		 */
+		sgsize = IO_PAGE_SIZE - ((u_long)vaddr & IO_PAGE_MASK);
+		if (buflen < sgsize)
+			sgsize = buflen;
+
+		DPRINTF(IDB_BUSDMA,
+		    ("iommu_dvmamap_load: map %p loading va %p "
+			    "dva %lx at pa %lx\n",
+			    map, (void *)vaddr, (long)dvmaddr,
+			    (long)(curaddr & ~(PAGE_SIZE - 1))));
+		iommu_enter(is, trunc_io_page(dvmaddr), trunc_io_page(curaddr),
+		    flags);
+
+		if (sgcnt == -1 || sgs[sgcnt].ds_len + sgsize > t->maxsegsz) {
+			if (sgsize > t->maxsegsz) {
+				/* XXX: add fixup */
+				panic("iommu_dvmamap_load: magsegsz too "
+				    "small\n");
+			}
+			sgcnt++;
+			if (sgcnt > t->nsegments || sgcnt > BUS_DMAMAP_NSEGS) {
+				error = ENOMEM;
+				break;
+			}
+			sgs[sgcnt].ds_addr = dvmaddr;
+			sgs[sgcnt].ds_len = sgsize;
+		} else
+			sgs[sgcnt].ds_len += sgsize;
+		dvmaddr += sgsize;
+		vaddr += sgsize;
+		buflen -= sgsize;
+	}
+	(*cb)(cba, sgs, sgcnt + 1, error);
+	return (0);
+}
+
+
+void
+iommu_dvmamap_unload(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map)
+{
+	size_t len;
+	int error;
+	bus_addr_t dvmaddr;
+	bus_size_t sgsize;
+
+	/*
+	 * If the resource is already deallocated, just pass to the parent
+	 * tag.
+	 */
+	if (map->res != NULL) {
+		dvmaddr = (vm_offset_t)trunc_io_page(map->start);
+		len = map->buflen;
+		sgsize = 0;
+		if (len == 0 || dvmaddr == 0) {
+			printf("iommu_dvmamap_unload: map = %p, len = %d, "
+			    "addr = %lx\n", map, (int)len,
+			    (unsigned long)dvmaddr);
+		}
+
+		DPRINTF(IDB_BUSDMA,
+		    ("iommu_dvmamap_unload: map %p removing va %lx size "
+			"%lx\n", map, (long)dvmaddr, (long)len));
+		iommu_remove(is, dvmaddr, len);
+
+		error = rman_release_resource(map->res);
+		map->res = NULL;
+		if (error != 0)
+			printf("warning: approx. %lx of DVMA space lost\n",
+			    sgsize);
+	}
+	/* Flush the caches */
+	bus_dmamap_unload(t->parent->parent, map);
+
+}
+
+void
+iommu_dvmamap_sync(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map,
+    bus_dmasync_op_t op)
+{
+	vm_offset_t va;
+	vm_size_t len;
+
+	va = (vm_offset_t)map->buf;
+	len = map->buflen;
+	if ((op & BUS_DMASYNC_PREREAD) != 0) {
+		DPRINTF(IDB_SYNC,
+		    ("iommu_dvmamap_sync: syncing va %p len %lu "
+		    "BUS_DMASYNC_PREREAD\n", (void *)(u_long)va, (u_long)len));
+		membar(Sync);
+	}
+	if ((op & BUS_DMASYNC_POSTREAD) != 0) {
+		DPRINTF(IDB_SYNC,
+		    ("iommu_dvmamap_sync: syncing va %p len %lu "
+		    "BUS_DMASYNC_POSTREAD\n", (void *)(u_long)va, (u_long)len));
+		/* if we have a streaming buffer, flush it here first */
+		if (is->is_sb) {
+			while (len > 0) {
+				DPRINTF(IDB_BUSDMA,
+				    ("iommu_dvmamap_sync: flushing va %p, %lu "
+				    "bytes left\n", (void *)(u_long)va,
+				    (u_long)len));
+				iommu_strbuf_flush(is, va);
+				if (len <= IO_PAGE_SIZE) {
+					iommu_strbuf_flush_done(is);
+					len = 0;
+				} else
+					len -= IO_PAGE_SIZE;
+				va += IO_PAGE_SIZE;
+			}
+		}
+	}
+	if ((op & BUS_DMASYNC_PREWRITE) != 0) {
+		DPRINTF(IDB_SYNC,
+		    ("iommu_dvmamap_sync: syncing va %p len %lu "
+		     "BUS_DMASYNC_PREWRITE\n", (void *)(u_long)va, (u_long)len));
+		/* if we have a streaming buffer, flush it here first */
+		if (is->is_sb) {
+			while (len > 0) {
+				DPRINTF(IDB_BUSDMA,
+				    ("iommu_dvmamap_sync: flushing va %p, %lu "
+				    "bytes left\n", (void *)(u_long)va,
+				    (u_long)len));
+				iommu_strbuf_flush(is, va);
+				if (len <= IO_PAGE_SIZE) {
+					iommu_strbuf_flush_done(is);
+					len = 0;
+				} else
+					len -= IO_PAGE_SIZE;
+				va += IO_PAGE_SIZE;
+			}
+		}
+		membar(Sync);
+	}
+	if ((op & BUS_DMASYNC_POSTWRITE) != 0) {
+		DPRINTF(IDB_SYNC,
+		    ("iommu_dvmamap_sync: syncing va %p len %lu "
+		    "BUS_DMASYNC_POSTWRITE\n", (void *)(u_long)va,
+		    (u_long)len));
+		/* Nothing to do */
+	}
+}
+
+#ifdef IOMMU_DIAG
+
+#define	IOMMU_DTAG_VPNBITS	19
+#define	IOMMU_DTAG_VPNMASK	((1 << IOMMU_DTAG_VPNBITS) - 1)
+#define	IOMMU_DTAG_VPNSHIFT	13
+#define IOMMU_DTAG_ERRBITS	3
+#define	IOMMU_DTAG_ERRSHIFT	22
+#define	IOMMU_DTAG_ERRMASK \
+	(((1 << IOMMU_DTAG_ERRBITS) - 1) << IOMMU_DTAG_ERRSHIFT)
+
+#define	IOMMU_DDATA_PGBITS	21
+#define	IOMMU_DDATA_PGMASK	((1 << IOMMU_DDATA_PGBITS) - 1)
+#define	IOMMU_DDATA_PGSHIFT	13
+#define	IOMMU_DDATA_C		(1 << 28)
+#define	IOMMU_DDATA_V		(1 << 30)
+
+/*
+ * Perform an IOMMU diagnostic access and print the tag belonging to va.
+ */
+static void
+iommu_diag(struct iommu_state *is, vm_offset_t va)
+{
+	int i;
+	u_int64_t tag, data;
+
+	*is->is_dva = trunc_io_page(va);
+	membar(StoreStore | StoreLoad);
+	printf("iommu_diag: tte entry %#lx, tag compare register is %#lx\n",
+	    is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)], *is->is_dtcmp);
+	for (i = 0; i < 16; i++) {
+		tag = is->is_dtag[i];
+		data = is->is_ddram[i];
+		printf("iommu_diag: tag %d: %#lx, vpn %#lx, err %lx; "
+		    "data %#lx, pa %#lx, v %d, c %d\n", i,
+		    tag, (tag & IOMMU_DTAG_VPNMASK) << IOMMU_DTAG_VPNSHIFT,
+		    (tag & IOMMU_DTAG_ERRMASK) >> IOMMU_DTAG_ERRSHIFT, data,
+		    (data & IOMMU_DDATA_PGMASK) << IOMMU_DDATA_PGSHIFT,
+		    (data & IOMMU_DDATA_V) != 0, (data & IOMMU_DDATA_C) != 0);
+	}
+}
+
+#endif /* IOMMU_DIAG */