diff options
author | tmm <tmm@FreeBSD.org> | 2001-11-09 20:14:41 +0000 |
---|---|---|
committer | tmm <tmm@FreeBSD.org> | 2001-11-09 20:14:41 +0000 |
commit | 904109a4f9c52cc69f2df3836e4b3b9d4c2bf99d (patch) | |
tree | 3b79fc582c211394f4970af3c582b7f883426899 /sys | |
parent | dcba1df2157c55e7fe21c0fe64ad9975bdd3eccb (diff) | |
download | FreeBSD-src-904109a4f9c52cc69f2df3836e4b3b9d4c2bf99d.zip FreeBSD-src-904109a4f9c52cc69f2df3836e4b3b9d4c2bf99d.tar.gz |
Support for the UltraSpac DVMA MMU (IOMMU), ported from NetBSD.
Diffstat (limited to 'sys')
-rw-r--r-- | sys/sparc64/include/iommureg.h | 156 | ||||
-rw-r--r-- | sys/sparc64/include/iommuvar.h | 79 | ||||
-rw-r--r-- | sys/sparc64/sparc64/iommu.c | 730 |
3 files changed, 965 insertions, 0 deletions
diff --git a/sys/sparc64/include/iommureg.h b/sys/sparc64/include/iommureg.h new file mode 100644 index 0000000..10b5cea --- /dev/null +++ b/sys/sparc64/include/iommureg.h @@ -0,0 +1,156 @@ +/* + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * This software was developed by the Computer Systems Engineering group + * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and + * contributed to Berkeley. + * + * All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Lawrence Berkeley Laboratory. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: @(#)sbusreg.h 8.1 (Berkeley) 6/11/93 + * from: NetBSD: iommureg.h,v 1.6 2001/07/20 00:07:13 eeh Exp + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_IOMMUREG_H_ +#define _MACHINE_IOMMUREG_H_ + +/* + * UltraSPARC IOMMU registers, common to both the sbus and PCI + * controllers. + */ + +/* iommmu registers */ +struct iommureg { + u_int64_t iommu_cr; /* IOMMU control register */ + u_int64_t iommu_tsb; /* IOMMU TSB base register */ + u_int64_t iommu_flush; /* IOMMU flush register */ +}; + +/* streaming buffer registers */ +struct iommu_strbuf { + u_int64_t strbuf_ctl; /* streaming buffer control reg */ + u_int64_t strbuf_pgflush; /* streaming buffer page flush */ + u_int64_t strbuf_flushsync;/* streaming buffer flush sync */ +}; + +/* streaming buffer control register */ +#define STRBUF_EN 0x0000000000000001UL +#define STRBUF_D 0x0000000000000002UL + +#define IOMMU_BITS 34 +#define IOMMU_MAXADDR (1UL << IOMMU_BITS) + +/* + * control register bits + */ +/* Nummber of entries in IOTSB */ +#define IOMMUCR_TSB1K 0x0000000000000000UL +#define IOMMUCR_TSB2K 0x0000000000010000UL +#define IOMMUCR_TSB4K 0x0000000000020000UL +#define IOMMUCR_TSB8K 0x0000000000030000UL +#define IOMMUCR_TSB16K 0x0000000000040000UL +#define IOMMUCR_TSB32K 0x0000000000050000UL +#define IOMMUCR_TSB64K 0x0000000000060000UL +#define IOMMUCR_TSB128K 0x0000000000070000UL +/* Mask for above */ +#define IOMMUCR_TSBMASK 0xfffffffffff8ffffUL +/* 8K iommu page size */ +#define IOMMUCR_8KPG 0x0000000000000000UL +/* 64K iommu page size */ +#define IOMMUCR_64KPG 0x0000000000000004UL +/* Diag enable */ +#define IOMMUCR_DE 0x0000000000000002UL +/* Enable IOMMU */ +#define IOMMUCR_EN 0x0000000000000001UL + +/* + * IOMMU stuff + */ +/* Entry valid */ +#define IOTTE_V 0x8000000000000000UL +/* 8K or 64K page? */ +#define IOTTE_64K 0x2000000000000000UL +#define IOTTE_8K 0x0000000000000000UL +/* Is page streamable? */ +#define IOTTE_STREAM 0x1000000000000000UL +/* Accesses to same bus segment? */ +#define IOTTE_LOCAL 0x0800000000000000UL +/* Let's assume this is correct */ +#define IOTTE_PAMASK 0x000001ffffffe000UL +/* Accesses to cacheable space */ +#define IOTTE_C 0x0000000000000010UL +/* Writeable */ +#define IOTTE_W 0x0000000000000002UL + +/* + * On sun4u each bus controller has a separate IOMMU. The IOMMU has + * a TSB which must be page aligned and physically contiguous. Mappings + * can be of 8K IOMMU pages or 64K IOMMU pages. We use 8K for compatibility + * with the CPU's MMU. + * + * On sysio, psycho, and psycho+, IOMMU TSBs using 8K pages can map the + * following size segments: + * + * VA size VA base TSB size tsbsize + * -------- -------- --------- ------- + * 8MB ff800000 8K 0 + * 16MB ff000000 16K 1 + * 32MB fe000000 32K 2 + * 64MB fc000000 64K 3 + * 128MB f8000000 128K 4 + * 256MB f0000000 256K 5 + * 512MB e0000000 512K 6 + * 1GB c0000000 1MB 7 + * + * Unfortunately, sabres on UltraSPARC IIi and IIe processors does not use + * this scheme to determine the IOVA base address. Instead, bits 31-29 are + * used to check against the Target Address Space register in the IIi and + * the the IOMMU is used if they hit. God knows what goes on in the IIe. + * + */ + +#define IOTSB_VEND (~PAGE_MASK) +#define IOTSB_VSTART(sz) (u_int)(IOTSB_VEND << ((sz) + 10)) + +#define MAKEIOTTE(pa,w,c,s) \ + (((pa) & IOTTE_PAMASK) | ((w) ? IOTTE_W : 0) | \ + ((c) ? IOTTE_C : 0) | ((s) ? IOTTE_STREAM : 0) | \ + (IOTTE_V | IOTTE_8K)) +#define IOTSBSLOT(va,sz) \ + ((u_int)(((vm_offset_t)(va)) - (is->is_dvmabase)) >> PAGE_SHIFT) + +#endif /* !_MACHINE_IOMMUREG_H_ */ diff --git a/sys/sparc64/include/iommuvar.h b/sys/sparc64/include/iommuvar.h new file mode 100644 index 0000000..81fe19d --- /dev/null +++ b/sys/sparc64/include/iommuvar.h @@ -0,0 +1,79 @@ +/* + * Copyright (c) 1999 Matthew R. Green + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: NetBSD: iommuvar.h,v 1.7 2001/07/20 00:07:13 eeh Exp + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_IOMMUVAR_H_ +#define _MACHINE_IOMMUVAR_H_ + +/* + * per-IOMMU state + */ +struct iommu_state { + vm_offset_t is_ptsb; /* TSB physical address */ + u_int64_t *is_tsb; /* TSB virtual address */ + int is_tsbsize; /* 0 = 8K, ... */ + u_int64_t is_dvmabase; + int64_t is_cr; /* IOMMU control regiter value */ + struct rman is_dvma_rman; /* DVMA map for this instance */ + + vm_offset_t is_flushpa; /* used to flush the SBUS */ + /* Needs to be volatile or egcs optimizes away loads */ + volatile int64_t is_flush; + + /* copies of our parents state, to allow us to be self contained */ + bus_space_tag_t is_bustag; /* our bus tag */ + struct iommureg *is_iommu; /* IOMMU registers */ + struct iommu_strbuf *is_sb; /* streaming buffer */ + u_int64_t *is_dtag; /* tag diagnostics access */ + u_int64_t *is_ddram; /* data ram diag. access */ + u_int64_t *is_dqueue; /* LRU queue diag. access */ + u_int64_t *is_dva; /* VA diag. register */ + u_int64_t *is_dtcmp; /* tag compare diag. access */ +}; + +/* interfaces for PCI/SBUS code */ +void iommu_init __P((char *, struct iommu_state *, int, u_int32_t)); +void iommu_reset __P((struct iommu_state *)); +void iommu_enter __P((struct iommu_state *, vm_offset_t, vm_offset_t, int)); +void iommu_remove __P((struct iommu_state *, vm_offset_t, size_t)); + +int iommu_dvmamem_alloc __P((bus_dma_tag_t, struct iommu_state *, void **, int, + bus_dmamap_t *)); +void iommu_dvmamem_free __P((bus_dma_tag_t, struct iommu_state *, void *, + bus_dmamap_t)); +int iommu_dvmamap_load __P((bus_dma_tag_t, struct iommu_state *, bus_dmamap_t, + void *, bus_size_t, bus_dmamap_callback_t *, void *, int)); +void iommu_dvmamap_unload __P((bus_dma_tag_t, struct iommu_state *, + bus_dmamap_t)); +void iommu_dvmamap_sync __P((bus_dma_tag_t, struct iommu_state *, bus_dmamap_t, + bus_dmasync_op_t)); + +#endif /* !_MACHINE_IOMMUVAR_H_ */ diff --git a/sys/sparc64/sparc64/iommu.c b/sys/sparc64/sparc64/iommu.c new file mode 100644 index 0000000..6111da9 --- /dev/null +++ b/sys/sparc64/sparc64/iommu.c @@ -0,0 +1,730 @@ +/* + * Copyright (c) 1999, 2000 Matthew R. Green + * Copyright (c) 2001 Thomas Moestl + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. The name of the author may not be used to endorse or promote products + * derived from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, + * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; + * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED + * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, + * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ +/*- + * Copyright (c) 1998 The NetBSD Foundation, Inc. + * All rights reserved. + * + * This code is derived from software contributed to The NetBSD Foundation + * by Paul Kranenburg. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the NetBSD + * Foundation, Inc. and its contributors. + * 4. Neither the name of The NetBSD Foundation nor the names of its + * contributors may be used to endorse or promote products derived + * from this software without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS + * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED + * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR + * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS + * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR + * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF + * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS + * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN + * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) + * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE + * POSSIBILITY OF SUCH DAMAGE. + */ +/* + * Copyright (c) 1992, 1993 + * The Regents of the University of California. All rights reserved. + * + * This software was developed by the Computer Systems Engineering group + * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and + * contributed to Berkeley. + * + * All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Lawrence Berkeley Laboratory. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. All advertising materials mentioning features or use of this software + * must display the following acknowledgement: + * This product includes software developed by the University of + * California, Berkeley and its contributors. + * 4. Neither the name of the University nor the names of its contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from: NetBSD: sbus.c,v 1.13 1999/05/23 07:24:02 mrg Exp + * from: @(#)sbus.c 8.1 (Berkeley) 6/11/93 + * from: NetBSD: iommu.c,v 1.37 2001/08/06 22:02:58 eeh Exp + * + * $FreeBSD$ + */ + +/* + * UltraSPARC IOMMU support; used by both the sbus and pci code. + */ +#include "opt_iommu.h" + +#include <sys/param.h> +#include <sys/kernel.h> +#include <sys/systm.h> +#include <sys/malloc.h> + +#include <vm/vm.h> +#include <vm/pmap.h> + +#include <machine/bus.h> +#include <machine/cache.h> +#include <machine/iommureg.h> +#include <machine/pmap.h> +#include <machine/resource.h> + +#include <sys/rman.h> + +#include <machine/iommuvar.h> + +#ifdef IOMMU_DEBUG +#define IDB_BUSDMA 0x1 +#define IDB_IOMMU 0x2 +#define IDB_INFO 0x4 +#define IDB_SYNC 0x8 +int iommudebug = 0xff; +#define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) +#else +#define DPRINTF(l, s) +#endif + +MALLOC_DEFINE(M_IOMMU, "dvmamem", "IOMMU DVMA Buffers"); + +#define iommu_strbuf_flush(i,v) (i)->is_sb->strbuf_pgflush = (v) + +static int iommu_strbuf_flush_done(struct iommu_state *); +#ifdef IOMMU_DIAG +static void iommu_diag(struct iommu_state *, vm_offset_t va); +#endif + +#define IO_PAGE_SIZE PAGE_SIZE +#define IO_PAGE_MASK PAGE_MASK +#define round_io_page(x) round_page(x) +#define trunc_io_page(x) trunc_page(x) + +/* + * initialise the UltraSPARC IOMMU (SBUS or PCI): + * - allocate and setup the iotsb. + * - enable the IOMMU + * - initialise the streaming buffers (if they exist) + * - create a private DVMA map. + */ +void +iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase) +{ + vm_size_t size; + + /* + * Setup the iommu. + * + * The sun4u iommu is part of the SBUS or PCI controller so we + * will deal with it here.. + * + * The IOMMU address space always ends at 0xffffe000, but the starting + * address depends on the size of the map. The map size is 1024 * 2 ^ + * is->is_tsbsize entries, where each entry is 8 bytes. The start of + * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)). + */ + is->is_cr = (tsbsize << 16) | IOMMUCR_EN; + is->is_tsbsize = tsbsize; + is->is_dvmabase = iovabase; + if (iovabase == -1) + is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); + + /* + * Allocate memory for I/O pagetables. They need to be physically + * contiguous. + */ + size = PAGE_SIZE << is->is_tsbsize; + is->is_tsb = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, ~0UL, PAGE_SIZE, + 0); + if (is->is_tsb == 0) + panic("iommu_init: no memory"); + is->is_ptsb = pmap_kextract((vm_offset_t)is->is_tsb); + + bzero(is->is_tsb, size); + +#ifdef IOMMU_DEBUG + if (iommudebug & IDB_INFO) { + /* Probe the iommu */ + struct iommureg *regs = is->is_iommu; + + printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", + (u_long)®s->iommu_cr, + (u_long)®s->iommu_tsb, + (u_long)®s->iommu_flush); + printf("iommu cr=%lx tsb=%lx\n", + (unsigned long)regs->iommu_cr, + (unsigned long)regs->iommu_tsb); + printf("TSB base %p phys %lx\n", (void *)is->is_tsb, + (unsigned long)is->is_ptsb); + DELAY(1000000); /* 1 s */ + } +#endif + + /* + * Initialize streaming buffer, if it is there. + */ + if (is->is_sb) + is->is_flushpa = pmap_kextract((vm_offset_t)&is->is_flush); + + /* + * now actually start up the IOMMU + */ + iommu_reset(is); + + /* + * Now all the hardware's working we need to setup dvma resource + * management. + */ + printf("DVMA map: %lx to %lx\n", + is->is_dvmabase, is->is_dvmabase + (size << 10) - 1); + + is->is_dvma_rman.rm_type = RMAN_ARRAY; + is->is_dvma_rman.rm_descr = "DVMA Memory"; + if (rman_init(&is->is_dvma_rman) != 0 || + rman_manage_region(&is->is_dvma_rman, + is->is_dvmabase / IO_PAGE_SIZE, + (is->is_dvmabase + (size << 10)) / IO_PAGE_SIZE)) + panic("iommu_init: can't initialize dvma rman"); +} + +/* + * Streaming buffers don't exist on the UltraSPARC IIi; we should have + * detected that already and disabled them. If not, we will notice that + * they aren't there when the STRBUF_EN bit does not remain. + */ +void +iommu_reset(struct iommu_state *is) +{ + + is->is_iommu->iommu_tsb = is->is_ptsb; + /* Enable IOMMU in diagnostic mode */ + is->is_iommu->iommu_cr = is->is_cr | IOMMUCR_DE; + + if (!is->is_sb) + return; + + /* Enable diagnostics mode? */ + is->is_sb->strbuf_ctl = STRBUF_EN; + + /* No streaming buffers? Disable them */ + if (is->is_sb->strbuf_ctl == 0) + is->is_sb = 0; +} + +/* + * Here are the iommu control routines. + */ +void +iommu_enter(struct iommu_state *is, vm_offset_t va, vm_offset_t pa, int flags) +{ + int64_t tte; + +#ifdef DIAGNOSTIC + if (va < is->is_dvmabase) + panic("iommu_enter: va %#lx not in DVMA space", va); +#endif + + tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), + !(flags & BUS_DMA_NOCACHE), (flags & BUS_DMA_STREAMING)); + + /* Is the streamcache flush really needed? */ + if (is->is_sb) { + iommu_strbuf_flush(is, va); + iommu_strbuf_flush_done(is); + } + DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", + (int)IOTSBSLOT(va, is->is_tsbsize), (void *)(u_long)va)); + is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] = tte; + is->is_iommu->iommu_flush = va; + DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", + va, (long)pa, (u_long)IOTSBSLOT(va, is->is_tsbsize), + (void *)(u_long)&is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)], + (u_long)tte)); +#ifdef IOMMU_DIAGA + iommu_diag(is, va); +#endif +} + +/* + * iommu_remove: removes mappings created by iommu_enter + * Only demap from IOMMU if flag is set. + * + * XXX: this function needs better internal error checking. + */ +void +iommu_remove(struct iommu_state *is, vm_offset_t va, vm_size_t len) +{ + +#ifdef IOMMU_DIAG + iommu_diag(is, va); +#endif +#ifdef DIAGNOSTIC + if (va < is->is_dvmabase) + panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); + if ((long)(va + len) < (long)va) { + panic("iommu_remove: va 0x%lx + len 0x%lx wraps", + (long)va, (long)len); + } + if (len & ~0xfffffff) + panic("iommu_remove: ridiculous len 0x%lx", (u_long)len); +#endif + + va = trunc_io_page(va); + DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", + va, (u_long)IOTSBSLOT(va, is->is_tsbsize), + &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)])); + while (len > 0) { + DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d for va " + "%p size %lx\n", (int)IOTSBSLOT(va, is->is_tsbsize), + (void *)(u_long)va, (u_long)len)); + if (is->is_sb) { + DPRINTF(IDB_IOMMU, ("iommu_remove: flushing va %p " + "TSB[%lx]@%p=%lx, %lu bytes left\n", + (void *)(u_long)va, + (long)IOTSBSLOT(va, is->is_tsbsize), + (void *)(u_long)&is->is_tsb[ + IOTSBSLOT(va, is->is_tsbsize)], + (long)(is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]), + (u_long)len)); + iommu_strbuf_flush(is, va); + if (len <= IO_PAGE_SIZE) + iommu_strbuf_flush_done(is); + DPRINTF(IDB_IOMMU, ("iommu_remove: flushed va %p " + "TSB[%lx]@%p=%lx, %lu bytes left\n", + (void *)(u_long)va, + (long)IOTSBSLOT(va, is->is_tsbsize), + (void *)(u_long)&is->is_tsb[ + IOTSBSLOT(va,is->is_tsbsize)], + (long)(is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]), + (u_long)len)); + } + + if (len <= IO_PAGE_SIZE) + len = 0; + else + len -= IO_PAGE_SIZE; + + is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] = 0; + is->is_iommu->iommu_flush = va; + va += IO_PAGE_SIZE; + } +} + +static int +iommu_strbuf_flush_done(struct iommu_state *is) +{ + struct timeval cur, end; + + if (!is->is_sb) + return (0); + + /* + * Streaming buffer flushes: + * + * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If + * we're not on a cache line boundary (64-bits): + * 2 Store 0 in flag + * 3 Store pointer to flag in flushsync + * 4 wait till flushsync becomes 0x1 + * + * If it takes more than .5 sec, something + * went wrong. + */ + is->is_flush = 0; + membar(StoreStore); + is->is_sb->strbuf_flushsync = is->is_flushpa; + membar(Sync); /* Prolly not needed at all. */ + + microtime(&cur); + end.tv_sec = 0; + end.tv_usec = 500000; + timevaladd(&end, &cur); + + DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx at va = %lx " + "pa = %lx now=%lx:%lx\n", (long)is->is_flush, (long)&is->is_flush, + (long)is->is_flushpa, cur.tv_sec, cur.tv_usec)); + while (!is->is_flush && timevalcmp(&cur, &end, <=)) + microtime(&cur); + +#ifdef DIAGNOSTIC + if (!is->is_flush) { + panic("iommu_strbuf_flush_done: flush timeout %p at %p", + (void *)(u_long)is->is_flush, + (void *)(u_long)is->is_flushpa); + } +#endif + DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n")); + return (is->is_flush); +} + +int +iommu_dvmamem_alloc(bus_dma_tag_t t, struct iommu_state *is, void **vaddr, + int flags, bus_dmamap_t *mapp) +{ + int error; + + /* + * XXX: This will break for 32 bit transfers on machines with more than + * 16G (2 << 34 bytes) of memory. + */ + if ((error = sparc64_dmamem_alloc_map(t, mapp)) != 0) + return (error); + if ((*vaddr = malloc(t->maxsize, M_IOMMU, + (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK)) == NULL) + return (ENOMEM); + return (0); +} + +void +iommu_dvmamem_free(bus_dma_tag_t t, struct iommu_state *is, void *vaddr, + bus_dmamap_t map) +{ + + sparc64_dmamem_free_map(t, map); + free(vaddr, M_IOMMU); +} + +#define BUS_DMAMAP_NSEGS ((BUS_SPACE_MAXSIZE / PAGE_SIZE) + 1) + +/* + * IOMMU DVMA operations, common to SBUS and PCI. + */ +int +iommu_dvmamap_load(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map, + void *buf, bus_size_t buflen, bus_dmamap_callback_t *cb, void *cba, + int flags) +{ +#ifdef __GNUC__ + bus_dma_segment_t sgs[t->nsegments]; +#else + bus_dma_segment_t sgs[BUS_DMAMAP_NSEGS]; +#endif + bus_size_t sgsize; + vm_offset_t curaddr; + u_long dvmaddr; + bus_size_t align, bound; + vm_offset_t vaddr; + u_long astart, aoffs; + int error, sgcnt; + + if (map->res) { + /* Already in use?? */ +#ifdef DIAGNOSTIC + printf("iommu_dvmamap_load: map still in use\n"); +#endif + bus_dmamap_unload(t, map); + } + if (buflen > t->maxsize) { + DPRINTF(IDB_BUSDMA, + ("iommu_dvmamap_load(): error %d > %d -- " + "map size exceeded!\n", (int)buflen, (int)map->buflen)); + return (EINVAL); + } + + vaddr = (vm_offset_t)buf; + map->buf = buf; + map->buflen = buflen; + + /* + * Allocate a virtual region in the dvma map. Alignment to a page + * boundary is always enforced. + */ + align = t->alignment / IO_PAGE_SIZE; + sgsize = round_io_page(buflen + ((int)vaddr & PAGE_MASK)) / + IO_PAGE_SIZE; + /* + * XXX: This is a bad kluge: the resource manager does not support + * boundaries for allocations. To support this feature anyway, we + * allocate at size + lcm(boundary, align) bytes (which is an upper + * boundary to avoid more complex calculations), select the right + * subregion, free and reallocate. There is a race in this, too... + * + * Should smaller boundaries be inforced using bounce buffers? + */ + if (t->boundary > 0 && (buflen > t->boundary || + t->boundary < IO_PAGE_SIZE)) + panic("immu_dvmamap_load: illegal boundary specified"); + bound = ulmax(t->boundary / IO_PAGE_SIZE, 1); + map->res = rman_reserve_resource(&is->is_dvma_rman, 0L, t->lowaddr, + sgsize + align * bound, + RF_ACTIVE | rman_make_alignment_flags(align), NULL); + if (map->res == NULL) + return (ENOMEM); + astart = rman_get_start(map->res); + /* + * Move astart to the start of a boundary region. The alignment is + * maintained by rman_reserve resource(). + */ + if (t->boundary > 0) + aoffs = (bound - (astart & (bound - 1))) % bound; + else + aoffs = 0; + astart += aoffs; + if (astart + sgsize - 1 > rman_get_end(map->res)) { + /* This only should happen with bad a boundary. */ + panic("iommo_dvmamap_load: boundary could not be " + "maintained: boundary %lu (%lu), offset %lu, start %lu, " + "end %lu\n", t->boundary, bound, aoffs, + rman_get_start(map->res), rman_get_end(map->res)); + } + rman_release_resource(map->res); + map->res = rman_reserve_resource(&is->is_dvma_rman, astart, + astart + sgsize + align, sgsize, RF_ACTIVE | + rman_make_alignment_flags(align), NULL); + if (map->res == NULL) + panic("iommo_dvmamap_load: reallocation failed"); + + dvmaddr = (rman_get_start(map->res) * IO_PAGE_SIZE) | + (vaddr & IO_PAGE_MASK); + map->start = dvmaddr; + sgcnt = -1; + error = 0; + for (; buflen > 0; ) { + /* + * Get the physical address for this page. + */ + curaddr = pmap_kextract((vm_offset_t)vaddr); + + /* + * Compute the segment size, and adjust counts. + */ + sgsize = IO_PAGE_SIZE - ((u_long)vaddr & IO_PAGE_MASK); + if (buflen < sgsize) + sgsize = buflen; + + DPRINTF(IDB_BUSDMA, + ("iommu_dvmamap_load: map %p loading va %p " + "dva %lx at pa %lx\n", + map, (void *)vaddr, (long)dvmaddr, + (long)(curaddr & ~(PAGE_SIZE - 1)))); + iommu_enter(is, trunc_io_page(dvmaddr), trunc_io_page(curaddr), + flags); + + if (sgcnt == -1 || sgs[sgcnt].ds_len + sgsize > t->maxsegsz) { + if (sgsize > t->maxsegsz) { + /* XXX: add fixup */ + panic("iommu_dvmamap_load: magsegsz too " + "small\n"); + } + sgcnt++; + if (sgcnt > t->nsegments || sgcnt > BUS_DMAMAP_NSEGS) { + error = ENOMEM; + break; + } + sgs[sgcnt].ds_addr = dvmaddr; + sgs[sgcnt].ds_len = sgsize; + } else + sgs[sgcnt].ds_len += sgsize; + dvmaddr += sgsize; + vaddr += sgsize; + buflen -= sgsize; + } + (*cb)(cba, sgs, sgcnt + 1, error); + return (0); +} + + +void +iommu_dvmamap_unload(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map) +{ + size_t len; + int error; + bus_addr_t dvmaddr; + bus_size_t sgsize; + + /* + * If the resource is already deallocated, just pass to the parent + * tag. + */ + if (map->res != NULL) { + dvmaddr = (vm_offset_t)trunc_io_page(map->start); + len = map->buflen; + sgsize = 0; + if (len == 0 || dvmaddr == 0) { + printf("iommu_dvmamap_unload: map = %p, len = %d, " + "addr = %lx\n", map, (int)len, + (unsigned long)dvmaddr); + } + + DPRINTF(IDB_BUSDMA, + ("iommu_dvmamap_unload: map %p removing va %lx size " + "%lx\n", map, (long)dvmaddr, (long)len)); + iommu_remove(is, dvmaddr, len); + + error = rman_release_resource(map->res); + map->res = NULL; + if (error != 0) + printf("warning: approx. %lx of DVMA space lost\n", + sgsize); + } + /* Flush the caches */ + bus_dmamap_unload(t->parent->parent, map); + +} + +void +iommu_dvmamap_sync(bus_dma_tag_t t, struct iommu_state *is, bus_dmamap_t map, + bus_dmasync_op_t op) +{ + vm_offset_t va; + vm_size_t len; + + va = (vm_offset_t)map->buf; + len = map->buflen; + if ((op & BUS_DMASYNC_PREREAD) != 0) { + DPRINTF(IDB_SYNC, + ("iommu_dvmamap_sync: syncing va %p len %lu " + "BUS_DMASYNC_PREREAD\n", (void *)(u_long)va, (u_long)len)); + membar(Sync); + } + if ((op & BUS_DMASYNC_POSTREAD) != 0) { + DPRINTF(IDB_SYNC, + ("iommu_dvmamap_sync: syncing va %p len %lu " + "BUS_DMASYNC_POSTREAD\n", (void *)(u_long)va, (u_long)len)); + /* if we have a streaming buffer, flush it here first */ + if (is->is_sb) { + while (len > 0) { + DPRINTF(IDB_BUSDMA, + ("iommu_dvmamap_sync: flushing va %p, %lu " + "bytes left\n", (void *)(u_long)va, + (u_long)len)); + iommu_strbuf_flush(is, va); + if (len <= IO_PAGE_SIZE) { + iommu_strbuf_flush_done(is); + len = 0; + } else + len -= IO_PAGE_SIZE; + va += IO_PAGE_SIZE; + } + } + } + if ((op & BUS_DMASYNC_PREWRITE) != 0) { + DPRINTF(IDB_SYNC, + ("iommu_dvmamap_sync: syncing va %p len %lu " + "BUS_DMASYNC_PREWRITE\n", (void *)(u_long)va, (u_long)len)); + /* if we have a streaming buffer, flush it here first */ + if (is->is_sb) { + while (len > 0) { + DPRINTF(IDB_BUSDMA, + ("iommu_dvmamap_sync: flushing va %p, %lu " + "bytes left\n", (void *)(u_long)va, + (u_long)len)); + iommu_strbuf_flush(is, va); + if (len <= IO_PAGE_SIZE) { + iommu_strbuf_flush_done(is); + len = 0; + } else + len -= IO_PAGE_SIZE; + va += IO_PAGE_SIZE; + } + } + membar(Sync); + } + if ((op & BUS_DMASYNC_POSTWRITE) != 0) { + DPRINTF(IDB_SYNC, + ("iommu_dvmamap_sync: syncing va %p len %lu " + "BUS_DMASYNC_POSTWRITE\n", (void *)(u_long)va, + (u_long)len)); + /* Nothing to do */ + } +} + +#ifdef IOMMU_DIAG + +#define IOMMU_DTAG_VPNBITS 19 +#define IOMMU_DTAG_VPNMASK ((1 << IOMMU_DTAG_VPNBITS) - 1) +#define IOMMU_DTAG_VPNSHIFT 13 +#define IOMMU_DTAG_ERRBITS 3 +#define IOMMU_DTAG_ERRSHIFT 22 +#define IOMMU_DTAG_ERRMASK \ + (((1 << IOMMU_DTAG_ERRBITS) - 1) << IOMMU_DTAG_ERRSHIFT) + +#define IOMMU_DDATA_PGBITS 21 +#define IOMMU_DDATA_PGMASK ((1 << IOMMU_DDATA_PGBITS) - 1) +#define IOMMU_DDATA_PGSHIFT 13 +#define IOMMU_DDATA_C (1 << 28) +#define IOMMU_DDATA_V (1 << 30) + +/* + * Perform an IOMMU diagnostic access and print the tag belonging to va. + */ +static void +iommu_diag(struct iommu_state *is, vm_offset_t va) +{ + int i; + u_int64_t tag, data; + + *is->is_dva = trunc_io_page(va); + membar(StoreStore | StoreLoad); + printf("iommu_diag: tte entry %#lx, tag compare register is %#lx\n", + is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)], *is->is_dtcmp); + for (i = 0; i < 16; i++) { + tag = is->is_dtag[i]; + data = is->is_ddram[i]; + printf("iommu_diag: tag %d: %#lx, vpn %#lx, err %lx; " + "data %#lx, pa %#lx, v %d, c %d\n", i, + tag, (tag & IOMMU_DTAG_VPNMASK) << IOMMU_DTAG_VPNSHIFT, + (tag & IOMMU_DTAG_ERRMASK) >> IOMMU_DTAG_ERRSHIFT, data, + (data & IOMMU_DDATA_PGMASK) << IOMMU_DDATA_PGSHIFT, + (data & IOMMU_DDATA_V) != 0, (data & IOMMU_DDATA_C) != 0); + } +} + +#endif /* IOMMU_DIAG */ |