- Add support for the IOMMUs of Fire JBus to PCIe and Oberon Uranus

  to PCIe bridges.
- Add support for talking the PROM mappings over to the kernel IOTSB
  just like we do with the kernel TSB in order to allow OFW drivers
  to continue to work.
- Change some members, parameters and variables to unsigned where
  more appropriate.

3 files changed, 166 insertions, 43 deletions

diff --git a/sys/sparc64/include/iommureg.h b/sys/sparc64/include/iommureg.h
index 0665cde..f4d0349 100644
--- a/sys/sparc64/include/iommureg.h
+++ b/sys/sparc64/include/iommureg.h
@@ -44,10 +44,13 @@
  * controllers.
  */
 
-/* iommmu registers */
+/* IOMMU registers */
 #define	IMR_CTL		0x0000	/* IOMMU control register */
 #define	IMR_TSB		0x0008	/* IOMMU TSB base register */
 #define	IMR_FLUSH	0x0010	/* IOMMU flush register */
+/* The TTE Cache is Fire and Oberon only. */
+#define	IMR_CACHE_FLUSH	0x0100	/* IOMMU TTE cache flush address register */
+#define	IMR_CACHE_INVAL	0x0108	/* IOMMU TTE cache invalidate register */
 
 /* streaming buffer registers */
 #define	ISR_CTL		0x0000	/* streaming buffer control reg */
@@ -70,28 +73,57 @@
 /*
  * control register bits
  */
-/* Nummber of entries in IOTSB */
+/* Nummber of entries in the IOTSB - pre-Fire only */
+#define	IOMMUCR_TSBSZ_MASK	0x0000000000070000UL
 #define	IOMMUCR_TSBSZ_SHIFT	16
-#define	IOMMUCR_TSB1K		0x0000000000000000UL
-#define	IOMMUCR_TSB2K		0x0000000000010000UL
-#define	IOMMUCR_TSB4K		0x0000000000020000UL
-#define	IOMMUCR_TSB8K		0x0000000000030000UL
-#define	IOMMUCR_TSB16K		0x0000000000040000UL
-#define	IOMMUCR_TSB32K		0x0000000000050000UL
-#define	IOMMUCR_TSB64K		0x0000000000060000UL
-#define	IOMMUCR_TSB128K		0x0000000000070000UL
-/* Mask for above */
-#define	IOMMUCR_TSBMASK		0xfffffffffff8ffffUL
-/* 8K iommu page size */
+/* TSB cache snoop enable */
+#define	IOMMUCR_SE		0x0000000000000400UL
+/* Cache modes - Fire and Oberon */
+#define	IOMMUCR_CM_NC_TLB_TBW	0x0000000000000000UL
+#define	IOMMUCR_CM_LC_NTLB_NTBW	0x0000000000000100UL
+#define	IOMMUCR_CM_LC_TLB_TBW	0x0000000000000200UL
+#define	IOMMUCR_CM_C_TLB_TBW	0x0000000000000300UL
+/* IOMMU page size - pre-Fire only */
 #define	IOMMUCR_8KPG		0x0000000000000000UL
-/* 64K iommu page size */
 #define	IOMMUCR_64KPG		0x0000000000000004UL
-/* Diag enable */
+/* Bypass enable - Fire and Oberon */
+#define	IOMMUCR_BE		0x0000000000000002UL
+/* Diagnostic mode enable - pre-Fire only */
 #define	IOMMUCR_DE		0x0000000000000002UL
-/* Enable IOMMU */
+/* IOMMU/translation enable */
 #define	IOMMUCR_EN		0x0000000000000001UL
 
 /*
+ * TSB base register bits
+ */
+ /* TSB base address */
+#define	IOMMUTB_TB_MASK		0x000007ffffffe000UL
+#define	IOMMUTB_TB_SHIFT	13
+/* IOMMU page size - Fire and Oberon */
+#define	IOMMUTB_8KPG		0x0000000000000000UL
+#define	IOMMUTB_64KPG		0x0000000000000100UL
+/* Nummber of entries in the IOTSB - Fire and Oberon */
+#define	IOMMUTB_TSBSZ_MASK	0x0000000000000004UL
+#define	IOMMUTB_TSBSZ_SHIFT	0
+
+/*
+ * TSB size definitions for both control and TSB base register */
+#define	IOMMU_TSB1K		0
+#define	IOMMU_TSB2K		1
+#define	IOMMU_TSB4K		2
+#define	IOMMU_TSB8K		3
+#define	IOMMU_TSB16K		4
+#define	IOMMU_TSB32K		5
+#define	IOMMU_TSB64K		6
+#define	IOMMU_TSB128K		7
+/* Fire and Oberon */
+#define	IOMMU_TSB256K		8
+/* Fire and Oberon */
+#define	IOMMU_TSB512K		9
+#define	IOMMU_TSBENTRIES(tsbsz)						\
+	((1 << (tsbsz)) << (IO_PAGE_SHIFT - IOTTE_SHIFT))
+
+/*
  * Diagnostic register definitions
  */
 #define	IOMMU_DTAG_VPNBITS	19
@@ -113,16 +145,16 @@
  */
 /* Entry valid */
 #define	IOTTE_V			0x8000000000000000UL
-/* 8K or 64K page? */
+/* Page size - pre-Fire only */
 #define	IOTTE_64K		0x2000000000000000UL
 #define	IOTTE_8K		0x0000000000000000UL
-/* Is page streamable? */
+/* Streamable page - streaming buffer equipped variants only */
 #define	IOTTE_STREAM		0x1000000000000000UL
-/* Accesses to same bus segment? */
+/* Accesses to the same bus segment - SBus only */
 #define	IOTTE_LOCAL		0x0800000000000000UL
-/* Let's assume this is correct */
-#define	IOTTE_PAMASK		0x000007ffffffe000UL
-/* Accesses to cacheable space */
+/* Physical address mask (based on Oberon) */
+#define	IOTTE_PAMASK		0x00007fffffffe000UL
+/* Accesses to cacheable space - pre-Fire only */
 #define	IOTTE_C			0x0000000000000010UL
 /* Writeable */
 #define	IOTTE_W			0x0000000000000002UL
diff --git a/sys/sparc64/include/iommuvar.h b/sys/sparc64/include/iommuvar.h
index 5904f04..1ef4e0b 100644
--- a/sys/sparc64/include/iommuvar.h
+++ b/sys/sparc64/include/iommuvar.h
@@ -66,10 +66,10 @@ struct iommu_state {
 	int			is_tsbsize;	/* (r) 0 = 8K, ... */
 	uint64_t		is_pmaxaddr;	/* (r) max. physical address */
 	uint64_t		is_dvmabase;	/* (r) */
-	int64_t			is_cr;		/* (r) Control reg value */
+	uint64_t		is_cr;		/* (r) Control reg value */
 
 	vm_paddr_t		is_flushpa[2];	/* (r) */
-	volatile int64_t	*is_flushva[2];	/* (r, *i) */
+	volatile uint64_t	*is_flushva[2];	/* (r, *i) */
 	/*
 	 * (i)
 	 * When a flush is completed, 64 bytes will be stored at the given
@@ -99,11 +99,14 @@ struct iommu_state {
 	/* behavior flags */
 	u_int			is_flags;	/* (r) */
 #define	IOMMU_RERUN_DISABLE	(1 << 0)
+#define	IOMMU_FIRE		(1 << 1)
+#define	IOMMU_FLUSH_CACHE	(1 << 2)
+#define	IOMMU_PRESERVE_PROM	(1 << 3)
 };
 
 /* interfaces for PCI/SBus code */
-void iommu_init(const char *name, struct iommu_state *is, int tsbsize,
-    uint32_t iovabase, int resvpg);
+void iommu_init(const char *name, struct iommu_state *is, u_int tsbsize,
+    uint32_t iovabase, u_int resvpg);
 void iommu_reset(struct iommu_state *is);
 void iommu_decode_fault(struct iommu_state *is, vm_offset_t phys);
 
diff --git a/sys/sparc64/sparc64/iommu.c b/sys/sparc64/sparc64/iommu.c
index 88ab2c9..b5f24db 100644
--- a/sys/sparc64/sparc64/iommu.c
+++ b/sys/sparc64/sparc64/iommu.c
@@ -138,11 +138,13 @@ __FBSDID("$FreeBSD$");
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 
+#include <machine/asi.h>
 #include <machine/bus.h>
 #include <machine/bus_private.h>
 #include <machine/iommureg.h>
 #include <machine/pmap.h>
 #include <machine/resource.h>
+#include <machine/ver.h>
 
 #include <sys/rman.h>
 
@@ -212,6 +214,12 @@ static __inline void
 iommu_tlb_flush(struct iommu_state *is, bus_addr_t va)
 {
 
+	if ((is->is_flags & IOMMU_FIRE) != 0)
+		/*
+		 * Direct page flushing is not supported and also not
+		 * necessary due to cache snooping.
+		 */
+		return;
 	IOMMU_WRITE8(is, is_iommu, IMR_FLUSH, va);
 }
 
@@ -282,18 +290,19 @@ iommu_map_remq(struct iommu_state *is, bus_dmamap_t map)
  *	- create a private DVMA map.
  */
 void
-iommu_init(const char *name, struct iommu_state *is, int tsbsize,
-    uint32_t iovabase, int resvpg)
+iommu_init(const char *name, struct iommu_state *is, u_int tsbsize,
+    uint32_t iovabase, u_int resvpg)
 {
 	vm_size_t size;
 	vm_offset_t offs;
-	uint64_t end;
+	uint64_t end, obpmap, obpptsb, tte;
+	u_int maxtsbsize, obptsbentries, obptsbsize, slot, tsbentries;
 	int i;
 
 	/*
-	 * Setup the iommu.
+	 * Setup the IOMMU.
 	 *
-	 * The sun4u iommu is part of the PCI or SBus controller so we
+	 * The sun4u IOMMU is part of the PCI or SBus controller so we
 	 * will deal with it here..
 	 *
 	 * The IOMMU address space always ends at 0xffffe000, but the starting
@@ -301,16 +310,30 @@ iommu_init(const char *name, struct iommu_state *is, int tsbsize,
 	 * is->is_tsbsize entries, where each entry is 8 bytes.  The start of
 	 * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)).
 	 */
-	is->is_cr = (tsbsize << IOMMUCR_TSBSZ_SHIFT) | IOMMUCR_EN;
+	if ((is->is_flags & IOMMU_FIRE) != 0) {
+		maxtsbsize = IOMMU_TSB512K;
+		/*
+		 * We enable bypass in order to be able to use a physical
+		 * address for the event queue base.
+		 */
+		is->is_cr = IOMMUCR_SE | IOMMUCR_CM_C_TLB_TBW | IOMMUCR_BE;
+	} else {
+		maxtsbsize = IOMMU_TSB128K;
+		is->is_cr = (tsbsize << IOMMUCR_TSBSZ_SHIFT) | IOMMUCR_DE;
+	}
+	if (tsbsize > maxtsbsize)
+		panic("%s: unsupported TSB size	", __func__);
+	tsbentries = IOMMU_TSBENTRIES(tsbsize);
+	is->is_cr |= IOMMUCR_EN;
 	is->is_tsbsize = tsbsize;
 	is->is_dvmabase = iovabase;
 	if (iovabase == -1)
 		is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize);
 
 	size = IOTSB_BASESZ << is->is_tsbsize;
-	printf("%s: DVMA map: %#lx to %#lx%s\n", name,
+	printf("%s: DVMA map: %#lx to %#lx %d entries%s\n", name,
 	    is->is_dvmabase, is->is_dvmabase +
-	    (size << (IO_PAGE_SHIFT - IOTTE_SHIFT)) - 1,
+	    (size << (IO_PAGE_SHIFT - IOTTE_SHIFT)) - 1, tsbentries,
 	    IOMMU_HAS_SB(is) ? ", streaming buffer" : "");
 
 	/*
@@ -333,12 +356,54 @@ iommu_init(const char *name, struct iommu_state *is, int tsbsize,
 	 */
 	is->is_tsb = contigmalloc(size, M_DEVBUF, M_NOWAIT, 0, ~0UL,
 	    PAGE_SIZE, 0);
-	if (is->is_tsb == 0)
+	if (is->is_tsb == NULL)
 		panic("%s: contigmalloc failed", __func__);
 	is->is_ptsb = pmap_kextract((vm_offset_t)is->is_tsb);
 	bzero(is->is_tsb, size);
 
 	/*
+	 * Add the PROM mappings to the kernel IOTSB if desired.
+	 * Note that the firmware of certain Darwin boards doesn't set
+	 * the TSB size correctly.
+	 */
+	if ((is->is_flags & IOMMU_FIRE) != 0)
+		obptsbsize = (IOMMU_READ8(is, is_iommu, IMR_TSB) &
+		    IOMMUTB_TSBSZ_MASK) >> IOMMUTB_TSBSZ_SHIFT;
+	else
+		obptsbsize = (IOMMU_READ8(is, is_iommu, IMR_CTL) &
+		    IOMMUCR_TSBSZ_MASK) >> IOMMUCR_TSBSZ_SHIFT;
+	obptsbentries = IOMMU_TSBENTRIES(obptsbsize);
+	if (bootverbose)
+		printf("%s: PROM IOTSB size: %d (%d entries)\n", name,
+		    obptsbsize, obptsbentries);
+	if ((is->is_flags & IOMMU_PRESERVE_PROM) != 0 &&
+	    !(cpu_impl == CPU_IMPL_ULTRASPARCIIi && obptsbsize == 7)) {
+		if (obptsbentries > tsbentries)
+			panic("%s: PROM IOTSB entries exceed kernel",
+			    __func__);
+		obpptsb = IOMMU_READ8(is, is_iommu, IMR_TSB) &
+		    IOMMUTB_TB_MASK;
+		for (i = 0; i < obptsbentries; i++) {
+			tte = ldxa(obpptsb + i * 8, ASI_PHYS_USE_EC);
+			if ((tte & IOTTE_V) == 0)
+				continue;
+			slot = tsbentries - obptsbentries + i;
+			if (bootverbose)
+				printf("%s: adding PROM IOTSB slot %d "
+				    "(kernel slot %d) TTE: %#lx\n", name,
+				    i, slot, tte);
+			obpmap = (is->is_dvmabase + slot * IO_PAGE_SIZE) >>
+			    IO_PAGE_SHIFT;
+			if (rman_reserve_resource(&is->is_dvma_rman, obpmap,
+			    obpmap, IO_PAGE_SIZE >> IO_PAGE_SHIFT, RF_ACTIVE,
+			    NULL) == NULL)
+				panic("%s: could not reserve PROM IOTSB slot "
+				    "%d (kernel slot %d)", __func__, i, slot);
+			is->is_tsb[slot] = tte;
+		}
+	}
+
+	/*
 	 * Initialize streaming buffer, if it is there.
 	 */
 	if (IOMMU_HAS_SB(is)) {
@@ -349,7 +414,7 @@ iommu_init(const char *name, struct iommu_state *is, int tsbsize,
 		offs = roundup2((vm_offset_t)is->is_flush,
 		    STRBUF_FLUSHSYNC_NBYTES);
 		for (i = 0; i < 2; i++, offs += STRBUF_FLUSHSYNC_NBYTES) {
-			is->is_flushva[i] = (int64_t *)offs;
+			is->is_flushva[i] = (uint64_t *)offs;
 			is->is_flushpa[i] = pmap_kextract(offs);
 		}
 	}
@@ -368,11 +433,16 @@ iommu_init(const char *name, struct iommu_state *is, int tsbsize,
 void
 iommu_reset(struct iommu_state *is)
 {
+	uint64_t tsb;
 	int i;
 
-	IOMMU_WRITE8(is, is_iommu, IMR_TSB, is->is_ptsb);
-	/* Enable IOMMU in diagnostic mode */
-	IOMMU_WRITE8(is, is_iommu, IMR_CTL, is->is_cr | IOMMUCR_DE);
+	tsb = is->is_ptsb;
+	if ((is->is_flags & IOMMU_FIRE) != 0) {
+		tsb |= is->is_tsbsize;
+		IOMMU_WRITE8(is, is_iommu, IMR_CACHE_INVAL, ~0ULL);
+	}
+	IOMMU_WRITE8(is, is_iommu, IMR_TSB, tsb);
+	IOMMU_WRITE8(is, is_iommu, IMR_CTL, is->is_cr);
 
 	for (i = 0; i < 2; i++) {
 		if (is->is_sb[i] != 0) {
@@ -386,6 +456,8 @@ iommu_reset(struct iommu_state *is)
 				is->is_sb[i] = 0;
 		}
 	}
+
+	(void)IOMMU_READ8(is, is_iommu, IMR_CTL);
 }
 
 /*
@@ -396,7 +468,7 @@ static void
 iommu_enter(struct iommu_state *is, vm_offset_t va, vm_paddr_t pa,
     int stream, int flags)
 {
-	int64_t tte;
+	uint64_t tte;
 
 	KASSERT(va >= is->is_dvmabase,
 	    ("%s: va %#lx not in DVMA space", __func__, va));
@@ -423,7 +495,7 @@ iommu_enter(struct iommu_state *is, vm_offset_t va, vm_paddr_t pa,
 static int
 iommu_remove(struct iommu_state *is, vm_offset_t va, vm_size_t len)
 {
-	int streamed = 0;
+	int slot, streamed = 0;
 
 #ifdef IOMMU_DIAG
 	iommu_diag(is, va);
@@ -443,6 +515,12 @@ iommu_remove(struct iommu_state *is, vm_offset_t va, vm_size_t len)
 		len -= ulmin(len, IO_PAGE_SIZE);
 		IOMMU_SET_TTE(is, va, 0);
 		iommu_tlb_flush(is, va);
+		if ((is->is_flags & IOMMU_FLUSH_CACHE) != 0) {
+			slot = IOTSBSLOT(va);
+			if (len <= IO_PAGE_SIZE || slot % 8 == 7)
+				IOMMU_WRITE8(is, is_iommu, IMR_CACHE_FLUSH,
+				    is->is_ptsb + slot * 8);
+		}
 		va += IO_PAGE_SIZE;
 	}
 	return (streamed);
@@ -829,12 +907,13 @@ iommu_dvmamap_load_buffer(bus_dma_tag_t dt, struct iommu_state *is,
     bus_dmamap_t map, void *buf, bus_size_t buflen, struct thread *td,
     int flags, bus_dma_segment_t *segs, int *segp, int align)
 {
-	bus_addr_t amask, dvmaddr;
+	bus_addr_t amask, dvmaddr, dvmoffs;
 	bus_size_t sgsize, esize;
 	vm_offset_t vaddr, voffs;
 	vm_paddr_t curaddr;
 	pmap_t pmap = NULL;
 	int error, firstpg, sgcnt;
+	u_int slot;
 
 	KASSERT(buflen != 0, ("%s: buflen == 0!", __func__));
 	if (buflen > dt->dt_maxsize)
@@ -877,8 +956,15 @@ iommu_dvmamap_load_buffer(bus_dma_tag_t dt, struct iommu_state *is,
 		buflen -= sgsize;
 		vaddr += sgsize;
 
-		iommu_enter(is, trunc_io_page(dvmaddr), trunc_io_page(curaddr),
+		dvmoffs = trunc_io_page(dvmaddr);
+		iommu_enter(is, dvmoffs, trunc_io_page(curaddr),
 		    (map->dm_flags & DMF_STREAMED) != 0, flags);
+		if ((is->is_flags & IOMMU_FLUSH_CACHE) != 0) {
+			slot = IOTSBSLOT(dvmoffs);
+			if (buflen <= 0 || slot % 8 == 7)
+				IOMMU_WRITE8(is, is_iommu, IMR_CACHE_FLUSH,
+				    is->is_ptsb + slot * 8);
+		}
 
 		/*
 		 * Chop the chunk up into segments of at most maxsegsz, but try
@@ -1183,6 +1269,8 @@ iommu_diag(struct iommu_state *is, vm_offset_t va)
 	int i;
 	uint64_t data, tag;
 
+	if ((is->is_flags & IOMMU_FIRE) != 0)
+		return;
 	IS_LOCK_ASSERT(is);
 	IOMMU_WRITE8(is, is_dva, 0, trunc_io_page(va));
 	membar(StoreStore | StoreLoad);