Lock down the IOMMU bus_dma implementation to make it safe to use

without Giant held.

A quick outline of the locking strategy:
Since all IOMMUs are synchronized, there is a single lock, iommu_mtx,
which protects the hardware registers (where needed) and the global and
per-IOMMU software states. As soon as the IOMMUs are divorced, each struct
iommu_state will have its own mutex (and the remaining global state
will be moved into the struct).
The dvma rman has its own internal mutex; the TSB slots may only be
accessed by the owner of the corresponding resource, so neither needs
extra protection.
Since there is a second access path to maps via LRU queues, the consumer-
provided locking is not sufficient; therefore, each map which is on a
queue is additionally protected by iommu_mtx (in part, there is one
member which only the map owner may access). Each map on a queue may
be accessed and removed from or repositioned in a queue in any context as
long as the lock is held; only the owner may insert a map.
To reduce lock contention, some bus_dma functions remove the map from
the queue temporarily (on behalf of the map owner) for some operations and
reinsert it when they are done. Shorter operations and operations which are
not done on behalf of the lock owner are completely covered by the lock.

To facilitate the locking, reorganize the streaming buffer handling;
while being there, fix an old oversight which would cause the streaming
buffer to always be flushed, regardless of whether streaming was enabled
in the TSB entry. The streaming buffer is still disabled for now, since
there are a number of drivers which lack critical bus_dmamp_sync() calls.

Additional testing by:	jake

1 files changed, 32 insertions, 19 deletions

diff --git a/sys/sparc64/include/iommuvar.h b/sys/sparc64/include/iommuvar.h
index 6640573..2bdf55d 100644
--- a/sys/sparc64/include/iommuvar.h
+++ b/sys/sparc64/include/iommuvar.h
@@ -40,45 +40,58 @@
 #define	trunc_io_page(x)	trunc_page(x)
 
 /*
- * per-IOMMU state
+ * Per-IOMMU state. The parenthesized comments indicate the locking strategy:
+ *	i - protected by iommu_mtx.
+ *	r - read-only after initialization.
+ *	* - comment refers to pointer target / target hardware registers
+ *	    (for bus_addr_t).
+ * iommu_map_lruq is also locked by iommu_mtx. Elements of iommu_tsb may only
+ * be accessed from functions operating on the map owning the corresponding
+ * resource, so the locking the user is required to do to protect the map is
+ * sufficient. As soon as the TSBs are divorced, these will be moved into struct
+ * iommu_state, and each state struct will get its own lock.
+ * iommu_dvma_rman needs to be moved there too, but has its own internal lock.
  */
 struct iommu_state {
-	int			is_tsbsize;	/* 0 = 8K, ... */
-	u_int64_t		is_dvmabase;
-	int64_t			is_cr;		/* IOMMU control register value */
+	int			is_tsbsize;	/* (r) 0 = 8K, ... */
+	u_int64_t		is_dvmabase;	/* (r) */
+	int64_t			is_cr;		/* (r) Control reg value */
 
-	vm_paddr_t		is_flushpa[2];
-	volatile int64_t	*is_flushva[2];
+	vm_paddr_t		is_flushpa[2];	/* (r) */
+	volatile int64_t	*is_flushva[2];	/* (r, *i) */
 	/*
+	 * (i)
 	 * When a flush is completed, 64 bytes will be stored at the given
 	 * location, the first double word being 1, to indicate completion.
 	 * The lower 6 address bits are ignored, so the addresses need to be
 	 * suitably aligned; over-allocate a large enough margin to be able
 	 * to adjust it.
 	 * Two such buffers are needed.
-	 * Needs to be volatile or egcs optimizes away loads.
 	 */
 	volatile char		is_flush[STRBUF_FLUSHSYNC_NBYTES * 3 - 1];
 
 	/* copies of our parents state, to allow us to be self contained */
-	bus_space_tag_t		is_bustag;	/* our bus tag */
-	bus_space_handle_t	is_bushandle;
-	bus_addr_t		is_iommu;	/* IOMMU registers */
-	bus_addr_t		is_sb[2];	/* streaming buffer */
-	bus_addr_t		is_dtag;	/* tag diagnostics access */
-	bus_addr_t		is_ddram;	/* data ram diag. access */
-	bus_addr_t		is_dqueue;	/* LRU queue diag. access */
-	bus_addr_t		is_dva;		/* VA diag. register */
-	bus_addr_t		is_dtcmp;	/* tag compare diag. access */
+	bus_space_tag_t		is_bustag;	/* (r) Our bus tag */
+	bus_space_handle_t	is_bushandle;	/* (r) */
+	bus_addr_t		is_iommu;	/* (r, *i) IOMMU registers */
+	bus_addr_t		is_sb[2];	/* (r, *i) Streaming buffer */
+	/* Tag diagnostics access */
+	bus_addr_t		is_dtag;	/* (r, *r) */
+	/* Data RAM diagnostic access */
+	bus_addr_t		is_ddram;	/* (r, *r) */
+	/* LRU queue diag. access */
+	bus_addr_t		is_dqueue;	/* (r, *r) */
+	/* Virtual address diagnostics register */
+	bus_addr_t		is_dva;		/* (r, *r) */
+	/* Tag compare diagnostics access */
+	bus_addr_t		is_dtcmp;	/* (r, *r) */
 
-	STAILQ_ENTRY(iommu_state)	is_link;
+	STAILQ_ENTRY(iommu_state)	is_link;	/* (r) */
 };
 
 /* interfaces for PCI/SBUS code */
 void iommu_init(char *, struct iommu_state *, int, u_int32_t, int);
 void iommu_reset(struct iommu_state *);
-void iommu_enter(struct iommu_state *, vm_offset_t, vm_paddr_t, int);
-void iommu_remove(struct iommu_state *, vm_offset_t, size_t);
 void iommu_decode_fault(struct iommu_state *, vm_offset_t);
 
 extern struct bus_dma_methods iommu_dma_methods;