MFC r277454, r277460, r277465, r277466, r277467, r277469, r277470, r277471,

    r277472, r277473, r277474, r277475, r277476, r277477, r277478, r277479,
    r277480, r277512, r277516:

  Add inline implementations of arm bus_space_read/write_N().

  Revise the arm bus_space implementation to avoid dereferencing the tag on
  every operation to retrieve the bs_cookie value almost nothing actually uses.

  Use the explicit member initializer style to init the bus_space struct.

  Use arm/bus_space-v6.c for all armv6 systems

  Consolidate many identical implementations of bus_space to a single
  common tag and implementation shared by armv4 and armv6.

  Micro-optimize the new arm inline bus_space implementation by grouping all
  the data the inline functions access together at the start of the bus_space
  struct so that they all fit in a single cache line.

1 files changed, 263 insertions, 234 deletions

diff --git a/sys/arm/include/bus.h b/sys/arm/include/bus.h
index 0749838..ed9d317 100644
--- a/sys/arm/include/bus.h
+++ b/sys/arm/include/bus.h
@@ -79,172 +79,180 @@
 #define	BUS_SPACE_MAP_LINEAR		0x02
 #define	BUS_SPACE_MAP_PREFETCHABLE     	0x04
 
+/*
+ * Bus space for ARM.
+ *
+ * The functions used most often are grouped together at the beginning to ensure
+ * that all the data fits into a single cache line.  The inline implementations
+ * of single read/write access these values a lot.
+ */
 struct bus_space {
-	/* cookie */
-	void		*bs_cookie;
+	/* Read/write single and barrier: the most commonly used functions. */
+	uint8_t	 (*bs_r_1)(bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint32_t (*bs_r_4)(bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	void	 (*bs_w_1)(bus_space_tag_t, bus_space_handle_t,
+			   bus_size_t, uint8_t);
+	void	 (*bs_w_4)(bus_space_tag_t, bus_space_handle_t,
+			   bus_size_t, uint32_t);
+	void	 (*bs_barrier)(bus_space_tag_t, bus_space_handle_t,
+			       bus_size_t, bus_size_t, int);
+
+	/* Backlink to parent (if copied), and implementation private data. */
+	struct bus_space *bs_parent;
+	void		 *bs_privdata;
 
 	/* mapping/unmapping */
-	int		(*bs_map) (void *, bus_addr_t, bus_size_t,
+	int		(*bs_map) (bus_space_tag_t, bus_addr_t, bus_size_t,
 			    int, bus_space_handle_t *);
-	void		(*bs_unmap) (void *, bus_space_handle_t, bus_size_t);
-	int		(*bs_subregion) (void *, bus_space_handle_t,
+	void		(*bs_unmap) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	int		(*bs_subregion) (bus_space_tag_t, bus_space_handle_t,
 			    bus_size_t, bus_size_t, bus_space_handle_t *);
 
 	/* allocation/deallocation */
-	int		(*bs_alloc) (void *, bus_addr_t, bus_addr_t,
+	int		(*bs_alloc) (bus_space_tag_t, bus_addr_t, bus_addr_t,
 			    bus_size_t, bus_size_t, bus_size_t, int,
 			    bus_addr_t *, bus_space_handle_t *);
-	void		(*bs_free) (void *, bus_space_handle_t,
+	void		(*bs_free) (bus_space_tag_t, bus_space_handle_t,
 			    bus_size_t);
 
-	/* get kernel virtual address */
-	/* barrier */
-	void		(*bs_barrier) (void *, bus_space_handle_t,
-			    bus_size_t, bus_size_t, int);
-
-	/* read (single) */
-	u_int8_t	(*bs_r_1) (void *, bus_space_handle_t, bus_size_t);
-	u_int16_t	(*bs_r_2) (void *, bus_space_handle_t, bus_size_t);
-	u_int32_t	(*bs_r_4) (void *, bus_space_handle_t, bus_size_t);
-	u_int64_t	(*bs_r_8) (void *, bus_space_handle_t, bus_size_t);
+	/* Read single, the less commonly used functions. */
+	uint16_t	(*bs_r_2) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint64_t	(*bs_r_8) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
 
 	/* read multiple */
-	void		(*bs_rm_1) (void *, bus_space_handle_t, bus_size_t,
-	    u_int8_t *, bus_size_t);
-	void		(*bs_rm_2) (void *, bus_space_handle_t, bus_size_t,
-	    u_int16_t *, bus_size_t);
-	void		(*bs_rm_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t *, bus_size_t);
-	void		(*bs_rm_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t *, bus_size_t);
+	void		(*bs_rm_1) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
+	    uint8_t *, bus_size_t);
+	void		(*bs_rm_2) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
+	    uint16_t *, bus_size_t);
+	void		(*bs_rm_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t *, bus_size_t);
+	void		(*bs_rm_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t *, bus_size_t);
 					
 	/* read region */
-	void		(*bs_rr_1) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t *, bus_size_t);
-	void		(*bs_rr_2) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t *, bus_size_t);
-	void		(*bs_rr_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t *, bus_size_t);
-	void		(*bs_rr_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t *, bus_size_t);
+	void		(*bs_rr_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t *, bus_size_t);
+	void		(*bs_rr_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t *, bus_size_t);
+	void		(*bs_rr_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t *, bus_size_t);
+	void		(*bs_rr_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t *, bus_size_t);
 					
-	/* write (single) */
-	void		(*bs_w_1) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t);
-	void		(*bs_w_2) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t);
-	void		(*bs_w_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t);
-	void		(*bs_w_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t);
+	/* Write single, the less commonly used functions. */
+	void		(*bs_w_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t);
+	void		(*bs_w_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t);
 
 	/* write multiple */
-	void		(*bs_wm_1) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int8_t *, bus_size_t);
-	void		(*bs_wm_2) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int16_t *, bus_size_t);
-	void		(*bs_wm_4) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int32_t *, bus_size_t);
-	void		(*bs_wm_8) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*bs_wm_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint8_t *, bus_size_t);
+	void		(*bs_wm_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint16_t *, bus_size_t);
+	void		(*bs_wm_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint32_t *, bus_size_t);
+	void		(*bs_wm_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint64_t *, bus_size_t);
 					
 	/* write region */
-	void		(*bs_wr_1) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int8_t *, bus_size_t);
-	void		(*bs_wr_2) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int16_t *, bus_size_t);
-	void		(*bs_wr_4) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int32_t *, bus_size_t);
-	void		(*bs_wr_8) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*bs_wr_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint8_t *, bus_size_t);
+	void		(*bs_wr_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint16_t *, bus_size_t);
+	void		(*bs_wr_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint32_t *, bus_size_t);
+	void		(*bs_wr_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint64_t *, bus_size_t);
 
 	/* set multiple */
-	void		(*bs_sm_1) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t, bus_size_t);
-	void		(*bs_sm_2) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t, bus_size_t);
-	void		(*bs_sm_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t, bus_size_t);
-	void		(*bs_sm_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t, bus_size_t);
+	void		(*bs_sm_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t, bus_size_t);
+	void		(*bs_sm_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t, bus_size_t);
+	void		(*bs_sm_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t, bus_size_t);
+	void		(*bs_sm_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t, bus_size_t);
 
 	/* set region */
-	void		(*bs_sr_1) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t, bus_size_t);
-	void		(*bs_sr_2) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t, bus_size_t);
-	void		(*bs_sr_4) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t, bus_size_t);
-	void		(*bs_sr_8) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t, bus_size_t);
+	void		(*bs_sr_1) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t, bus_size_t);
+	void		(*bs_sr_2) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t, bus_size_t);
+	void		(*bs_sr_4) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t, bus_size_t);
+	void		(*bs_sr_8) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t, bus_size_t);
 
 	/* copy */
-	void		(*bs_c_1) (void *, bus_space_handle_t, bus_size_t,
+	void		(*bs_c_1) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
 			    bus_space_handle_t, bus_size_t, bus_size_t);
-	void		(*bs_c_2) (void *, bus_space_handle_t, bus_size_t,
+	void		(*bs_c_2) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
 			    bus_space_handle_t, bus_size_t, bus_size_t);
-	void		(*bs_c_4) (void *, bus_space_handle_t, bus_size_t,
+	void		(*bs_c_4) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
 			    bus_space_handle_t, bus_size_t, bus_size_t);
-	void		(*bs_c_8) (void *, bus_space_handle_t, bus_size_t,
+	void		(*bs_c_8) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
 			    bus_space_handle_t, bus_size_t, bus_size_t);
 
 	/* read stream (single) */
-	u_int8_t	(*bs_r_1_s) (void *, bus_space_handle_t, bus_size_t);
-	u_int16_t	(*bs_r_2_s) (void *, bus_space_handle_t, bus_size_t);
-	u_int32_t	(*bs_r_4_s) (void *, bus_space_handle_t, bus_size_t);
-	u_int64_t	(*bs_r_8_s) (void *, bus_space_handle_t, bus_size_t);
+	uint8_t	(*bs_r_1_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint16_t	(*bs_r_2_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint32_t	(*bs_r_4_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
+	uint64_t	(*bs_r_8_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t);
 
 	/* read multiple stream */
-	void		(*bs_rm_1_s) (void *, bus_space_handle_t, bus_size_t,
-	    u_int8_t *, bus_size_t);
-	void		(*bs_rm_2_s) (void *, bus_space_handle_t, bus_size_t,
-	    u_int16_t *, bus_size_t);
-	void		(*bs_rm_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t *, bus_size_t);
-	void		(*bs_rm_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t *, bus_size_t);
+	void		(*bs_rm_1_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
+	    uint8_t *, bus_size_t);
+	void		(*bs_rm_2_s) (bus_space_tag_t, bus_space_handle_t, bus_size_t,
+	    uint16_t *, bus_size_t);
+	void		(*bs_rm_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t *, bus_size_t);
+	void		(*bs_rm_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t *, bus_size_t);
 					
 	/* read region stream */
-	void		(*bs_rr_1_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t *, bus_size_t);
-	void		(*bs_rr_2_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t *, bus_size_t);
-	void		(*bs_rr_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t *, bus_size_t);
-	void		(*bs_rr_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t *, bus_size_t);
+	void		(*bs_rr_1_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t *, bus_size_t);
+	void		(*bs_rr_2_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t *, bus_size_t);
+	void		(*bs_rr_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t *, bus_size_t);
+	void		(*bs_rr_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t *, bus_size_t);
 					
 	/* write stream (single) */
-	void		(*bs_w_1_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int8_t);
-	void		(*bs_w_2_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int16_t);
-	void		(*bs_w_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int32_t);
-	void		(*bs_w_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, u_int64_t);
+	void		(*bs_w_1_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint8_t);
+	void		(*bs_w_2_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint16_t);
+	void		(*bs_w_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint32_t);
+	void		(*bs_w_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, uint64_t);
 
 	/* write multiple stream */
-	void		(*bs_wm_1_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int8_t *, bus_size_t);
-	void		(*bs_wm_2_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int16_t *, bus_size_t);
-	void		(*bs_wm_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int32_t *, bus_size_t);
-	void		(*bs_wm_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*bs_wm_1_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint8_t *, bus_size_t);
+	void		(*bs_wm_2_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint16_t *, bus_size_t);
+	void		(*bs_wm_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint32_t *, bus_size_t);
+	void		(*bs_wm_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint64_t *, bus_size_t);
 					
 	/* write region stream */
-	void		(*bs_wr_1_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int8_t *, bus_size_t);
-	void		(*bs_wr_2_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int16_t *, bus_size_t);
-	void		(*bs_wr_4_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int32_t *, bus_size_t);
-	void		(*bs_wr_8_s) (void *, bus_space_handle_t,
-			    bus_size_t, const u_int64_t *, bus_size_t);
+	void		(*bs_wr_1_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint8_t *, bus_size_t);
+	void		(*bs_wr_2_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint16_t *, bus_size_t);
+	void		(*bs_wr_4_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint32_t *, bus_size_t);
+	void		(*bs_wr_8_s) (bus_space_tag_t, bus_space_handle_t,
+			    bus_size_t, const uint64_t *, bus_size_t);
 };
 
+extern bus_space_tag_t arm_base_bs_tag;
 
 /*
  * Utility macros; INTERNAL USE ONLY.
@@ -252,51 +260,69 @@ struct bus_space {
 #define	__bs_c(a,b)		__CONCAT(a,b)
 #define	__bs_opname(op,size)	__bs_c(__bs_c(__bs_c(bs_,op),_),size)
 
-#define	__bs_rs(sz, t, h, o)						\
-	(*(t)->__bs_opname(r,sz))((t)->bs_cookie, h, o)
-#define	__bs_ws(sz, t, h, o, v)						\
-	(*(t)->__bs_opname(w,sz))((t)->bs_cookie, h, o, v)
 #define	__bs_nonsingle(type, sz, t, h, o, a, c)				\
-	(*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, a, c)
+	(*(t)->__bs_opname(type,sz))((t), h, o, a, c)
 #define	__bs_set(type, sz, t, h, o, v, c)				\
-	(*(t)->__bs_opname(type,sz))((t)->bs_cookie, h, o, v, c)
+	(*(t)->__bs_opname(type,sz))((t), h, o, v, c)
 #define	__bs_copy(sz, t, h1, o1, h2, o2, cnt)				\
-	(*(t)->__bs_opname(c,sz))((t)->bs_cookie, h1, o1, h2, o2, cnt)
+	(*(t)->__bs_opname(c,sz))((t), h1, o1, h2, o2, cnt)
 
 #define	__bs_opname_s(op,size)	__bs_c(__bs_c(__bs_c(__bs_c(bs_,op),_),size),_s)
 #define	__bs_rs_s(sz, t, h, o)						\
-	(*(t)->__bs_opname_s(r,sz))((t)->bs_cookie, h, o)
+	(*(t)->__bs_opname_s(r,sz))((t), h, o)
 #define	__bs_ws_s(sz, t, h, o, v)					\
-	(*(t)->__bs_opname_s(w,sz))((t)->bs_cookie, h, o, v)
+	(*(t)->__bs_opname_s(w,sz))((t), h, o, v)
 #define	__bs_nonsingle_s(type, sz, t, h, o, a, c)			\
-	(*(t)->__bs_opname_s(type,sz))((t)->bs_cookie, h, o, a, c)
-
+	(*(t)->__bs_opname_s(type,sz))((t), h, o, a, c)
+
+
+#define __generate_inline_bs_rs(IFN, MBR, TYP)					\
+	static inline TYP						\
+	IFN(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o)	\
+	{								\
+									\
+		if (__predict_true(t->MBR == NULL))			\
+			return (*(volatile TYP *)(h + o));		\
+		else							\
+			return (t->MBR(t, h, o));		\
+	}
+
+#define __generate_inline_bs_ws(IFN, MBR, TYP)					\
+	static inline void						\
+	IFN(bus_space_tag_t t, bus_space_handle_t h, bus_size_t o, TYP v)\
+	{								\
+									\
+		if (__predict_true(t->MBR == NULL))			\
+			*(volatile TYP *)(h + o) = v;			\
+		else							\
+			t->MBR(t, h, o, v);			\
+	}
 
 /*
  * Mapping and unmapping operations.
  */
 #define	bus_space_map(t, a, s, c, hp)					\
-	(*(t)->bs_map)((t)->bs_cookie, (a), (s), (c), (hp))
+	(*(t)->bs_map)((t), (a), (s), (c), (hp))
 #define	bus_space_unmap(t, h, s)					\
-	(*(t)->bs_unmap)((t)->bs_cookie, (h), (s))
+	(*(t)->bs_unmap)((t), (h), (s))
 #define	bus_space_subregion(t, h, o, s, hp)				\
-	(*(t)->bs_subregion)((t)->bs_cookie, (h), (o), (s), (hp))
+	(*(t)->bs_subregion)((t), (h), (o), (s), (hp))
 
 
 /*
  * Allocation and deallocation operations.
  */
 #define	bus_space_alloc(t, rs, re, s, a, b, c, ap, hp)			\
-	(*(t)->bs_alloc)((t)->bs_cookie, (rs), (re), (s), (a), (b),	\
+	(*(t)->bs_alloc)((t), (rs), (re), (s), (a), (b),	\
 	    (c), (ap), (hp))
 #define	bus_space_free(t, h, s)						\
-	(*(t)->bs_free)((t)->bs_cookie, (h), (s))
+	(*(t)->bs_free)((t), (h), (s))
 
 /*
  * Bus barrier operations.
  */
 #define	bus_space_barrier(t, h, o, l, f)				\
-	(*(t)->bs_barrier)((t)->bs_cookie, (h), (o), (l), (f))
+	(*(t)->bs_barrier)((t), (h), (o), (l), (f))
 
 #define	BUS_SPACE_BARRIER_READ	0x01
 #define	BUS_SPACE_BARRIER_WRITE	0x02
@@ -304,15 +330,15 @@ struct bus_space {
 /*
  * Bus read (single) operations.
  */
-#define	bus_space_read_1(t, h, o)	__bs_rs(1,(t),(h),(o))
-#define	bus_space_read_2(t, h, o)	__bs_rs(2,(t),(h),(o))
-#define	bus_space_read_4(t, h, o)	__bs_rs(4,(t),(h),(o))
-#define	bus_space_read_8(t, h, o)	__bs_rs(8,(t),(h),(o))
+__generate_inline_bs_rs(bus_space_read_1, bs_r_1, uint8_t);
+__generate_inline_bs_rs(bus_space_read_2, bs_r_2, uint16_t);
+__generate_inline_bs_rs(bus_space_read_4, bs_r_4, uint32_t);
+__generate_inline_bs_rs(bus_space_read_8, bs_r_8, uint64_t);
 
-#define bus_space_read_stream_1(t, h, o)        __bs_rs_s(1,(t), (h), (o))
-#define bus_space_read_stream_2(t, h, o)        __bs_rs_s(2,(t), (h), (o))
-#define bus_space_read_stream_4(t, h, o)        __bs_rs_s(4,(t), (h), (o))
-#define	bus_space_read_stream_8(t, h, o)	__bs_rs_s(8,8,(t),(h),(o))
+__generate_inline_bs_rs(bus_space_read_stream_1, bs_r_1_s, uint8_t);           
+__generate_inline_bs_rs(bus_space_read_stream_2, bs_r_2_s, uint16_t);          
+__generate_inline_bs_rs(bus_space_read_stream_4, bs_r_4_s, uint32_t);          
+__generate_inline_bs_rs(bus_space_read_stream_8, bs_r_8_s, uint64_t);          
 
 /*
  * Bus read multiple operations.
@@ -361,15 +387,15 @@ struct bus_space {
 /*
  * Bus write (single) operations.
  */
-#define	bus_space_write_1(t, h, o, v)	__bs_ws(1,(t),(h),(o),(v))
-#define	bus_space_write_2(t, h, o, v)	__bs_ws(2,(t),(h),(o),(v))
-#define	bus_space_write_4(t, h, o, v)	__bs_ws(4,(t),(h),(o),(v))
-#define	bus_space_write_8(t, h, o, v)	__bs_ws(8,(t),(h),(o),(v))
+__generate_inline_bs_ws(bus_space_write_1, bs_w_1, uint8_t);
+__generate_inline_bs_ws(bus_space_write_2, bs_w_2, uint16_t);
+__generate_inline_bs_ws(bus_space_write_4, bs_w_4, uint32_t);
+__generate_inline_bs_ws(bus_space_write_8, bs_w_8, uint64_t);
 
-#define	bus_space_write_stream_1(t, h, o, v)	__bs_ws_s(1,(t),(h),(o),(v))
-#define	bus_space_write_stream_2(t, h, o, v)	__bs_ws_s(2,(t),(h),(o),(v))
-#define	bus_space_write_stream_4(t, h, o, v)	__bs_ws_s(4,(t),(h),(o),(v))
-#define	bus_space_write_stream_8(t, h, o, v)	__bs_ws_s(8,(t),(h),(o),(v))
+__generate_inline_bs_ws(bus_space_write_stream_1, bs_w_1_s, uint8_t);           
+__generate_inline_bs_ws(bus_space_write_stream_2, bs_w_2_s, uint16_t);          
+__generate_inline_bs_ws(bus_space_write_stream_4, bs_w_4_s, uint32_t);          
+__generate_inline_bs_ws(bus_space_write_stream_8, bs_w_8_s, uint64_t);          
 
 
 /*
@@ -460,204 +486,204 @@ struct bus_space {
  */
 
 #define bs_map_proto(f)							\
-int	__bs_c(f,_bs_map) (void *t, bus_addr_t addr,		\
+int	__bs_c(f,_bs_map) (bus_space_tag_t t, bus_addr_t addr,		\
 	    bus_size_t size, int cacheable, bus_space_handle_t *bshp);
 
 #define bs_unmap_proto(f)						\
-void	__bs_c(f,_bs_unmap) (void *t, bus_space_handle_t bsh,		\
+void	__bs_c(f,_bs_unmap) (bus_space_tag_t t, bus_space_handle_t bsh,		\
 	    bus_size_t size);
 
 #define bs_subregion_proto(f)						\
-int	__bs_c(f,_bs_subregion) (void *t, bus_space_handle_t bsh,	\
+int	__bs_c(f,_bs_subregion) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 	    bus_size_t offset, bus_size_t size, 			\
 	    bus_space_handle_t *nbshp);
 
 #define bs_alloc_proto(f)						\
-int	__bs_c(f,_bs_alloc) (void *t, bus_addr_t rstart,		\
+int	__bs_c(f,_bs_alloc) (bus_space_tag_t t, bus_addr_t rstart,		\
 	    bus_addr_t rend, bus_size_t size, bus_size_t align,		\
 	    bus_size_t boundary, int cacheable, bus_addr_t *addrp,	\
 	    bus_space_handle_t *bshp);
 
 #define bs_free_proto(f)						\
-void	__bs_c(f,_bs_free) (void *t, bus_space_handle_t bsh,	\
+void	__bs_c(f,_bs_free) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 	    bus_size_t size);
 
 #define bs_mmap_proto(f)						\
 int	__bs_c(f,_bs_mmap) (struct cdev *, vm_offset_t, vm_paddr_t *, int);
 
 #define bs_barrier_proto(f)						\
-void	__bs_c(f,_bs_barrier) (void *t, bus_space_handle_t bsh,	\
+void	__bs_c(f,_bs_barrier) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 	    bus_size_t offset, bus_size_t len, int flags);
 
 #define	bs_r_1_proto(f)							\
-u_int8_t	__bs_c(f,_bs_r_1) (void *t, bus_space_handle_t bsh,	\
+uint8_t	__bs_c(f,_bs_r_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_2_proto(f)							\
-u_int16_t	__bs_c(f,_bs_r_2) (void *t, bus_space_handle_t bsh,	\
+uint16_t	__bs_c(f,_bs_r_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_4_proto(f)							\
-u_int32_t	__bs_c(f,_bs_r_4) (void *t, bus_space_handle_t bsh,	\
+uint32_t	__bs_c(f,_bs_r_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_8_proto(f)							\
-u_int64_t	__bs_c(f,_bs_r_8) (void *t, bus_space_handle_t bsh,	\
+uint64_t	__bs_c(f,_bs_r_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_1_s_proto(f)						\
-u_int8_t	__bs_c(f,_bs_r_1_s) (void *t, bus_space_handle_t bsh,	\
+uint8_t	__bs_c(f,_bs_r_1_s) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_2_s_proto(f)						\
-u_int16_t	__bs_c(f,_bs_r_2_s) (void *t, bus_space_handle_t bsh,	\
+uint16_t	__bs_c(f,_bs_r_2_s) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_r_4_s_proto(f)						\
-u_int32_t	__bs_c(f,_bs_r_4_s) (void *t, bus_space_handle_t bsh,	\
+uint32_t	__bs_c(f,_bs_r_4_s) (bus_space_tag_t t, bus_space_handle_t bsh,	\
 		    bus_size_t offset);
 
 #define	bs_w_1_proto(f)							\
-void	__bs_c(f,_bs_w_1) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int8_t value);
+void	__bs_c(f,_bs_w_1) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint8_t value);
 
 #define	bs_w_2_proto(f)							\
-void	__bs_c(f,_bs_w_2) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int16_t value);
+void	__bs_c(f,_bs_w_2) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint16_t value);
 
 #define	bs_w_4_proto(f)							\
-void	__bs_c(f,_bs_w_4) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int32_t value);
+void	__bs_c(f,_bs_w_4) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint32_t value);
 
 #define	bs_w_8_proto(f)							\
-void	__bs_c(f,_bs_w_8) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int64_t value);
+void	__bs_c(f,_bs_w_8) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint64_t value);
 
 #define	bs_w_1_s_proto(f)						\
-void	__bs_c(f,_bs_w_1_s) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int8_t value);
+void	__bs_c(f,_bs_w_1_s) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint8_t value);
 
 #define	bs_w_2_s_proto(f)						\
-void	__bs_c(f,_bs_w_2_s) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int16_t value);
+void	__bs_c(f,_bs_w_2_s) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint16_t value);
 
 #define	bs_w_4_s_proto(f)						\
-void	__bs_c(f,_bs_w_4_s) (void *t, bus_space_handle_t bsh,		\
-	    bus_size_t offset, u_int32_t value);
+void	__bs_c(f,_bs_w_4_s) (bus_space_tag_t t, bus_space_handle_t bsh,		\
+	    bus_size_t offset, uint32_t value);
 
 #define	bs_rm_1_proto(f)						\
-void	__bs_c(f,_bs_rm_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int8_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_rm_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint8_t *addr, bus_size_t count);
 
 #define	bs_rm_2_proto(f)						\
-void	__bs_c(f,_bs_rm_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int16_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_rm_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint16_t *addr, bus_size_t count);
 
 #define	bs_rm_4_proto(f)						\
-void	__bs_c(f,_bs_rm_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int32_t *addr, bus_size_t count);		
+void	__bs_c(f,_bs_rm_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint32_t *addr, bus_size_t count);		
 
 #define	bs_rm_8_proto(f)						\
-void	__bs_c(f,_bs_rm_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int64_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_rm_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint64_t *addr, bus_size_t count);
 
 #define	bs_wm_1_proto(f)						\
-void	__bs_c(f,_bs_wm_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int8_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_wm_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint8_t *addr, bus_size_t count);
 
 #define	bs_wm_2_proto(f)						\
-void	__bs_c(f,_bs_wm_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int16_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_wm_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint16_t *addr, bus_size_t count);
 
 #define	bs_wm_4_proto(f)						\
-void	__bs_c(f,_bs_wm_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int32_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_wm_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint32_t *addr, bus_size_t count);
 
 #define	bs_wm_8_proto(f)						\
-void	__bs_c(f,_bs_wm_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int64_t *addr, bus_size_t count);
+void	__bs_c(f,_bs_wm_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint64_t *addr, bus_size_t count);
 
 #define	bs_rr_1_proto(f)						\
-void	__bs_c(f, _bs_rr_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int8_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_rr_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint8_t *addr, bus_size_t count);
 
 #define	bs_rr_2_proto(f)						\
-void	__bs_c(f, _bs_rr_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int16_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_rr_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint16_t *addr, bus_size_t count);
 
 #define	bs_rr_4_proto(f)						\
-void	__bs_c(f, _bs_rr_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int32_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_rr_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint32_t *addr, bus_size_t count);
 
 #define	bs_rr_8_proto(f)						\
-void	__bs_c(f, _bs_rr_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int64_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_rr_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint64_t *addr, bus_size_t count);
 
 #define	bs_wr_1_proto(f)						\
-void	__bs_c(f, _bs_wr_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int8_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_wr_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint8_t *addr, bus_size_t count);
 
 #define	bs_wr_2_proto(f)						\
-void	__bs_c(f, _bs_wr_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int16_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_wr_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint16_t *addr, bus_size_t count);
 
 #define	bs_wr_4_proto(f)						\
-void	__bs_c(f, _bs_wr_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int32_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_wr_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint32_t *addr, bus_size_t count);
 
 #define	bs_wr_8_proto(f)						\
-void	__bs_c(f, _bs_wr_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, const u_int64_t *addr, bus_size_t count);
+void	__bs_c(f, _bs_wr_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, const uint64_t *addr, bus_size_t count);
 
 #define	bs_sm_1_proto(f)						\
-void	__bs_c(f,_bs_sm_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int8_t value, bus_size_t count);
+void	__bs_c(f,_bs_sm_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint8_t value, bus_size_t count);
 
 #define	bs_sm_2_proto(f)						\
-void	__bs_c(f,_bs_sm_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int16_t value, bus_size_t count);
+void	__bs_c(f,_bs_sm_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint16_t value, bus_size_t count);
 
 #define	bs_sm_4_proto(f)						\
-void	__bs_c(f,_bs_sm_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int32_t value, bus_size_t count);
+void	__bs_c(f,_bs_sm_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint32_t value, bus_size_t count);
 
 #define	bs_sm_8_proto(f)						\
-void	__bs_c(f,_bs_sm_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int64_t value, bus_size_t count);
+void	__bs_c(f,_bs_sm_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint64_t value, bus_size_t count);
 
 #define	bs_sr_1_proto(f)						\
-void	__bs_c(f,_bs_sr_1) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int8_t value, bus_size_t count);
+void	__bs_c(f,_bs_sr_1) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint8_t value, bus_size_t count);
 
 #define	bs_sr_2_proto(f)						\
-void	__bs_c(f,_bs_sr_2) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int16_t value, bus_size_t count);
+void	__bs_c(f,_bs_sr_2) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint16_t value, bus_size_t count);
 
 #define	bs_sr_4_proto(f)						\
-void	__bs_c(f,_bs_sr_4) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int32_t value, bus_size_t count);
+void	__bs_c(f,_bs_sr_4) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint32_t value, bus_size_t count);
 
 #define	bs_sr_8_proto(f)						\
-void	__bs_c(f,_bs_sr_8) (void *t, bus_space_handle_t bsh,	\
-	    bus_size_t offset, u_int64_t value, bus_size_t count);
+void	__bs_c(f,_bs_sr_8) (bus_space_tag_t t, bus_space_handle_t bsh,	\
+	    bus_size_t offset, uint64_t value, bus_size_t count);
 
 #define	bs_c_1_proto(f)							\
-void	__bs_c(f,_bs_c_1) (void *t, bus_space_handle_t bsh1,	\
+void	__bs_c(f,_bs_c_1) (bus_space_tag_t t, bus_space_handle_t bsh1,	\
 	    bus_size_t offset1, bus_space_handle_t bsh2,		\
 	    bus_size_t offset2, bus_size_t count);
 
 #define	bs_c_2_proto(f)							\
-void	__bs_c(f,_bs_c_2) (void *t, bus_space_handle_t bsh1,	\
+void	__bs_c(f,_bs_c_2) (bus_space_tag_t t, bus_space_handle_t bsh1,	\
 	    bus_size_t offset1, bus_space_handle_t bsh2,		\
 	    bus_size_t offset2, bus_size_t count);
 
 #define	bs_c_4_proto(f)							\
-void	__bs_c(f,_bs_c_4) (void *t, bus_space_handle_t bsh1,	\
+void	__bs_c(f,_bs_c_4) (bus_space_tag_t t, bus_space_handle_t bsh1,	\
 	    bus_size_t offset1, bus_space_handle_t bsh2,		\
 	    bus_size_t offset2, bus_size_t count);
 
 #define	bs_c_8_proto(f)							\
-void	__bs_c(f,_bs_c_8) (void *t, bus_space_handle_t bsh1,	\
+void	__bs_c(f,_bs_c_8) (bus_space_tag_t t, bus_space_handle_t bsh1,	\
 	    bus_size_t offset1, bus_space_handle_t bsh2,		\
 	    bus_size_t offset2, bus_size_t count);
 
@@ -712,6 +738,9 @@ bs_c_2_proto(f);		\
 bs_c_4_proto(f);		\
 bs_c_8_proto(f);
 
+void generic_bs_unimplemented(void);
+#define	BS_UNIMPLEMENTED	(void *)generic_bs_unimplemented
+
 #define BUS_SPACE_ALIGNED_POINTER(p, t) ALIGNED_POINTER(p, t)
 
 #define BUS_SPACE_MAXADDR_24BIT	0xFFFFFF