Remove Giant from CAM. Drivers (SIMs) now register a mutex that CAM will

use to synchornize and protect all data objects that are used for that
SIM.  Drivers that are not yet MPSAFE register Giant and operate as
usual.  RIght now, no drivers are MPSAFE, though a few will be changed
in the coming week as this work settles down.

The driver API has changed, so all CAM drivers will need to be recompiled.
The userland API has not changed, so tools like camcontrol do not need to
be recompiled.

51 files changed, 1275 insertions, 995 deletions

diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h
index 7b3770d..33799fb 100644
--- a/sys/cam/cam_ccb.h
+++ b/sys/cam/cam_ccb.h
@@ -242,8 +242,7 @@ typedef union {
 typedef union {
 	void		*ptr;
 	u_long		field;
-	u_int8_t	bytes[sizeof(void *) > sizeof(u_long)
-			      ? sizeof(void *) : sizeof(u_long)];
+	u_int8_t	bytes[sizeof(uintptr_t)];
 } ccb_priv_entry;
 
 typedef union {
@@ -274,8 +273,12 @@ struct ccb_hdr {
 	ccb_ppriv_area	periph_priv;
 	ccb_spriv_area	sim_priv;
 	u_int32_t	timeout;	/* Timeout value */
+
+	/*
+	 * Deprecated, only for use by non-MPSAFE SIMs.  All others must
+	 * allocate and initialize their own callout storage.
+	 */
 	struct		callout_handle timeout_ch;
-					/* Callout handle used for timeouts */
 };
 
 /* Get Device Information CCB */
diff --git a/sys/cam/cam_periph.c b/sys/cam/cam_periph.c
index 925e2a7..c0feec6 100644
--- a/sys/cam/cam_periph.c
+++ b/sys/cam/cam_periph.c
@@ -51,6 +51,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/cam_xpt_periph.h>
 #include <cam/cam_periph.h>
 #include <cam/cam_debug.h>
+#include <cam/cam_sim.h>
 
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_message.h>
@@ -122,6 +123,7 @@ cam_periph_alloc(periph_ctor_t *periph_ctor,
 		 ac_callback_t *ac_callback, ac_code code, void *arg)
 {
 	struct		periph_driver **p_drv;
+	struct		cam_sim *sim;
 	struct		cam_periph *periph;
 	struct		cam_periph *cur_periph;
 	path_id_t	path_id;
@@ -163,11 +165,14 @@ cam_periph_alloc(periph_ctor_t *periph_ctor,
 	
 	init_level++;
 
+	xpt_lock_buses();
 	for (p_drv = periph_drivers; *p_drv != NULL; p_drv++) {
 		if (strcmp((*p_drv)->driver_name, name) == 0)
 			break;
 	}
-	
+	xpt_unlock_buses();
+
+	sim = xpt_path_sim(path);
 	path_id = xpt_path_path_id(path);
 	target_id = xpt_path_target_id(path);
 	lun_id = xpt_path_lun_id(path);
@@ -181,6 +186,7 @@ cam_periph_alloc(periph_ctor_t *periph_ctor,
 	periph->unit_number = camperiphunit(*p_drv, path_id, target_id, lun_id);
 	periph->immediate_priority = CAM_PRIORITY_NONE;
 	periph->refcount = 0;
+	periph->sim = sim;
 	SLIST_INIT(&periph->ccb_list);
 	status = xpt_create_path(&path, periph, path_id, target_id, lun_id);
 	if (status != CAM_REQ_CMP)
@@ -276,14 +282,13 @@ cam_periph_find(struct cam_path *path, char *name)
 cam_status
 cam_periph_acquire(struct cam_periph *periph)
 {
-	int s;
 
 	if (periph == NULL)
 		return(CAM_REQ_CMP_ERR);
 
-	s = splsoftcam();
+	xpt_lock_buses();
 	periph->refcount++;
-	splx(s);
+	xpt_unlock_buses();
 
 	return(CAM_REQ_CMP);
 }
@@ -291,18 +296,66 @@ cam_periph_acquire(struct cam_periph *periph)
 void
 cam_periph_release(struct cam_periph *periph)
 {
-	int s;
 
 	if (periph == NULL)
 		return;
 
-	s = splsoftcam();
+	xpt_lock_buses();
 	if ((--periph->refcount == 0)
 	 && (periph->flags & CAM_PERIPH_INVALID)) {
 		camperiphfree(periph);
 	}
-	splx(s);
+	xpt_unlock_buses();
+
+}
+
+int
+cam_periph_hold(struct cam_periph *periph, int priority)
+{
+	struct mtx *mtx;
+	int error;
+
+	mtx_assert(periph->sim->mtx, MA_OWNED);
+
+	/*
+	 * Increment the reference count on the peripheral
+	 * while we wait for our lock attempt to succeed
+	 * to ensure the peripheral doesn't disappear out
+	 * from user us while we sleep.
+	 */
+
+	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
+		return (ENXIO);
+
+	mtx = periph->sim->mtx;
+	if (mtx == &Giant)
+		mtx = NULL;
+
+	while ((periph->flags & CAM_PERIPH_LOCKED) != 0) {
+		periph->flags |= CAM_PERIPH_LOCK_WANTED;
+		if ((error = msleep(periph, mtx, priority, "caplck", 0)) != 0) {
+			cam_periph_release(periph);
+			return (error);
+		}
+	}
+
+	periph->flags |= CAM_PERIPH_LOCKED;
+	return (0);
+}
+
+void
+cam_periph_unhold(struct cam_periph *periph)
+{
+
+	mtx_assert(periph->sim->mtx, MA_OWNED);
+
+	periph->flags &= ~CAM_PERIPH_LOCKED;
+	if ((periph->flags & CAM_PERIPH_LOCK_WANTED) != 0) {
+		periph->flags &= ~CAM_PERIPH_LOCK_WANTED;
+		wakeup(periph);
+	}
 
+	cam_periph_release(periph);
 }
 
 /*
@@ -424,9 +477,7 @@ camperiphunit(struct periph_driver *p_drv, path_id_t pathid,
 void
 cam_periph_invalidate(struct cam_periph *periph)
 {
-	int s;
 
-	s = splsoftcam();
 	/*
 	 * We only call this routine the first time a peripheral is
 	 * invalidated.  The oninvalidate() routine is always called at
@@ -439,11 +490,12 @@ cam_periph_invalidate(struct cam_periph *periph)
 	periph->flags |= CAM_PERIPH_INVALID;
 	periph->flags &= ~CAM_PERIPH_NEW_DEV_FOUND;
 
+	xpt_lock_buses();
 	if (periph->refcount == 0)
 		camperiphfree(periph);
 	else if (periph->refcount < 0)
 		printf("cam_invalidate_periph: refcount < 0!!\n");
-	splx(s);
+	xpt_unlock_buses();
 }
 
 static void
@@ -502,30 +554,11 @@ camperiphfree(struct cam_periph *periph)
 /*
  * Wait interruptibly for an exclusive lock.
  */
-int
-cam_periph_lock(struct cam_periph *periph, int priority)
+void
+cam_periph_lock(struct cam_periph *periph)
 {
-	int error;
-
-	/*
-	 * Increment the reference count on the peripheral
-	 * while we wait for our lock attempt to succeed
-	 * to ensure the peripheral doesn't disappear out
-	 * from under us while we sleep.
-	 */
-	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
-		return(ENXIO);
 
-	while ((periph->flags & CAM_PERIPH_LOCKED) != 0) {
-		periph->flags |= CAM_PERIPH_LOCK_WANTED;
-		if ((error = tsleep(periph, priority, "caplck", 0)) != 0) {
-			cam_periph_release(periph);
-			return error;
-		}
-	}
-
-	periph->flags |= CAM_PERIPH_LOCKED;
-	return 0;
+	mtx_lock(periph->sim->mtx);
 }
 
 /*
@@ -534,13 +567,8 @@ cam_periph_lock(struct cam_periph *periph, int priority)
 void
 cam_periph_unlock(struct cam_periph *periph)
 {
-	periph->flags &= ~CAM_PERIPH_LOCKED;
-	if ((periph->flags & CAM_PERIPH_LOCK_WANTED) != 0) {
-		periph->flags &= ~CAM_PERIPH_LOCK_WANTED;
-		wakeup(periph);
-	}
 
-	cam_periph_release(periph);
+	mtx_unlock(periph->sim->mtx);
 }
 
 /*
@@ -752,12 +780,11 @@ union ccb *
 cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
 {
 	struct ccb_hdr *ccb_h;
-	int s;
+	struct mtx *mtx;
 
+	mtx_assert(periph->sim->mtx, MA_OWNED);
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("entering cdgetccb\n"));
 
-	s = splsoftcam();
-	
 	while (SLIST_FIRST(&periph->ccb_list) == NULL) {
 		if (periph->immediate_priority > priority)
 			periph->immediate_priority = priority;
@@ -765,24 +792,35 @@ cam_periph_getccb(struct cam_periph *periph, u_int32_t priority)
 		if ((SLIST_FIRST(&periph->ccb_list) != NULL)
 		 && (SLIST_FIRST(&periph->ccb_list)->pinfo.priority == priority))
 			break;
-		tsleep(&periph->ccb_list, PRIBIO, "cgticb", 0);
+		mtx_assert(periph->sim->mtx, MA_OWNED);
+		if (periph->sim->mtx == &Giant)
+			mtx = NULL;
+		else
+			mtx = periph->sim->mtx;
+		msleep(&periph->ccb_list, mtx, PRIBIO, "cgticb", 0);
 	}
 
 	ccb_h = SLIST_FIRST(&periph->ccb_list);
 	SLIST_REMOVE_HEAD(&periph->ccb_list, periph_links.sle);
-	splx(s);
 	return ((union ccb *)ccb_h);
 }
 
 void
 cam_periph_ccbwait(union ccb *ccb)
 {
+	struct mtx *mtx;
+	struct cam_sim *sim;
 	int s;
 
 	s = splsoftcam();
+	sim = xpt_path_sim(ccb->ccb_h.path);
+	if (sim->mtx == &Giant)
+		mtx = NULL;
+	else
+		mtx = sim->mtx;
 	if ((ccb->ccb_h.pinfo.index != CAM_UNQUEUED_INDEX)
 	 || ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_INPROG))
-		tsleep(&ccb->ccb_h.cbfcnp, PRIBIO, "cbwait", 0);
+		msleep(&ccb->ccb_h.cbfcnp, mtx, PRIBIO, "cbwait", 0);
 
 	splx(s);
 }
@@ -857,10 +895,13 @@ cam_periph_runccb(union ccb *ccb,
 		  cam_flags camflags, u_int32_t sense_flags,
 		  struct devstat *ds)
 {
+	struct cam_sim *sim;
 	int error;
  
 	error = 0;
-        
+	sim = xpt_path_sim(ccb->ccb_h.path);
+	mtx_assert(sim->mtx, MA_OWNED);
+
 	/*
 	 * If the user has supplied a stats structure, and if we understand
 	 * this particular type of ccb, record the transaction start.
diff --git a/sys/cam/cam_periph.h b/sys/cam/cam_periph.h
index c5d02fc..8a6c1eb 100644
--- a/sys/cam/cam_periph.h
+++ b/sys/cam/cam_periph.h
@@ -104,6 +104,7 @@ struct cam_periph {
 	char			*periph_name;
 	struct cam_path		*path;	/* Compiled path to device */
 	void			*softc;
+	struct cam_sim		*sim;
 	u_int32_t		 unit_number;
 	cam_periph_type		 type;
 	u_int32_t		 flags;
@@ -113,6 +114,7 @@ struct cam_periph {
 #define CAM_PERIPH_INVALID		0x08
 #define CAM_PERIPH_NEW_DEV_FOUND	0x10
 #define CAM_PERIPH_RECOVERY_INPROG	0x20
+#define CAM_PERIPH_POLLED		0x40
 	u_int32_t		 immediate_priority;
 	u_int32_t		 refcount;
 	SLIST_HEAD(, ccb_hdr)	 ccb_list;	/* For "immediate" requests */
@@ -136,10 +138,12 @@ cam_status cam_periph_alloc(periph_ctor_t *periph_ctor,
 			    char *name, cam_periph_type type, struct cam_path *,
 			    ac_callback_t *, ac_code, void *arg);
 struct cam_periph *cam_periph_find(struct cam_path *path, char *name);
-int		cam_periph_lock(struct cam_periph *periph, int priority);
+void		cam_periph_lock(struct cam_periph *periph);
 void		cam_periph_unlock(struct cam_periph *periph);
 cam_status	cam_periph_acquire(struct cam_periph *periph);
 void		cam_periph_release(struct cam_periph *periph);
+int		cam_periph_hold(struct cam_periph *periph, int priority);
+void		cam_periph_unhold(struct cam_periph *periph);
 void		cam_periph_invalidate(struct cam_periph *periph);
 int		cam_periph_mapmem(union ccb *ccb,
 				  struct cam_periph_map_info *mapinfo);
diff --git a/sys/cam/cam_sim.c b/sys/cam/cam_sim.c
index 322915f..cc8f86d 100644
--- a/sys/cam/cam_sim.c
+++ b/sys/cam/cam_sim.c
@@ -33,6 +33,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 
 #include <cam/cam.h>
 #include <cam/cam_ccb.h>
@@ -58,39 +60,42 @@ cam_simq_free(struct cam_devq *devq)
 struct cam_sim *
 cam_sim_alloc(sim_action_func sim_action, sim_poll_func sim_poll,
 	      const char *sim_name, void *softc, u_int32_t unit,
-	      int max_dev_transactions,
+	      struct mtx *mtx, int max_dev_transactions,
 	      int max_tagged_dev_transactions, struct cam_devq *queue)
 {
 	struct cam_sim *sim;
 
-	/*
-	 * If this is the xpt layer creating a sim, then it's OK
-	 * to wait for an allocation.
-	 *
-	 * XXX Should we pass in a flag to indicate that wait is OK?
-	 */
-	if (strcmp(sim_name, "xpt") == 0)
-		sim = (struct cam_sim *)malloc(sizeof(struct cam_sim),
-					       M_CAMSIM, M_WAITOK);
-	else
-		sim = (struct cam_sim *)malloc(sizeof(struct cam_sim),
-					       M_CAMSIM, M_NOWAIT);
-
-	if (sim != NULL) {
-		sim->sim_action = sim_action;
-		sim->sim_poll = sim_poll;
-		sim->sim_name = sim_name;
-		sim->softc = softc;
-		sim->path_id = CAM_PATH_ANY;
-		sim->unit_number = unit;
-		sim->bus_id = 0;	/* set in xpt_bus_register */
-		sim->max_tagged_dev_openings = max_tagged_dev_transactions;
-		sim->max_dev_openings = max_dev_transactions;
-		sim->flags = 0;
-		callout_handle_init(&sim->c_handle);
-		sim->devq = queue;
+	if (mtx == NULL)
+		return (NULL);
+
+	sim = (struct cam_sim *)malloc(sizeof(struct cam_sim),
+	    M_CAMSIM, M_NOWAIT);
+
+	if (sim == NULL)
+		return (NULL);
+
+	sim->sim_action = sim_action;
+	sim->sim_poll = sim_poll;
+	sim->sim_name = sim_name;
+	sim->softc = softc;
+	sim->path_id = CAM_PATH_ANY;
+	sim->unit_number = unit;
+	sim->bus_id = 0;	/* set in xpt_bus_register */
+	sim->max_tagged_dev_openings = max_tagged_dev_transactions;
+	sim->max_dev_openings = max_dev_transactions;
+	sim->flags = 0;
+	sim->devq = queue;
+	sim->mtx = mtx;
+	if (mtx == &Giant) {
+		sim->flags |= 0;
+		callout_init(&sim->callout, 0);
+	} else {
+		sim->flags |= CAM_SIM_MPSAFE;
+		callout_init(&sim->callout, 1);
 	}
 
+	SLIST_INIT(&sim->ccb_freeq);
+
 	return (sim);
 }
 
diff --git a/sys/cam/cam_sim.h b/sys/cam/cam_sim.h
index c4e3910..8976529 100644
--- a/sys/cam/cam_sim.h
+++ b/sys/cam/cam_sim.h
@@ -56,6 +56,7 @@ struct cam_sim *  cam_sim_alloc(sim_action_func sim_action,
 				const char *sim_name,
 				void *softc,
 				u_int32_t unit,
+				struct mtx *mtx,
 				int max_dev_transactions,
 				int max_tagged_dev_transactions,
 				struct cam_devq *queue);
@@ -90,17 +91,33 @@ struct cam_sim {
 	sim_poll_func		sim_poll;
 	const char		*sim_name;
 	void			*softc;
+	struct mtx		*mtx;
 	u_int32_t		path_id;/* The Boot device may set this to 0? */
 	u_int32_t		unit_number;
 	u_int32_t		bus_id;
 	int			max_tagged_dev_openings;
 	int			max_dev_openings;
 	u_int32_t		flags;
-#define		CAM_SIM_REL_TIMEOUT_PENDING	0x01
-	struct callout_handle	c_handle;
+#define	CAM_SIM_REL_TIMEOUT_PENDING	0x01
+#define	CAM_SIM_MPSAFE			0x02
+	struct callout		callout;
 	struct cam_devq 	*devq;	/* Device Queue to use for this SIM */
+
+	/* "Pool" of inactive ccbs managed by xpt_alloc_ccb and xpt_free_ccb */
+	SLIST_HEAD(,ccb_hdr)	ccb_freeq;
+	/*
+	 * Maximum size of ccb pool.  Modified as devices are added/removed
+	 * or have their * opening counts changed.
+	 */
+	u_int			max_ccbs;
+	/* Current count of allocated ccbs */
+	u_int			ccb_count;
+
 };
 
+#define CAM_SIM_LOCK(sim)	mtx_lock((sim)->mtx);
+#define CAM_SIM_UNLOCK(sim)	mtx_unlock((sim)->mtx);
+
 static __inline u_int32_t
 cam_sim_path(struct cam_sim *sim)
 {
diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index c123b5c..d13209e 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/md5.h>
 #include <sys/interrupt.h>
 #include <sys/sbuf.h>
+#include <sys/taskqueue.h>
 
 #include <sys/lock.h>
 #include <sys/mutex.h>
@@ -70,6 +71,12 @@ __FBSDID("$FreeBSD$");
 /* Datastructures internal to the xpt layer */
 MALLOC_DEFINE(M_CAMXPT, "CAM XPT", "CAM XPT buffers");
 
+/* Object for defering XPT actions to a taskqueue */
+struct xpt_task {
+	struct task	task;
+	void		*data;
+};
+
 /*
  * Definition of an async handler callback block.  These are used to add
  * SIMs and peripherals to the async callback lists.
@@ -84,7 +91,6 @@ struct async_node {
 
 SLIST_HEAD(async_list, async_node);
 SLIST_HEAD(periph_list, cam_periph);
-static STAILQ_HEAD(highpowerlist, ccb_hdr) highpowerq;
 
 /*
  * This is the maximum number of high powered commands (e.g. start unit)
@@ -94,9 +100,6 @@ static STAILQ_HEAD(highpowerlist, ccb_hdr) highpowerq;
 #define CAM_MAX_HIGHPOWER  4
 #endif
 
-/* number of high powered commands that can go through right now */
-static int num_highpower = CAM_MAX_HIGHPOWER;
-
 /*
  * Structure for queueing a device in a run queue.
  * There is one run queue for allocating new ccbs,
@@ -117,6 +120,7 @@ struct cam_ed {
 	struct	cam_ed_qinfo alloc_ccb_entry;
 	struct	cam_ed_qinfo send_ccb_entry;
 	struct	cam_et	 *target;
+	struct	cam_sim  *sim;
 	lun_id_t	 lun_id;
 	struct	camq drvq;		/*
 					 * Queue of type drivers wanting to do
@@ -158,7 +162,7 @@ struct cam_ed {
 #define	CAM_TAG_DELAY_COUNT		5
 	u_int32_t	 tag_saved_openings;
 	u_int32_t	 refcount;
-	struct		 callout_handle c_handle;
+	struct callout	 callout;
 };
 
 /*
@@ -242,8 +246,24 @@ typedef enum {
 } xpt_flags;
 
 struct xpt_softc {
-	xpt_flags	flags;
-	u_int32_t	generation;
+	xpt_flags		flags;
+	u_int32_t		xpt_generation;
+
+	/* number of high powered commands that can go through right now */
+	STAILQ_HEAD(highpowerlist, ccb_hdr)	highpowerq;
+	int			num_highpower;
+
+	/* queue for handling async rescan requests. */
+	TAILQ_HEAD(, ccb_hdr) ccb_scanq;
+
+	/* Registered busses */
+	TAILQ_HEAD(,cam_eb)	xpt_busses;
+	u_int			bus_generation;
+
+	struct intr_config_hook	*xpt_config_hook;
+
+	struct mtx		xpt_topo_lock;
+	struct mtx		xpt_lock;
 };
 
 static const char quantum[] = "QUANTUM";
@@ -647,14 +667,8 @@ typedef TAILQ_HEAD(cam_isrq, ccb_hdr) cam_isrq_t;
 static cam_isrq_t cam_bioq;
 static struct mtx cam_bioq_lock;
 
-/* "Pool" of inactive ccbs managed by xpt_alloc_ccb and xpt_free_ccb */
-static SLIST_HEAD(,ccb_hdr) ccb_freeq;
-static u_int xpt_max_ccbs;	/*
-				 * Maximum size of ccb pool.  Modified as
-				 * devices are added/removed or have their
-				 * opening counts changed.
-				 */
-static u_int xpt_ccb_count;	/* Current count of allocated ccbs */
+/* Pointers to software interrupt handlers */
+static void *cambio_ih;
 
 struct cam_periph *xpt_periph;
 
@@ -684,14 +698,13 @@ static d_ioctl_t xptioctl;
 
 static struct cdevsw xpt_cdevsw = {
 	.d_version =	D_VERSION,
-	.d_flags =	D_NEEDGIANT,
+	.d_flags =	0,
 	.d_open =	xptopen,
 	.d_close =	xptclose,
 	.d_ioctl =	xptioctl,
 	.d_name =	"xpt",
 };
 
-static struct intr_config_hook *xpt_config_hook;
 
 static void dead_sim_action(struct cam_sim *sim, union ccb *ccb);
 static void dead_sim_poll(struct cam_sim *sim);
@@ -705,9 +718,6 @@ static struct cam_sim cam_dead_sim = {
 
 #define SIM_DEAD(sim)	((sim) == &cam_dead_sim)
 
-/* Registered busses */
-static TAILQ_HEAD(,cam_eb) xpt_busses;
-static u_int bus_generation;
 
 /* Storage for debugging datastructures */
 #ifdef	CAMDEBUG
@@ -716,9 +726,6 @@ u_int32_t cam_dflags;
 u_int32_t cam_debug_delay;
 #endif
 
-/* Pointers to software interrupt handlers */
-static void *cambio_ih;
-
 #if defined(CAM_DEBUG_FLAGS) && !defined(CAMDEBUG)
 #error "You must have options CAMDEBUG to use options CAM_DEBUG_FLAGS"
 #endif
@@ -750,7 +757,7 @@ static moduledata_t cam_moduledata = {
 	NULL
 };
 
-static void	xpt_init(void *);
+static int	xpt_init(void *);
 
 DECLARE_MODULE(cam, cam_moduledata, SI_SUB_CONFIGURE, SI_ORDER_SECOND);
 MODULE_VERSION(cam, 1);
@@ -781,7 +788,7 @@ static int	 xpt_schedule_dev(struct camq *queue, cam_pinfo *dev_pinfo,
 static void	 xpt_run_dev_allocq(struct cam_eb *bus);
 static void	 xpt_run_dev_sendq(struct cam_eb *bus);
 static timeout_t xpt_release_devq_timeout;
-static timeout_t xpt_release_simq_timeout;
+static void	 xpt_release_simq_timeout(void *arg) __unused;
 static void	 xpt_release_bus(struct cam_eb *bus);
 static void	 xpt_release_devq_device(struct cam_ed *dev, u_int count,
 					 int run_queue);
@@ -813,11 +820,6 @@ static void	 xpt_finishconfig(struct cam_periph *periph, union ccb *ccb);
 static void	 xptaction(struct cam_sim *sim, union ccb *work_ccb);
 static void	 xptpoll(struct cam_sim *sim);
 static void	 camisr(void *);
-#if 0
-static void	 xptstart(struct cam_periph *periph, union ccb *work_ccb);
-static void	 xptasync(struct cam_periph *periph,
-			  u_int32_t code, cam_path *path);
-#endif
 static dev_match_ret	xptbusmatch(struct dev_match_pattern *patterns,
 				    u_int num_patterns, struct cam_eb *bus);
 static dev_match_ret	xptdevicematch(struct dev_match_pattern *patterns,
@@ -856,16 +858,8 @@ static xpt_targetfunc_t	xptdeftargetfunc;
 static xpt_devicefunc_t	xptdefdevicefunc;
 static xpt_periphfunc_t	xptdefperiphfunc;
 static int		xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg);
-#ifdef notusedyet
-static int		xpt_for_all_targets(xpt_targetfunc_t *tr_func,
-					    void *arg);
-#endif
 static int		xpt_for_all_devices(xpt_devicefunc_t *tr_func,
 					    void *arg);
-#ifdef notusedyet
-static int		xpt_for_all_periphs(xpt_periphfunc_t *tr_func,
-					    void *arg);
-#endif
 static xpt_devicefunc_t	xptsetasyncfunc;
 static xpt_busfunc_t	xptsetasyncbusfunc;
 static cam_status	xptregister(struct cam_periph *periph,
@@ -996,9 +990,6 @@ xptdone(struct cam_periph *periph, union ccb *done_ccb)
 static int
 xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 {
-	int unit;
-
-	unit = minor(dev) & 0xff;
 
 	/*
 	 * Only allow read-write access.
@@ -1010,22 +1001,14 @@ xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	 * We don't allow nonblocking access.
 	 */
 	if ((flags & O_NONBLOCK) != 0) {
-		printf("xpt%d: can't do nonblocking access\n", unit);
+		printf("%s: can't do nonblocking access\n", devtoname(dev));
 		return(ENODEV);
 	}
 
-	/*
-	 * We only have one transport layer right now.  If someone accesses
-	 * us via something other than minor number 1, point out their
-	 * mistake.
-	 */
-	if (unit != 0) {
-		printf("xptopen: got invalid xpt unit %d\n", unit);
-		return(ENXIO);
-	}
-
 	/* Mark ourselves open */
+	mtx_lock(&xsoftc.xpt_lock);
 	xsoftc.flags |= XPT_FLAG_OPEN;
+	mtx_unlock(&xsoftc.xpt_lock);
 	
 	return(0);
 }
@@ -1033,43 +1016,27 @@ xptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 static int
 xptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 {
-	int unit;
-
-	unit = minor(dev) & 0xff;
-
-	/*
-	 * We only have one transport layer right now.  If someone accesses
-	 * us via something other than minor number 1, point out their
-	 * mistake.
-	 */
-	if (unit != 0) {
-		printf("xptclose: got invalid xpt unit %d\n", unit);
-		return(ENXIO);
-	}
 
 	/* Mark ourselves closed */
+	mtx_lock(&xsoftc.xpt_lock);
 	xsoftc.flags &= ~XPT_FLAG_OPEN;
+	mtx_unlock(&xsoftc.xpt_lock);
 
 	return(0);
 }
 
+/*
+ * Don't automatically grab the xpt softc lock here even though this is going
+ * through the xpt device.  The xpt device is really just a back door for
+ * accessing other devices and SIMs, so the right thing to do is to grab
+ * the appropriate SIM lock once the bus/SIM is located.
+ */
 static int
 xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 {
-	int unit, error;
+	int error;
 
 	error = 0;
-	unit = minor(dev) & 0xff;
-
-	/*
-	 * We only have one transport layer right now.  If someone accesses
-	 * us via something other than minor number 1, point out their
-	 * mistake.
-	 */
-	if (unit != 0) {
-		printf("xptioctl: got invalid xpt unit %d\n", unit);
-		return(ENXIO);
-	}
 
 	switch(cmd) {
 	/*
@@ -1081,9 +1048,16 @@ xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td
 	case CAMIOCOMMAND: {
 		union ccb *ccb;
 		union ccb *inccb;
+		struct cam_eb *bus;
 
 		inccb = (union ccb *)addr;
 
+		bus = xpt_find_bus(inccb->ccb_h.path_id);
+		if (bus == NULL) {
+			error = EINVAL;
+			break;
+		}
+
 		switch(inccb->ccb_h.func_code) {
 		case XPT_SCAN_BUS:
 		case XPT_RESET_BUS:
@@ -1097,7 +1071,9 @@ xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td
 		case XPT_ENG_INQ:
 		case XPT_SCAN_LUN:
 
-			ccb = xpt_alloc_ccb();
+			ccb = xpt_alloc_ccb(bus->sim);
+
+			CAM_SIM_LOCK(bus->sim);
 
 			/*
 			 * Create a path using the bus, target, and lun the
@@ -1109,6 +1085,7 @@ xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td
 					    inccb->ccb_h.target_lun) !=
 					    CAM_REQ_CMP){
 				error = EINVAL;
+				CAM_SIM_UNLOCK(bus->sim);
 				xpt_free_ccb(ccb);
 				break;
 			}
@@ -1121,6 +1098,7 @@ xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td
 			bcopy(ccb, inccb, sizeof(union ccb));
 			xpt_free_path(ccb->ccb_h.path);
 			xpt_free_ccb(ccb);
+			CAM_SIM_UNLOCK(bus->sim);
 			break;
 
 		case XPT_DEBUG: {
@@ -1131,6 +1109,8 @@ xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td
 			 * allocate it on the stack.
 			 */
 
+			CAM_SIM_LOCK(bus->sim);
+
 			/*
 			 * Create a path using the bus, target, and lun the
 			 * user passed in.
@@ -1149,6 +1129,7 @@ xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td
 			xpt_merge_ccb(&ccb, inccb);
 			ccb.ccb_h.cbfcnp = xptdone;
 			xpt_action(&ccb);
+			CAM_SIM_UNLOCK(bus->sim);
 			bcopy(&ccb, inccb, sizeof(union ccb));
 			xpt_free_path(ccb.ccb_h.path);
 			break;
@@ -1268,7 +1249,7 @@ xptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td
 		/* Keep the list from changing while we traverse it */
 		s = splcam();
 ptstartover:
-		cur_generation = xsoftc.generation;
+		cur_generation = xsoftc.xpt_generation;
 
 		/* first find our driver in the list of drivers */
 		for (p_drv = periph_drivers; *p_drv != NULL; p_drv++)
@@ -1301,7 +1282,7 @@ ptstartover:
 				splx(s);
 				s = splcam();
 				splbreaknum = 100;
-				if (cur_generation != xsoftc.generation)
+				if (cur_generation != xsoftc.xpt_generation)
 				       goto ptstartover;
 			}
 		}
@@ -1402,11 +1383,16 @@ ptstartover:
 static int
 cam_module_event_handler(module_t mod, int what, void *arg)
 {
-	if (what == MOD_LOAD) {
-		xpt_init(NULL);
-	} else if (what == MOD_UNLOAD) {
+	int error;
+
+	switch (what) {
+	case MOD_LOAD:
+		if ((error = xpt_init(NULL)) != 0)
+			return (error);
+		break;
+	case MOD_UNLOAD:
 		return EBUSY;
-	} else {
+	default:
 		return EOPNOTSUPP;
 	}
 
@@ -1414,22 +1400,40 @@ cam_module_event_handler(module_t mod, int what, void *arg)
 }
 
 /* thread to handle bus rescans */
-static TAILQ_HEAD(, ccb_hdr) ccb_scanq;
 static void
 xpt_scanner_thread(void *dummy)
 {
-	mtx_lock(&Giant);
+	cam_isrq_t	queue;
+	union ccb	*ccb;
+	struct cam_sim	*sim;
+
 	for (;;) {
-		union ccb *ccb;
-		tsleep(&ccb_scanq, PRIBIO, "ccb_scanq", 0);
-		while ((ccb = (union ccb *)TAILQ_FIRST(&ccb_scanq)) != NULL) {
-			TAILQ_REMOVE(&ccb_scanq, &ccb->ccb_h, sim_links.tqe);
+		/*
+		 * Wait for a rescan request to come in.  When it does, splice
+		 * it onto a queue from local storage so that the xpt lock
+		 * doesn't need to be held while the requests are being
+		 * processed.
+		 */
+		xpt_lock_buses();
+		msleep(&xsoftc.ccb_scanq, &xsoftc.xpt_topo_lock, PRIBIO,
+		    "ccb_scanq", 0);
+		TAILQ_INIT(&queue);
+		TAILQ_CONCAT(&queue, &xsoftc.ccb_scanq, sim_links.tqe);
+		xpt_unlock_buses();
+
+		while ((ccb = (union ccb *)TAILQ_FIRST(&queue)) != NULL) {
+			TAILQ_REMOVE(&queue, &ccb->ccb_h, sim_links.tqe);
+
+			sim = ccb->ccb_h.path->bus->sim;
+			mtx_lock(sim->mtx);
+
 			ccb->ccb_h.func_code = XPT_SCAN_BUS;
 			ccb->ccb_h.cbfcnp = xptdone;
 			xpt_setup_ccb(&ccb->ccb_h, ccb->ccb_h.path, 5);
 			cam_periph_runccb(ccb, NULL, 0, 0, NULL);
 			xpt_free_path(ccb->ccb_h.path);
 			xpt_free_ccb(ccb);
+			mtx_unlock(sim->mtx);
 		}
 	}
 }
@@ -1438,24 +1442,27 @@ void
 xpt_rescan(union ccb *ccb)
 {
 	struct ccb_hdr *hdr;
-	GIANT_REQUIRED;
+
 	/*
 	 * Don't make duplicate entries for the same paths.
 	 */
-	TAILQ_FOREACH(hdr, &ccb_scanq, sim_links.tqe) {
+	xpt_lock_buses();
+	TAILQ_FOREACH(hdr, &xsoftc.ccb_scanq, sim_links.tqe) {
 		if (xpt_path_comp(hdr->path, ccb->ccb_h.path) == 0) {
+			xpt_unlock_buses();
 			xpt_print(ccb->ccb_h.path, "rescan already queued\n");
 			xpt_free_path(ccb->ccb_h.path);
 			xpt_free_ccb(ccb);
 			return;
 		}
 	}
-	TAILQ_INSERT_TAIL(&ccb_scanq, &ccb->ccb_h, sim_links.tqe);
-	wakeup(&ccb_scanq);
+	TAILQ_INSERT_TAIL(&xsoftc.ccb_scanq, &ccb->ccb_h, sim_links.tqe);
+	wakeup(&xsoftc.ccb_scanq);
+	xpt_unlock_buses();
 }
 
 /* Functions accessed by the peripheral drivers */
-static void
+static int
 xpt_init(void *dummy)
 {
 	struct cam_sim *xpt_sim;
@@ -1463,13 +1470,15 @@ xpt_init(void *dummy)
 	struct cam_devq *devq;
 	cam_status status;
 
-	TAILQ_INIT(&xpt_busses);
+	TAILQ_INIT(&xsoftc.xpt_busses);
 	TAILQ_INIT(&cam_bioq);
-	SLIST_INIT(&ccb_freeq);
-	TAILQ_INIT(&ccb_scanq);
-	STAILQ_INIT(&highpowerq);
+	TAILQ_INIT(&xsoftc.ccb_scanq);
+	STAILQ_INIT(&xsoftc.highpowerq);
+	xsoftc.num_highpower = CAM_MAX_HIGHPOWER;
 
 	mtx_init(&cam_bioq_lock, "CAM BIOQ lock", NULL, MTX_DEF);
+	mtx_init(&xsoftc.xpt_lock, "XPT lock", NULL, MTX_DEF);
+	mtx_init(&xsoftc.xpt_topo_lock, "XPT topology lock", NULL, MTX_DEF);
 
 	/*
 	 * The xpt layer is, itself, the equivelent of a SIM.
@@ -1483,15 +1492,20 @@ xpt_init(void *dummy)
 				"xpt",
 				/*softc*/NULL,
 				/*unit*/0,
+				/*mtx*/&xsoftc.xpt_lock,
 				/*max_dev_transactions*/0,
 				/*max_tagged_dev_transactions*/0,
 				devq);
-	xpt_max_ccbs = 16;
-				
+	if (xpt_sim == NULL)
+		return (ENOMEM);
+
+	xpt_sim->max_ccbs = 16;
+
+	mtx_lock(&xsoftc.xpt_lock);
 	if ((status = xpt_bus_register(xpt_sim, /*bus #*/0)) != CAM_SUCCESS) {
 		printf("xpt_init: xpt_bus_register failed with status %#x,"
 		       " failing attach\n", status);
-		return;
+		return (EINVAL);
 	}
 
 	/*
@@ -1504,30 +1518,29 @@ xpt_init(void *dummy)
 				      CAM_LUN_WILDCARD)) != CAM_REQ_CMP) {
 		printf("xpt_init: xpt_create_path failed with status %#x,"
 		       " failing attach\n", status);
-		return;
+		return (EINVAL);
 	}
 
 	cam_periph_alloc(xptregister, NULL, NULL, NULL, "xpt", CAM_PERIPH_BIO,
-			 path, NULL, 0, NULL);
+			 path, NULL, 0, xpt_sim);
 	xpt_free_path(path);
-
-	xpt_sim->softc = xpt_periph;
+	mtx_unlock(&xsoftc.xpt_lock);
 
 	/*
 	 * Register a callback for when interrupts are enabled.
 	 */
-	xpt_config_hook =
+	xsoftc.xpt_config_hook =
 	    (struct intr_config_hook *)malloc(sizeof(struct intr_config_hook),
 					      M_TEMP, M_NOWAIT | M_ZERO);
-	if (xpt_config_hook == NULL) {
+	if (xsoftc.xpt_config_hook == NULL) {
 		printf("xpt_init: Cannot malloc config hook "
 		       "- failing attach\n");
-		return;
+		return (ENOMEM);
 	}
 
-	xpt_config_hook->ich_func = xpt_config;
-	if (config_intrhook_establish(xpt_config_hook) != 0) {
-		free (xpt_config_hook, M_TEMP);
+	xsoftc.xpt_config_hook->ich_func = xpt_config;
+	if (config_intrhook_establish(xsoftc.xpt_config_hook) != 0) {
+		free (xsoftc.xpt_config_hook, M_TEMP);
 		printf("xpt_init: config_intrhook_establish failed "
 		       "- failing attach\n");
 	}
@@ -1537,20 +1550,25 @@ xpt_init(void *dummy)
 		printf("xpt_init: failed to create rescan thread\n");
 	}
 	/* Install our software interrupt handlers */
-	swi_add(NULL, "cambio", camisr, &cam_bioq, SWI_CAMBIO, 0, &cambio_ih);
+	swi_add(NULL, "cambio", camisr, &cam_bioq, SWI_CAMBIO, INTR_MPSAFE, &cambio_ih);
+
+	return (0);
 }
 
 static cam_status
 xptregister(struct cam_periph *periph, void *arg)
 {
+	struct cam_sim *xpt_sim;
+
 	if (periph == NULL) {
 		printf("xptregister: periph was NULL!!\n");
 		return(CAM_REQ_CMP_ERR);
 	}
 
-	periph->softc = NULL;
-
+	xpt_sim = (struct cam_sim *)arg;
+	xpt_sim->softc = periph;
 	xpt_periph = periph;
+	periph->softc = NULL;
 
 	return(CAM_REQ_CMP);
 }
@@ -1562,7 +1580,7 @@ xpt_add_periph(struct cam_periph *periph)
 	int32_t	 status;
 	struct periph_list *periph_head;
 
-	GIANT_REQUIRED;
+	mtx_assert(periph->sim->mtx, MA_OWNED);
 
 	device = periph->path->device;
 
@@ -1589,7 +1607,7 @@ xpt_add_periph(struct cam_periph *periph)
 		splx(s);
 	}
 
-	xsoftc.generation++;
+	atomic_add_int(&xsoftc.xpt_generation, 1);
 
 	return (status);
 }
@@ -1599,7 +1617,7 @@ xpt_remove_periph(struct cam_periph *periph)
 {
 	struct cam_ed *device;
 
-	GIANT_REQUIRED;
+	mtx_assert(periph->sim->mtx, MA_OWNED);
 
 	device = periph->path->device;
 
@@ -1620,7 +1638,7 @@ xpt_remove_periph(struct cam_periph *periph)
 		splx(s);
 	}
 
-	xsoftc.generation++;
+	atomic_add_int(&xsoftc.xpt_generation, 1);
 
 }
 
@@ -1636,7 +1654,7 @@ xpt_announce_periph(struct cam_periph *periph, char *announce_string)
 	u_int	mb;
 	int	s;
 
-	GIANT_REQUIRED;
+	mtx_assert(periph->sim->mtx, MA_OWNED);
 
 	path = periph->path;
 	/*
@@ -2147,7 +2165,7 @@ xptedtbusfunc(struct cam_eb *bus, void *arg)
 
 			cdm->pos.cookie.bus = bus;
 			cdm->pos.generations[CAM_BUS_GENERATION]=
-				bus_generation;
+				xsoftc.bus_generation;
 			cdm->status = CAM_DEV_MATCH_MORE;
 			return(0);
 		}
@@ -2279,7 +2297,7 @@ xptedtdevicefunc(struct cam_ed *device, void *arg)
 
 			cdm->pos.cookie.bus = device->target->bus;
 			cdm->pos.generations[CAM_BUS_GENERATION]=
-				bus_generation;
+				xsoftc.bus_generation;
 			cdm->pos.cookie.target = device->target;
 			cdm->pos.generations[CAM_TARGET_GENERATION] =
 				device->target->bus->generation;
@@ -2389,7 +2407,7 @@ xptedtperiphfunc(struct cam_periph *periph, void *arg)
 
 			cdm->pos.cookie.bus = periph->path->bus;
 			cdm->pos.generations[CAM_BUS_GENERATION]=
-				bus_generation;
+				xsoftc.bus_generation;
 			cdm->pos.cookie.target = periph->path->target;
 			cdm->pos.generations[CAM_TARGET_GENERATION] =
 				periph->path->bus->generation;
@@ -2434,7 +2452,7 @@ xptedtmatch(struct ccb_dev_match *cdm)
 	 */
 	if ((cdm->pos.position_type & CAM_DEV_POS_BUS)
 	 && (cdm->pos.generations[CAM_BUS_GENERATION] != 0)
-	 && (cdm->pos.generations[CAM_BUS_GENERATION] != bus_generation)) {
+	 && (cdm->pos.generations[CAM_BUS_GENERATION] != xsoftc.bus_generation)) {
 		cdm->status = CAM_DEV_MATCH_LIST_CHANGED;
 		return(0);
 	}
@@ -2633,15 +2651,21 @@ xptbustraverse(struct cam_eb *start_bus, xpt_busfunc_t *tr_func, void *arg)
 
 	retval = 1;
 
-	for (bus = (start_bus ? start_bus : TAILQ_FIRST(&xpt_busses));
+	mtx_lock(&xsoftc.xpt_topo_lock);
+	for (bus = (start_bus ? start_bus : TAILQ_FIRST(&xsoftc.xpt_busses));
 	     bus != NULL;
 	     bus = next_bus) {
 		next_bus = TAILQ_NEXT(bus, links);
 
+		mtx_unlock(&xsoftc.xpt_topo_lock);
+		mtx_lock(bus->sim->mtx);
 		retval = tr_func(bus, arg);
+		mtx_unlock(bus->sim->mtx);
 		if (retval == 0)
 			return(retval);
+		mtx_lock(&xsoftc.xpt_topo_lock);
 	}
+	mtx_unlock(&xsoftc.xpt_topo_lock);
 
 	return(retval);
 }
@@ -2852,23 +2876,6 @@ xpt_for_all_busses(xpt_busfunc_t *tr_func, void *arg)
 	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
 }
 
-#ifdef notusedyet
-/*
- * Execute the given function for every target in the EDT.
- */
-static int
-xpt_for_all_targets(xpt_targetfunc_t *tr_func, void *arg)
-{
-	struct xpt_traverse_config tr_config;
-
-	tr_config.depth = XPT_DEPTH_TARGET;
-	tr_config.tr_func = tr_func;
-	tr_config.tr_arg = arg;
-
-	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
-}
-#endif /* notusedyet */
-
 /*
  * Execute the given function for every device in the EDT.
  */
@@ -2884,23 +2891,6 @@ xpt_for_all_devices(xpt_devicefunc_t *tr_func, void *arg)
 	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
 }
 
-#ifdef notusedyet
-/*
- * Execute the given function for every peripheral in the EDT.
- */
-static int
-xpt_for_all_periphs(xpt_periphfunc_t *tr_func, void *arg)
-{
-	struct xpt_traverse_config tr_config;
-
-	tr_config.depth = XPT_DEPTH_PERIPH;
-	tr_config.tr_func = tr_func;
-	tr_config.tr_arg = arg;
-
-	return(xptbustraverse(NULL, xptdefbusfunc, &tr_config));
-}
-#endif /* notusedyet */
-
 static int
 xptsetasyncfunc(struct cam_ed *device, void *arg)
 {
@@ -2959,13 +2949,93 @@ xptsetasyncbusfunc(struct cam_eb *bus, void *arg)
 	return(1);
 }
 
+static void
+xpt_action_sasync_cb(void *context, int pending)
+{
+	union ccb *start_ccb;
+	struct xpt_task *task;
+	struct ccb_setasync *csa;
+	struct async_node *cur_entry;
+	struct async_list *async_head;
+	u_int32_t added;
+	int s;
+
+	task = (struct xpt_task *)context;
+	start_ccb = (union ccb *)task->data;
+	csa = &start_ccb->csa;
+	added = csa->event_enable;
+	async_head = &csa->ccb_h.path->device->asyncs;
+
+	/*
+	 * If there is already an entry for us, simply
+	 * update it.
+	 */
+	s = splcam();
+	mtx_lock(csa->ccb_h.path->bus->sim->mtx);
+	cur_entry = SLIST_FIRST(async_head);
+	while (cur_entry != NULL) {
+		if ((cur_entry->callback_arg == csa->callback_arg)
+		 && (cur_entry->callback == csa->callback))
+			break;
+		cur_entry = SLIST_NEXT(cur_entry, links);
+	}
+
+	if (cur_entry != NULL) {
+	 	/*
+		 * If the request has no flags set,
+		 * remove the entry.
+		 */
+		added &= ~cur_entry->event_enable;
+		if (csa->event_enable == 0) {
+			SLIST_REMOVE(async_head, cur_entry,
+				     async_node, links);
+			csa->ccb_h.path->device->refcount--;
+			free(cur_entry, M_CAMXPT);
+		} else {
+			cur_entry->event_enable = csa->event_enable;
+		}
+	} else {
+		cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
+				   M_NOWAIT);
+		if (cur_entry == NULL) {
+			splx(s);
+			goto out;
+		}
+		cur_entry->event_enable = csa->event_enable;
+		cur_entry->callback_arg = csa->callback_arg;
+		cur_entry->callback = csa->callback;
+		SLIST_INSERT_HEAD(async_head, cur_entry, links);
+		csa->ccb_h.path->device->refcount++;
+	}
+	mtx_unlock(csa->ccb_h.path->bus->sim->mtx);
+
+	if ((added & AC_FOUND_DEVICE) != 0) {
+		/*
+		 * Get this peripheral up to date with all
+		 * the currently existing devices.
+		 */
+		xpt_for_all_devices(xptsetasyncfunc, cur_entry);
+	}
+	if ((added & AC_PATH_REGISTERED) != 0) {
+		/*
+		 * Get this peripheral up to date with all
+		 * the currently existing busses.
+		 */
+		xpt_for_all_busses(xptsetasyncbusfunc, cur_entry);
+		}
+	splx(s);
+
+out:
+	xpt_free_path(start_ccb->ccb_h.path);
+	xpt_free_ccb(start_ccb);
+	free(task, M_CAMXPT);
+}
+
 void
 xpt_action(union ccb *start_ccb)
 {
 	int iopl;
 
-	GIANT_REQUIRED;
-
 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("xpt_action\n"));
 
 	start_ccb->ccb_h.status = CAM_REQ_INPROG;
@@ -3363,73 +3433,42 @@ xpt_action(union ccb *start_ccb)
 	}
 	case XPT_SASYNC_CB:
 	{
-		struct ccb_setasync *csa;
-		struct async_node *cur_entry;
-		struct async_list *async_head;
-		u_int32_t added;
-		int s;
-
-		csa = &start_ccb->csa;
-		added = csa->event_enable;
-		async_head = &csa->ccb_h.path->device->asyncs;
+		union ccb *task_ccb;
+		struct xpt_task *task;
 
 		/*
-		 * If there is already an entry for us, simply
-		 * update it.
+		 * Need to decouple this operation via a taqskqueue so that
+		 * the locking doesn't become a mess.  Clone the ccb so that
+		 * we own the memory and can free it later.
 		 */
-		s = splcam();
-		cur_entry = SLIST_FIRST(async_head);
-		while (cur_entry != NULL) {
-			if ((cur_entry->callback_arg == csa->callback_arg)
-			 && (cur_entry->callback == csa->callback))
-				break;
-			cur_entry = SLIST_NEXT(cur_entry, links);
+		task_ccb = malloc(sizeof(union ccb), M_CAMXPT, M_NOWAIT);
+		if (task_ccb == NULL) {
+			start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
+			break;
 		}
-
-		if (cur_entry != NULL) {
-		 	/*
-			 * If the request has no flags set,
-			 * remove the entry.
-			 */
-			added &= ~cur_entry->event_enable;
-			if (csa->event_enable == 0) {
-				SLIST_REMOVE(async_head, cur_entry,
-					     async_node, links);
-				csa->ccb_h.path->device->refcount--;
-				free(cur_entry, M_CAMXPT);
-			} else {
-				cur_entry->event_enable = csa->event_enable;
-			}
-		} else {
-			cur_entry = malloc(sizeof(*cur_entry), M_CAMXPT,
-					   M_NOWAIT);
-			if (cur_entry == NULL) {
-				splx(s);
-				csa->ccb_h.status = CAM_RESRC_UNAVAIL;
-				break;
-			}
-			cur_entry->event_enable = csa->event_enable;
-			cur_entry->callback_arg = csa->callback_arg;
-			cur_entry->callback = csa->callback;
-			SLIST_INSERT_HEAD(async_head, cur_entry, links);
-			csa->ccb_h.path->device->refcount++;
+		bcopy(start_ccb, task_ccb, sizeof(union ccb));
+		if (xpt_create_path(&task_ccb->ccb_h.path, NULL,
+				    start_ccb->ccb_h.path_id,
+				    start_ccb->ccb_h.target_id,
+				    start_ccb->ccb_h.target_lun) !=
+				    CAM_REQ_CMP) {
+			start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
+			xpt_free_ccb(task_ccb);
+			break;
 		}
 
-		if ((added & AC_FOUND_DEVICE) != 0) {
-			/*
-			 * Get this peripheral up to date with all
-			 * the currently existing devices.
-			 */
-			xpt_for_all_devices(xptsetasyncfunc, cur_entry);
-		}
-		if ((added & AC_PATH_REGISTERED) != 0) {
-			/*
-			 * Get this peripheral up to date with all
-			 * the currently existing busses.
-			 */
-			xpt_for_all_busses(xptsetasyncbusfunc, cur_entry);
+		task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
+		if (task == NULL) {
+			start_ccb->ccb_h.status = CAM_RESRC_UNAVAIL;
+			xpt_free_path(task_ccb->ccb_h.path);
+			xpt_free_ccb(task_ccb);
+			break;
 		}
-		splx(s);
+
+		TASK_INIT(&task->task, 0, xpt_action_sasync_cb, task);
+		task->data = task_ccb;
+		taskqueue_enqueue(taskqueue_thread, &task->task);
+
 		start_ccb->ccb_h.status = CAM_REQ_CMP;
 		break;
 	}
@@ -3476,17 +3515,15 @@ xpt_action(union ccb *start_ccb)
 				 * is sufficient for releasing the queue.
 				 */
 				start_ccb->ccb_h.flags &= ~CAM_DEV_QFREEZE;
-				untimeout(xpt_release_devq_timeout,
-					  dev, dev->c_handle);
+				callout_stop(&dev->callout);
 			} else {
 
 				start_ccb->ccb_h.flags |= CAM_DEV_QFREEZE;
 			}
 
-			dev->c_handle =
-				timeout(xpt_release_devq_timeout,
-					dev,
-					(crs->release_timeout * hz) / 1000);
+			callout_reset(&dev->callout,
+			    (crs->release_timeout * hz) / 1000,
+			    xpt_release_devq_timeout, dev);
 
 			dev->flags |= CAM_DEV_REL_TIMEOUT_PENDING;
 
@@ -3601,13 +3638,13 @@ xpt_polled_action(union ccb *start_ccb)
 	struct	  cam_devq *devq;
 	struct	  cam_ed *dev;
 
-	GIANT_REQUIRED;
 
 	timeout = start_ccb->ccb_h.timeout;
 	sim = start_ccb->ccb_h.path->bus->sim;
 	devq = sim->devq;
 	dev = start_ccb->ccb_h.path->device;
 
+	mtx_assert(sim->mtx, MA_OWNED);
 	s = splcam();
 
 	/*
@@ -3664,7 +3701,7 @@ xpt_schedule(struct cam_periph *perph, u_int32_t new_priority)
 	int s;
 	int runq;
 
-	GIANT_REQUIRED;
+	mtx_assert(perph->sim->mtx, MA_OWNED);
 
 	CAM_DEBUG(perph->path, CAM_DEBUG_TRACE, ("xpt_schedule\n"));
 	device = perph->path->device;
@@ -3885,7 +3922,8 @@ xpt_run_dev_sendq(struct cam_eb *bus)
 
 		if ((work_ccb->ccb_h.flags & CAM_HIGH_POWER) != 0) {
 
-		 	if (num_highpower <= 0) {
+			mtx_lock(&xsoftc.xpt_lock);
+		 	if (xsoftc.num_highpower <= 0) {
 				/*
 				 * We got a high power command, but we
 				 * don't have any available slots.  Freeze
@@ -3893,7 +3931,7 @@ xpt_run_dev_sendq(struct cam_eb *bus)
 				 * available.
 				 */
 				device->qfrozen_cnt++;
-				STAILQ_INSERT_TAIL(&highpowerq, 
+				STAILQ_INSERT_TAIL(&xsoftc.highpowerq, 
 						   &work_ccb->ccb_h, 
 						   xpt_links.stqe);
 
@@ -3904,8 +3942,9 @@ xpt_run_dev_sendq(struct cam_eb *bus)
 				 * Consume a high power slot while
 				 * this ccb runs.
 				 */
-				num_highpower--;
+				xsoftc.num_highpower--;
 			}
+			mtx_unlock(&xsoftc.xpt_lock);
 		}
 		devq->active_dev = device;
 		cam_ccbq_remove_ccb(&device->ccbq, work_ccb);
@@ -3972,7 +4011,6 @@ xpt_run_dev_sendq(struct cam_eb *bus)
 void
 xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
 {
-	GIANT_REQUIRED;
 
 	/*
 	 * Pull fields that are valid for peripheral drivers to set
@@ -3989,7 +4027,6 @@ xpt_merge_ccb(union ccb *master_ccb, union ccb *slave_ccb)
 void
 xpt_setup_ccb(struct ccb_hdr *ccb_h, struct cam_path *path, u_int32_t priority)
 {
-	GIANT_REQUIRED;
 
 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_setup_ccb\n"));
 	ccb_h->pinfo.priority = priority;
@@ -4017,8 +4054,6 @@ xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
 	struct	   cam_path *path;
 	cam_status status;
 
-	GIANT_REQUIRED;
-
 	path = (struct cam_path *)malloc(sizeof(*path), M_CAMXPT, M_NOWAIT);
 
 	if (path == NULL) {
@@ -4034,6 +4069,36 @@ xpt_create_path(struct cam_path **new_path_ptr, struct cam_periph *perph,
 	return (status);
 }
 
+cam_status
+xpt_create_path_unlocked(struct cam_path **new_path_ptr,
+			 struct cam_periph *periph, path_id_t path_id,
+			 target_id_t target_id, lun_id_t lun_id)
+{
+	struct	   cam_path *path;
+	struct	   cam_eb *bus = NULL;
+	cam_status status;
+	int	   need_unlock = 0;
+
+	path = (struct cam_path *)malloc(sizeof(*path), M_CAMXPT, M_WAITOK);
+
+	if (path_id != CAM_BUS_WILDCARD) {
+		bus = xpt_find_bus(path_id);
+		if (bus != NULL) {
+			need_unlock = 1;
+			mtx_lock(bus->sim->mtx);
+		}
+	}
+	status = xpt_compile_path(path, periph, path_id, target_id, lun_id);
+	if (need_unlock)
+		mtx_unlock(bus->sim->mtx);
+	if (status != CAM_REQ_CMP) {
+		free(path, M_CAMXPT);
+		path = NULL;
+	}
+	*new_path_ptr = path;
+	return (status);
+}
+
 static cam_status
 xpt_compile_path(struct cam_path *new_path, struct cam_periph *perph,
 		 path_id_t path_id, target_id_t target_id, lun_id_t lun_id)
@@ -4129,7 +4194,6 @@ xpt_release_path(struct cam_path *path)
 void
 xpt_free_path(struct cam_path *path)
 {
-	GIANT_REQUIRED;
 
 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_free_path\n"));
 	xpt_release_path(path);
@@ -4144,8 +4208,6 @@ xpt_free_path(struct cam_path *path)
 int
 xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
 {
-	GIANT_REQUIRED;
-
 	int retval = 0;
 
 	if (path1->bus != path2->bus) {
@@ -4180,7 +4242,7 @@ xpt_path_comp(struct cam_path *path1, struct cam_path *path2)
 void
 xpt_print_path(struct cam_path *path)
 {
-	GIANT_REQUIRED;
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
 	if (path == NULL)
 		printf("(nopath): ");
@@ -4225,7 +4287,7 @@ xpt_path_string(struct cam_path *path, char *str, size_t str_len)
 {
 	struct sbuf sb;
 
-	GIANT_REQUIRED;
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
 	sbuf_new(&sb, str, str_len, 0);
 
@@ -4263,7 +4325,7 @@ xpt_path_string(struct cam_path *path, char *str, size_t str_len)
 path_id_t
 xpt_path_path_id(struct cam_path *path)
 {
-	GIANT_REQUIRED;
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
 	return(path->bus->path_id);
 }
@@ -4271,7 +4333,7 @@ xpt_path_path_id(struct cam_path *path)
 target_id_t
 xpt_path_target_id(struct cam_path *path)
 {
-	GIANT_REQUIRED;
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
 	if (path->target != NULL)
 		return (path->target->target_id);
@@ -4282,7 +4344,7 @@ xpt_path_target_id(struct cam_path *path)
 lun_id_t
 xpt_path_lun_id(struct cam_path *path)
 {
-	GIANT_REQUIRED;
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
 	if (path->device != NULL)
 		return (path->device->lun_id);
@@ -4293,7 +4355,6 @@ xpt_path_lun_id(struct cam_path *path)
 struct cam_sim *
 xpt_path_sim(struct cam_path *path)
 {
-	GIANT_REQUIRED;
 
 	return (path->bus->sim);
 }
@@ -4301,7 +4362,7 @@ xpt_path_sim(struct cam_path *path)
 struct cam_periph*
 xpt_path_periph(struct cam_path *path)
 {
-	GIANT_REQUIRED;
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
 	return (path->periph);
 }
@@ -4319,34 +4380,38 @@ xpt_release_ccb(union ccb *free_ccb)
 	struct	 cam_path *path;
 	struct	 cam_ed *device;
 	struct	 cam_eb *bus;
-
-	GIANT_REQUIRED;
+	struct   cam_sim *sim;
 
 	CAM_DEBUG_PRINT(CAM_DEBUG_XPT, ("xpt_release_ccb\n"));
 	path = free_ccb->ccb_h.path;
 	device = path->device;
 	bus = path->bus;
+	sim = bus->sim;
 	s = splsoftcam();
+
+	mtx_assert(sim->mtx, MA_OWNED);
+
 	cam_ccbq_release_opening(&device->ccbq);
-	if (xpt_ccb_count > xpt_max_ccbs) {
+	if (sim->ccb_count > sim->max_ccbs) {
 		xpt_free_ccb(free_ccb);
-		xpt_ccb_count--;
+		sim->ccb_count--;
 	} else {
-		SLIST_INSERT_HEAD(&ccb_freeq, &free_ccb->ccb_h, xpt_links.sle);
+		SLIST_INSERT_HEAD(&sim->ccb_freeq, &free_ccb->ccb_h,
+		    xpt_links.sle);
 	}
-	if (bus->sim->devq == NULL) {
+	if (sim->devq == NULL) {
 		splx(s);
 		return;
 	}
-	bus->sim->devq->alloc_openings++;
-	bus->sim->devq->alloc_active--;
+	sim->devq->alloc_openings++;
+	sim->devq->alloc_active--;
 	/* XXX Turn this into an inline function - xpt_run_device?? */
 	if ((device_is_alloc_queued(device) == 0)
 	 && (device->drvq.entries > 0)) {
 		xpt_schedule_dev_allocq(bus, device);
 	}
 	splx(s);
-	if (dev_allocq_is_runnable(bus->sim->devq))
+	if (dev_allocq_is_runnable(sim->devq))
 		xpt_run_dev_allocq(bus);
 }
 
@@ -4369,7 +4434,7 @@ xpt_bus_register(struct cam_sim *sim, u_int32_t bus)
 	struct ccb_pathinq cpi;
 	int s;
 
-	GIANT_REQUIRED;
+	mtx_assert(sim->mtx, MA_OWNED);
 
 	sim->bus_id = bus;
 	new_bus = (struct cam_eb *)malloc(sizeof(*new_bus),
@@ -4393,15 +4458,17 @@ xpt_bus_register(struct cam_sim *sim, u_int32_t bus)
 	new_bus->refcount = 1;	/* Held until a bus_deregister event */
 	new_bus->generation = 0;
 	s = splcam();
-	old_bus = TAILQ_FIRST(&xpt_busses);
+	mtx_lock(&xsoftc.xpt_topo_lock);
+	old_bus = TAILQ_FIRST(&xsoftc.xpt_busses);
 	while (old_bus != NULL
 	    && old_bus->path_id < new_bus->path_id)
 		old_bus = TAILQ_NEXT(old_bus, links);
 	if (old_bus != NULL)
 		TAILQ_INSERT_BEFORE(old_bus, new_bus, links);
 	else
-		TAILQ_INSERT_TAIL(&xpt_busses, new_bus, links);
-	bus_generation++;
+		TAILQ_INSERT_TAIL(&xsoftc.xpt_busses, new_bus, links);
+	xsoftc.bus_generation++;
+	mtx_unlock(&xsoftc.xpt_topo_lock);
 	splx(s);
 
 	/* Notify interested parties */
@@ -4431,7 +4498,6 @@ xpt_bus_deregister(path_id_t pathid)
 	union ccb *work_ccb;
 	cam_status status;
 
-	GIANT_REQUIRED;
 
 	status = xpt_compile_path(&bus_path, NULL, pathid,
 				  CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD);
@@ -4496,7 +4562,8 @@ xptnextfreepathid(void)
 	const char *strval;
 
 	pathid = 0;
-	bus = TAILQ_FIRST(&xpt_busses);
+	mtx_lock(&xsoftc.xpt_topo_lock);
+	bus = TAILQ_FIRST(&xsoftc.xpt_busses);
 retry:
 	/* Find an unoccupied pathid */
 	while (bus != NULL && bus->path_id <= pathid) {
@@ -4504,6 +4571,7 @@ retry:
 			pathid++;
 		bus = TAILQ_NEXT(bus, links);
 	}
+	mtx_unlock(&xsoftc.xpt_topo_lock);
 
 	/*
 	 * Ensure that this pathid is not reserved for
@@ -4512,6 +4580,7 @@ retry:
 	if (resource_string_value("scbus", pathid, "at", &strval) == 0) {
 		++pathid;
 		/* Start the search over */
+		mtx_lock(&xsoftc.xpt_topo_lock);
 		goto retry;
 	}
 	return (pathid);
@@ -4568,7 +4637,7 @@ xpt_async(u_int32_t async_code, struct cam_path *path, void *async_arg)
 	struct cam_ed *device, *next_device;
 	int s;
 
-	GIANT_REQUIRED;
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
 	CAM_DEBUG(path, CAM_DEBUG_TRACE, ("xpt_async\n"));
 
@@ -4735,7 +4804,7 @@ xpt_freeze_devq(struct cam_path *path, u_int count)
 	int s;
 	struct ccb_hdr *ccbh;
 
-	GIANT_REQUIRED;
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
 	s = splcam();
 	path->device->qfrozen_cnt += count;
@@ -4763,7 +4832,7 @@ xpt_freeze_devq(struct cam_path *path, u_int count)
 u_int32_t
 xpt_freeze_simq(struct cam_sim *sim, u_int count)
 {
-	GIANT_REQUIRED;
+	mtx_assert(sim->mtx, MA_OWNED);
 
 	sim->devq->send_queue.qfrozen_cnt += count;
 	if (sim->devq->active_dev != NULL) {
@@ -4790,7 +4859,7 @@ xpt_release_devq_timeout(void *arg)
 void
 xpt_release_devq(struct cam_path *path, u_int count, int run_queue)
 {
-	GIANT_REQUIRED;
+	mtx_assert(path->bus->sim->mtx, MA_OWNED);
 
 	xpt_release_devq_device(path->device, count, run_queue);
 }
@@ -4821,8 +4890,7 @@ xpt_release_devq_device(struct cam_ed *dev, u_int count, int run_queue)
 			 * to release this queue.
 			 */
 			if ((dev->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0) {
-				untimeout(xpt_release_devq_timeout, dev,
-					  dev->c_handle);
+				callout_stop(&dev->callout);
 				dev->flags &= ~CAM_DEV_REL_TIMEOUT_PENDING;
 			}
 
@@ -4850,7 +4918,7 @@ xpt_release_simq(struct cam_sim *sim, int run_queue)
 	int	s;
 	struct	camq *sendq;
 
-	GIANT_REQUIRED;
+	mtx_assert(sim->mtx, MA_OWNED);
 
 	sendq = &(sim->devq->send_queue);
 	s = splcam();
@@ -4866,8 +4934,7 @@ xpt_release_simq(struct cam_sim *sim, int run_queue)
 			 * already at 0.
 			 */
 			if ((sim->flags & CAM_SIM_REL_TIMEOUT_PENDING) != 0){
-				untimeout(xpt_release_simq_timeout, sim,
-					  sim->c_handle);
+				callout_stop(&sim->callout);
 				sim->flags &= ~CAM_SIM_REL_TIMEOUT_PENDING;
 			}
 			bus = xpt_find_bus(sim->path_id);
@@ -4886,6 +4953,9 @@ xpt_release_simq(struct cam_sim *sim, int run_queue)
 		splx(s);
 }
 
+/*
+ * XXX Appears to be unused.
+ */
 static void
 xpt_release_simq_timeout(void *arg)
 {
@@ -4915,7 +4985,9 @@ xpt_done(union ccb *done_ccb)
 					  sim_links.tqe);
 			done_ccb->ccb_h.pinfo.index = CAM_DONEQ_INDEX;
 			mtx_unlock(&cam_bioq_lock);
-			swi_sched(cambio_ih, 0);
+			if ((done_ccb->ccb_h.path->periph->flags &
+			    CAM_PERIPH_POLLED) == 0)
+				swi_sched(cambio_ih, 0);
 			break;
 		default:
 			panic("unknown periph type %d",
@@ -4926,24 +4998,26 @@ xpt_done(union ccb *done_ccb)
 }
 
 union ccb *
-xpt_alloc_ccb()
+xpt_alloc_ccb(struct cam_sim *sim)
 {
 	union ccb *new_ccb;
 
-	GIANT_REQUIRED;
-
 	new_ccb = malloc(sizeof(*new_ccb), M_CAMXPT, M_WAITOK);
+	if ((sim != NULL) && ((sim->flags & CAM_SIM_MPSAFE) == 0)) {
+		callout_handle_init(&new_ccb->ccb_h.timeout_ch);
+	}
 	return (new_ccb);
 }
 
 union ccb *
-xpt_alloc_ccb_nowait()
+xpt_alloc_ccb_nowait(struct cam_sim *sim)
 {
 	union ccb *new_ccb;
 
-	GIANT_REQUIRED;
-
 	new_ccb = malloc(sizeof(*new_ccb), M_CAMXPT, M_NOWAIT);
+	if ((sim != NULL) && ((sim->flags & CAM_SIM_MPSAFE) == 0)) {
+		callout_handle_init(&new_ccb->ccb_h.timeout_ch);
+	}
 	return (new_ccb);
 }
 
@@ -4968,22 +5042,23 @@ static union ccb *
 xpt_get_ccb(struct cam_ed *device)
 {
 	union ccb *new_ccb;
+	struct cam_sim *sim;
 	int s;
 
 	s = splsoftcam();
-	if ((new_ccb = (union ccb *)SLIST_FIRST(&ccb_freeq)) == NULL) {
-		new_ccb = xpt_alloc_ccb_nowait();
+	sim = device->sim;
+	if ((new_ccb = (union ccb *)SLIST_FIRST(&sim->ccb_freeq)) == NULL) {
+		new_ccb = xpt_alloc_ccb_nowait(sim);
                 if (new_ccb == NULL) {
 			splx(s);
 			return (NULL);
 		}
-		callout_handle_init(&new_ccb->ccb_h.timeout_ch);
-		SLIST_INSERT_HEAD(&ccb_freeq, &new_ccb->ccb_h,
+		SLIST_INSERT_HEAD(&sim->ccb_freeq, &new_ccb->ccb_h,
 				  xpt_links.sle);
-		xpt_ccb_count++;
+		sim->ccb_count++;
 	}
 	cam_ccbq_take_opening(&device->ccbq);
-	SLIST_REMOVE_HEAD(&ccb_freeq, xpt_links.sle);
+	SLIST_REMOVE_HEAD(&sim->ccb_freeq, xpt_links.sle);
 	splx(s);
 	return (new_ccb);
 }
@@ -4996,8 +5071,10 @@ xpt_release_bus(struct cam_eb *bus)
 	s = splcam();
 	if ((--bus->refcount == 0)
 	 && (TAILQ_FIRST(&bus->et_entries) == NULL)) {
-		TAILQ_REMOVE(&xpt_busses, bus, links);
-		bus_generation++;
+		mtx_lock(&xsoftc.xpt_topo_lock);
+		TAILQ_REMOVE(&xsoftc.xpt_busses, bus, links);
+		xsoftc.bus_generation++;
+		mtx_unlock(&xsoftc.xpt_topo_lock);
 		splx(s);
 		free(bus, M_CAMXPT);
 	} else
@@ -5088,6 +5165,7 @@ xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
 		device->send_ccb_entry.device = device;
 		device->target = target;
 		device->lun_id = lun_id;
+		device->sim = bus->sim;
 		/* Initialize our queues */
 		if (camq_init(&device->drvq, 0) != 0) {
 			free(device, M_CAMXPT);
@@ -5118,7 +5196,10 @@ xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
 		device->tag_delay_count = 0;
 		device->tag_saved_openings = 0;
 		device->refcount = 1;
-		callout_handle_init(&device->c_handle);
+		if (bus->sim->flags & CAM_SIM_MPSAFE)
+			callout_init_mtx(&device->callout, bus->sim->mtx, 0);
+		else
+			callout_init_mtx(&device->callout, &Giant, 0);
 
 		/*
 		 * Hold a reference to our parent target so it
@@ -5130,7 +5211,7 @@ xpt_alloc_device(struct cam_eb *bus, struct cam_et *target, lun_id_t lun_id)
 		 * XXX should be limited by number of CCBs this bus can
 		 * do.
 		 */
-		xpt_max_ccbs += device->ccbq.devq_openings;
+		bus->sim->max_ccbs += device->ccbq.devq_openings;
 		/* Insertion sort into our target's device list */
 		cur_device = TAILQ_FIRST(&target->ed_entries);
 		while (cur_device != NULL && cur_device->lun_id < lun_id)
@@ -5170,12 +5251,11 @@ xpt_release_device(struct cam_eb *bus, struct cam_et *target,
 			panic("Removing device while still queued for ccbs");
 
 		if ((device->flags & CAM_DEV_REL_TIMEOUT_PENDING) != 0)
-				untimeout(xpt_release_devq_timeout, device,
-					  device->c_handle);
+				callout_stop(&device->callout);
 
 		TAILQ_REMOVE(&target->ed_entries, device,links);
 		target->generation++;
-		xpt_max_ccbs -= device->ccbq.devq_openings;
+		bus->sim->max_ccbs -= device->ccbq.devq_openings;
 		if (!SIM_DEAD(bus->sim)) {
 			/* Release our slot in the devq */
 			devq = bus->sim->devq;
@@ -5210,7 +5290,7 @@ xpt_dev_ccbq_resize(struct cam_path *path, int newopenings)
 	 || (dev->inq_flags & SID_CmdQue) != 0)
 		dev->tag_saved_openings = newopenings;
 	/* Adjust the global limit */
-	xpt_max_ccbs += diff;
+	dev->sim->max_ccbs += diff;
 	splx(s);
 	return (result);
 }
@@ -5220,7 +5300,8 @@ xpt_find_bus(path_id_t path_id)
 {
 	struct cam_eb *bus;
 
-	for (bus = TAILQ_FIRST(&xpt_busses);
+	mtx_lock(&xsoftc.xpt_topo_lock);
+	for (bus = TAILQ_FIRST(&xsoftc.xpt_busses);
 	     bus != NULL;
 	     bus = TAILQ_NEXT(bus, links)) {
 		if (bus->path_id == path_id) {
@@ -5228,6 +5309,7 @@ xpt_find_bus(path_id_t path_id)
 			break;
 		}
 	}
+	mtx_unlock(&xsoftc.xpt_topo_lock);
 	return (bus);
 }
 
@@ -5290,7 +5372,7 @@ xpt_scan_bus(struct cam_periph *periph, union ccb *request_ccb)
 		u_int	initiator_id;
 
 		/* Find out the characteristics of the bus */
-		work_ccb = xpt_alloc_ccb();
+		work_ccb = xpt_alloc_ccb_nowait(periph->sim);
 		xpt_setup_ccb(&work_ccb->ccb_h, request_ccb->ccb_h.path,
 			      request_ccb->ccb_h.pinfo.priority);
 		work_ccb->ccb_h.func_code = XPT_PATH_INQ;
@@ -5315,7 +5397,7 @@ xpt_scan_bus(struct cam_periph *periph, union ccb *request_ccb)
 
 		/* Save some state for use while we probe for devices */
 		scan_info = (xpt_scan_bus_info *)
-		    malloc(sizeof(xpt_scan_bus_info), M_TEMP, M_WAITOK);
+		    malloc(sizeof(xpt_scan_bus_info), M_TEMP, M_NOWAIT);
 		scan_info->request_ccb = request_ccb;
 		scan_info->cpi = &work_ccb->cpi;
 
@@ -5355,7 +5437,7 @@ xpt_scan_bus(struct cam_periph *periph, union ccb *request_ccb)
 				xpt_done(request_ccb);
 				break;
 			}
-			work_ccb = xpt_alloc_ccb();
+			work_ccb = xpt_alloc_ccb_nowait(periph->sim);
 			xpt_setup_ccb(&work_ccb->ccb_h, path,
 				      request_ccb->ccb_h.pinfo.priority);
 			work_ccb->ccb_h.func_code = XPT_SCAN_LUN;
@@ -6870,6 +6952,9 @@ static int busses_to_reset;
 static int
 xptconfigbuscountfunc(struct cam_eb *bus, void *arg)
 {
+
+	mtx_assert(bus->sim->mtx, MA_OWNED);
+
 	if (bus->path_id != CAM_XPT_PATH_ID) {
 		struct cam_path path;
 		struct ccb_pathinq cpi;
@@ -6898,11 +6983,13 @@ xptconfigfunc(struct cam_eb *bus, void *arg)
 	struct	cam_path *path;
 	union	ccb *work_ccb;
 
+	mtx_assert(bus->sim->mtx, MA_OWNED);
+
 	if (bus->path_id != CAM_XPT_PATH_ID) {
 		cam_status status;
 		int can_negotiate;
 
-		work_ccb = xpt_alloc_ccb();
+		work_ccb = xpt_alloc_ccb_nowait(bus->sim);
 		if ((status = xpt_create_path(&path, xpt_periph, bus->path_id,
 					      CAM_TARGET_WILDCARD,
 					      CAM_LUN_WILDCARD)) !=CAM_REQ_CMP){
@@ -6962,6 +7049,11 @@ xpt_config(void *arg)
 #endif /* CAM_DEBUG_FLAGS */
 #ifdef CAM_DEBUG_BUS
 	if (cam_dflags != CAM_DEBUG_NONE) {
+		/*
+		 * Locking is specifically omitted here.  No SIMs have
+		 * registered yet, so xpt_create_path will only be searching
+		 * empty lists of targets and devices.
+		 */
 		if (xpt_create_path(&cam_dpath, xpt_periph,
 				    CAM_DEBUG_BUS, CAM_DEBUG_TARGET,
 				    CAM_DEBUG_LUN) != CAM_REQ_CMP) {
@@ -7017,11 +7109,40 @@ xptpassannouncefunc(struct cam_ed *device, void *arg)
 }
 
 static void
-xpt_finishconfig(struct cam_periph *periph, union ccb *done_ccb)
+xpt_finishconfig_task(void *context, int pending)
 {
 	struct	periph_driver **p_drv;
 	int	i;
 
+	if (busses_to_config == 0) {
+		/* Register all the peripheral drivers */
+		/* XXX This will have to change when we have loadable modules */
+		p_drv = periph_drivers;
+		for (i = 0; p_drv[i] != NULL; i++) {
+			(*p_drv[i]->init)();
+		}
+
+		/*
+		 * Check for devices with no "standard" peripheral driver
+		 * attached.  For any devices like that, announce the
+		 * passthrough driver so the user will see something.
+		 */
+		xpt_for_all_devices(xptpassannouncefunc, NULL);
+
+		/* Release our hook so that the boot can continue. */
+		config_intrhook_disestablish(xsoftc.xpt_config_hook);
+		free(xsoftc.xpt_config_hook, M_TEMP);
+		xsoftc.xpt_config_hook = NULL;
+	}
+
+	free(context, M_CAMXPT);
+}
+
+static void
+xpt_finishconfig(struct cam_periph *periph, union ccb *done_ccb)
+{
+	struct	xpt_task *task;
+
 	if (done_ccb != NULL) {
 		CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_TRACE,
 			  ("xpt_finishconfig\n"));
@@ -7043,26 +7164,12 @@ xpt_finishconfig(struct cam_periph *periph, union ccb *done_ccb)
 		}
 	}
 
-	if (busses_to_config == 0) {
-		/* Register all the peripheral drivers */
-		/* XXX This will have to change when we have loadable modules */
-		p_drv = periph_drivers;
-		for (i = 0; p_drv[i] != NULL; i++) {
-			(*p_drv[i]->init)();
-		}
-
-		/*
-		 * Check for devices with no "standard" peripheral driver
-		 * attached.  For any devices like that, announce the
-		 * passthrough driver so the user will see something.
-		 */
-		xpt_for_all_devices(xptpassannouncefunc, NULL);
-
-		/* Release our hook so that the boot can continue. */
-		config_intrhook_disestablish(xpt_config_hook);
-		free(xpt_config_hook, M_TEMP);
-		xpt_config_hook = NULL;
+	task = malloc(sizeof(struct xpt_task), M_CAMXPT, M_NOWAIT);
+	if (task != NULL) {
+		TASK_INIT(&task->task, 0, xpt_finishconfig_task, task);
+		taskqueue_enqueue(taskqueue_thread, &task->task);
 	}
+
 	if (done_ccb != NULL)
 		xpt_free_ccb(done_ccb);
 }
@@ -7117,6 +7224,18 @@ xptpoll(struct cam_sim *sim)
 {
 }
 
+void
+xpt_lock_buses(void)
+{
+	mtx_lock(&xsoftc.xpt_topo_lock);
+}
+
+void
+xpt_unlock_buses(void)
+{
+	mtx_unlock(&xsoftc.xpt_topo_lock);
+}
+
 static void
 camisr(void *V_queue)
 {
@@ -7124,6 +7243,7 @@ camisr(void *V_queue)
 	cam_isrq_t queue;
 	int	s;
 	struct	ccb_hdr *ccb_h;
+	struct	cam_sim	*sim;
 
 	/*
 	 * Transfer the ccb_bioq list to a temporary list so we can operate
@@ -7152,14 +7272,15 @@ camisr(void *V_queue)
 			struct highpowerlist	*hphead;
 			union ccb		*send_ccb;
 
-			hphead = &highpowerq;
+			mtx_lock(&xsoftc.xpt_lock);
+			hphead = &xsoftc.highpowerq;
 
 			send_ccb = (union ccb *)STAILQ_FIRST(hphead);
 
 			/*
 			 * Increment the count since this command is done.
 			 */
-			num_highpower++;
+			xsoftc.num_highpower++;
 
 			/* 
 			 * Any high powered commands queued up?
@@ -7167,11 +7288,17 @@ camisr(void *V_queue)
 			if (send_ccb != NULL) {
 
 				STAILQ_REMOVE_HEAD(hphead, xpt_links.stqe);
+				mtx_unlock(&xsoftc.xpt_lock);
 
 				xpt_release_devq(send_ccb->ccb_h.path,
 						 /*count*/1, /*runqueue*/TRUE);
-			}
+			} else
+				mtx_unlock(&xsoftc.xpt_lock);
 		}
+
+		sim = ccb_h->path->bus->sim;
+		mtx_lock(sim->mtx);
+
 		if ((ccb_h->func_code & XPT_FC_USER_CCB) == 0) {
 			struct cam_ed *dev;
 
@@ -7227,6 +7354,7 @@ camisr(void *V_queue)
 		(*ccb_h->cbfcnp)(ccb_h->path->periph, (union ccb *)ccb_h);
 
 		/* Raise IPL for while test */
+		mtx_unlock(sim->mtx);
 		s = splcam();
 	}
 	splx(s);
diff --git a/sys/cam/cam_xpt.h b/sys/cam/cam_xpt.h
index f8557f9..deed466 100644
--- a/sys/cam/cam_xpt.h
+++ b/sys/cam/cam_xpt.h
@@ -58,6 +58,10 @@ cam_status		xpt_create_path(struct cam_path **new_path_ptr,
 					struct cam_periph *perph,
 					path_id_t path_id,
 					target_id_t target_id, lun_id_t lun_id);
+cam_status		xpt_create_path_unlocked(struct cam_path **new_path_ptr,
+					struct cam_periph *perph,
+					path_id_t path_id,
+					target_id_t target_id, lun_id_t lun_id);
 void			xpt_free_path(struct cam_path *path);
 int			xpt_path_comp(struct cam_path *path1,
 				      struct cam_path *path2);
@@ -73,6 +77,8 @@ struct cam_periph	*xpt_path_periph(struct cam_path *path);
 void			xpt_async(u_int32_t async_code, struct cam_path *path,
 				  void *async_arg);
 void			xpt_rescan(union ccb *ccb);
+void			xpt_lock_buses(void);
+void			xpt_unlock_buses(void);
 #endif /* _KERNEL */
 
 #endif /* _CAM_CAM_XPT_H */
diff --git a/sys/cam/cam_xpt_periph.h b/sys/cam/cam_xpt_periph.h
index c6b8cc2..9c4a3bb 100644
--- a/sys/cam/cam_xpt_periph.h
+++ b/sys/cam/cam_xpt_periph.h
@@ -38,8 +38,8 @@
 /* Functions accessed by the peripheral drivers */
 #ifdef _KERNEL
 void		xpt_polled_action(union ccb *ccb);
-union ccb	*xpt_alloc_ccb(void);
-union ccb	*xpt_alloc_ccb_nowait(void);
+union ccb	*xpt_alloc_ccb(struct cam_sim *sim);
+union ccb	*xpt_alloc_ccb_nowait(struct cam_sim *sim);
 void		xpt_free_ccb(union ccb *free_ccb);
 void		xpt_release_ccb(union ccb *released_ccb);
 void		xpt_schedule(struct cam_periph *perph, u_int32_t new_priority);
diff --git a/sys/cam/scsi/scsi_cd.c b/sys/cam/scsi/scsi_cd.c
index 07782ba..fa85ada 100644
--- a/sys/cam/scsi/scsi_cd.c
+++ b/sys/cam/scsi/scsi_cd.c
@@ -70,6 +70,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/cam_periph.h>
 #include <cam/cam_xpt_periph.h>
 #include <cam/cam_queue.h>
+#include <cam/cam_sim.h>
 
 #include <cam/scsi/scsi_message.h>
 #include <cam/scsi/scsi_da.h>
@@ -103,7 +104,8 @@ typedef enum {
 	CD_FLAG_RETRY_UA	= 0x0200,
 	CD_FLAG_VALID_MEDIA	= 0x0400,
 	CD_FLAG_VALID_TOC	= 0x0800,
-	CD_FLAG_SCTX_INIT	= 0x1000
+	CD_FLAG_SCTX_INIT	= 0x1000,
+	CD_FLAG_OPEN		= 0x2000
 } cd_flags;
 
 typedef enum {
@@ -290,9 +292,6 @@ static struct periph_driver cddriver =
 
 PERIPHDRIVER_DECLARE(cd, cddriver);
 
-
-static int num_changers;
-
 #ifndef CHANGER_MIN_BUSY_SECONDS
 #define CHANGER_MIN_BUSY_SECONDS	5
 #endif
@@ -319,15 +318,16 @@ struct cdchanger {
 	struct camq			 devq;
 	struct timeval			 start_time;
 	struct cd_softc			 *cur_device;
-	struct callout_handle		 short_handle;
-	struct callout_handle		 long_handle;
+	struct callout			 short_handle;
+	struct callout			 long_handle;
 	volatile cd_changer_flags	 flags;
 	STAILQ_ENTRY(cdchanger)		 changer_links;
 	STAILQ_HEAD(chdevlist, cd_softc) chluns;
 };
 
+static struct mtx changerq_mtx;
 static STAILQ_HEAD(changerlist, cdchanger) changerq;
-
+static int num_changers;
 
 static void
 cdinit(void)
@@ -335,6 +335,9 @@ cdinit(void)
 	cam_status status;
 	struct cam_path *path;
 
+	mtx_init(&changerq_mtx, "cdchangerq", "SCSI CD Changer List", MTX_DEF);
+	STAILQ_INIT(&changerq);
+
 	/*
 	 * Install a global async callback.  This callback will
 	 * receive async callbacks like "new device found".
@@ -364,7 +367,6 @@ cdinit(void)
 static void
 cdoninvalidate(struct cam_periph *periph)
 {
-	int s;
 	struct cd_softc *softc;
 	struct ccb_setasync csa;
 
@@ -384,19 +386,11 @@ cdoninvalidate(struct cam_periph *periph)
 	softc->flags |= CD_FLAG_INVALID;
 
 	/*
-	 * Although the oninvalidate() routines are always called at
-	 * splsoftcam, we need to be at splbio() here to keep the buffer
-	 * queue from being modified while we traverse it.
-	 */
-	s = splbio();
-
-	/*
 	 * Return all queued I/O with ENXIO.
 	 * XXX Handle any transactions queued to the card
 	 *     with XPT_ABORT_CCB.
 	 */
 	bioq_flush(&softc->bio_queue, NULL, ENXIO);
-	splx(s);
 
 	/*
 	 * If this device is part of a changer, and it was scheduled
@@ -415,7 +409,6 @@ static void
 cdcleanup(struct cam_periph *periph)
 {
 	struct cd_softc *softc;
-	int s;
 
 	softc = (struct cd_softc *)periph->softc;
 
@@ -426,7 +419,6 @@ cdcleanup(struct cam_periph *periph)
 		xpt_print(periph->path, "can't remove sysctl context\n");
 	}
 
-	s = splsoftcam();
 	/*
 	 * In the queued, non-active case, the device in question
 	 * has already been removed from the changer run queue.  Since this
@@ -456,8 +448,7 @@ cdcleanup(struct cam_periph *periph)
 		 * be any bogus pointer references there.
 		 */
 		if (softc->changer->flags & CHANGER_SHORT_TMOUT_SCHED) {
-			untimeout(cdshorttimeout, softc->changer,
-				  softc->changer->short_handle);
+			callout_stop(&softc->changer->short_handle);
 			softc->changer->flags &= ~CHANGER_SHORT_TMOUT_SCHED;
 		}
 		softc->changer->devq.qfrozen_cnt--;
@@ -478,26 +469,25 @@ cdcleanup(struct cam_periph *periph)
 		 * it won't hurt to check and see if there are any left.
 		 */
 		if (softc->changer->flags & CHANGER_TIMEOUT_SCHED) {
-			untimeout(cdrunchangerqueue, softc->changer,
-				  softc->changer->long_handle);
+			callout_stop(&softc->changer->long_handle);
 			softc->changer->flags &= ~CHANGER_TIMEOUT_SCHED;
 		}
 
 		if (softc->changer->flags & CHANGER_SHORT_TMOUT_SCHED) {
-			untimeout(cdshorttimeout, softc->changer,
-				  softc->changer->short_handle);
+			callout_stop(&softc->changer->short_handle);
 			softc->changer->flags &= ~CHANGER_SHORT_TMOUT_SCHED;
 		}
 
+		mtx_lock(&changerq_mtx);
 		STAILQ_REMOVE(&changerq, softc->changer, cdchanger,
 			      changer_links);
+		num_changers--;
+		mtx_unlock(&changerq_mtx);
 		xpt_print(periph->path, "removing changer entry\n");
 		free(softc->changer, M_DEVBUF);
-		num_changers--;
 	}
 	disk_destroy(softc->disk);
 	free(softc, M_DEVBUF);
-	splx(s);
 }
 
 static void
@@ -544,10 +534,8 @@ cdasync(void *callback_arg, u_int32_t code,
 	{
 		struct cd_softc *softc;
 		struct ccb_hdr *ccbh;
-		int s;
 
 		softc = (struct cd_softc *)periph->softc;
-		s = splsoftcam();
 		/*
 		 * Don't fail on the expected unit attention
 		 * that will occur.
@@ -555,7 +543,6 @@ cdasync(void *callback_arg, u_int32_t code,
 		softc->flags |= CD_FLAG_RETRY_UA;
 		LIST_FOREACH(ccbh, &softc->pending_ccbs, periph_links.le)
 			ccbh->ccb_state |= CD_CCB_RETRY_UA;
-		splx(s);
 		/* FALLTHROUGH */
 	}
 	default:
@@ -572,8 +559,10 @@ cdsysctlinit(void *context, int pending)
 	char tmpstr[80], tmpstr2[80];
 
 	periph = (struct cam_periph *)context;
-	softc = (struct cd_softc *)periph->softc;
+	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
+		return;
 
+	softc = (struct cd_softc *)periph->softc;
 	snprintf(tmpstr, sizeof(tmpstr), "CAM CD unit %d", periph->unit_number);
 	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
 
@@ -588,6 +577,7 @@ cdsysctlinit(void *context, int pending)
 	if (softc->sysctl_tree == NULL) {
 		printf("cdsysctlinit: unable to allocate sysctl tree\n");
 		mtx_unlock(&Giant);
+		cam_periph_release(periph);
 		return;
 	}
 
@@ -601,6 +591,7 @@ cdsysctlinit(void *context, int pending)
 		"Minimum CDB size");
 
 	mtx_unlock(&Giant);
+	cam_periph_release(periph);
 }
 
 /*
@@ -734,6 +725,7 @@ cdregister(struct cam_periph *periph, void *arg)
 	 * WORM peripheral driver.  WORM drives will also have the WORM
 	 * driver attached to them.
 	 */
+	cam_periph_unlock(periph);
 	softc->disk = disk_alloc();
 	softc->disk->d_devstat = devstat_new_entry("cd", 
 			  periph->unit_number, 0,
@@ -747,8 +739,9 @@ cdregister(struct cam_periph *periph, void *arg)
 	softc->disk->d_name = "cd";
 	softc->disk->d_unit = periph->unit_number;
 	softc->disk->d_drv1 = periph;
-	softc->disk->d_flags = DISKFLAG_NEEDSGIANT;
+	softc->disk->d_flags = 0;
 	disk_create(softc->disk, DISK_VERSION);
+	cam_periph_lock(periph);
 
 	/*
 	 * Add an async callback so that we get
@@ -783,13 +776,11 @@ cdregister(struct cam_periph *periph, void *arg)
 		/* Set the changer flag in the current device's softc */
 		softc->flags |= CD_FLAG_CHANGER;
 
-		if (num_changers == 0)
-			STAILQ_INIT(&changerq);
-
 		/*
 		 * Now, look around for an existing changer device with the
 		 * same path and target ID as the current device.
 		 */
+		mtx_lock(&changerq_mtx);
 		for (found = 0,
 		     nchanger = (struct cdchanger *)STAILQ_FIRST(&changerq);
 		     nchanger != NULL;
@@ -800,6 +791,7 @@ cdregister(struct cam_periph *periph, void *arg)
 				break;
 			}
 		}
+		mtx_unlock(&changerq_mtx);
 
 		/*
 		 * If we found a matching entry, just add this device to
@@ -905,8 +897,6 @@ cdregister(struct cam_periph *periph, void *arg)
 				goto cdregisterexit;
 			}
 
-			num_changers++;
-
 			nchanger->path_id = cgd->ccb_h.path_id;
 			nchanger->target_id = cgd->ccb_h.target_id;
 
@@ -915,8 +905,16 @@ cdregister(struct cam_periph *periph, void *arg)
 
 			STAILQ_INIT(&nchanger->chluns);
 
+			callout_init_mtx(&nchanger->long_handle,
+			    periph->sim->mtx, 0);
+			callout_init_mtx(&nchanger->short_handle,
+			    periph->sim->mtx, 0);
+
+			mtx_lock(&changerq_mtx);
+			num_changers++;
 			STAILQ_INSERT_TAIL(&changerq, nchanger,
 					   changer_links);
+			mtx_unlock(&changerq_mtx);
 			
 			/*
 			 * Create a path with lun id 0, and see if we can
@@ -978,9 +976,11 @@ cdregister(struct cam_periph *periph, void *arg)
 
 cdregisterexit:
 
-	/* Lock this peripheral until we are setup */
-	/* Can't block */
-	cam_periph_lock(periph, PRIBIO); 
+	/*
+	 * Refcount and block open attempts until we are setup
+	 * Can't block
+	 */
+	(void)cam_periph_hold(periph, PRIBIO);
 
 	if ((softc->flags & CD_FLAG_CHANGER) == 0)
 		xpt_schedule(periph, /*priority*/5);
@@ -996,7 +996,6 @@ cdopen(struct disk *dp)
 	struct cam_periph *periph;
 	struct cd_softc *softc;
 	int error;
-	int s;
 
 	periph = (struct cam_periph *)dp->d_drv1;
 	if (periph == NULL)
@@ -1004,24 +1003,28 @@ cdopen(struct disk *dp)
 
 	softc = (struct cd_softc *)periph->softc;
 
-	/*
-	 * Grab splsoftcam and hold it until we lock the peripheral.
-	 */
-	s = splsoftcam();
+	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
+		return(ENXIO);
+
+	cam_periph_lock(periph);
+
 	if (softc->flags & CD_FLAG_INVALID) {
-		splx(s);
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return(ENXIO);
 	}
 
-	if ((error = cam_periph_lock(periph, PRIBIO | PCATCH)) != 0) {
-		splx(s);
+	if ((error = cam_periph_hold(periph, PRIBIO | PCATCH)) != 0) {
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return (error);
 	}
 
-	splx(s);
-
-	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
-		return(ENXIO);
+	/* Closes aren't symmetrical with opens, so fix up the refcounting. */
+	if (softc->flags & CD_FLAG_OPEN)
+		cam_periph_release(periph);
+	else
+		softc->flags |= CD_FLAG_OPEN;
 
 	/*
 	 * Check for media, and set the appropriate flags.  We don't bail
@@ -1030,11 +1033,11 @@ cdopen(struct disk *dp)
 	 */
 	cdcheckmedia(periph);
 
-	cam_periph_unlock(periph);
-
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("leaving cdopen\n"));
+	cam_periph_unhold(periph);
+	cam_periph_unlock(periph);
 
-	return (error);
+	return (0);
 }
 
 static int
@@ -1042,7 +1045,6 @@ cdclose(struct disk *dp)
 {
 	struct 	cam_periph *periph;
 	struct	cd_softc *softc;
-	int	error;
 
 	periph = (struct cam_periph *)dp->d_drv1;
 	if (periph == NULL)
@@ -1050,8 +1052,8 @@ cdclose(struct disk *dp)
 
 	softc = (struct cd_softc *)periph->softc;
 
-	if ((error = cam_periph_lock(periph, PRIBIO)) != 0)
-		return (error);
+	cam_periph_lock(periph);
+	cam_periph_hold(periph, PRIBIO);
 
 	if ((softc->flags & CD_FLAG_DISC_REMOVABLE) != 0)
 		cdprevent(periph, PR_ALLOW);
@@ -1065,8 +1067,9 @@ cdclose(struct disk *dp)
 	/*
 	 * We'll check the media and toc again at the next open().
 	 */
-	softc->flags &= ~(CD_FLAG_VALID_MEDIA|CD_FLAG_VALID_TOC);
+	softc->flags &= ~(CD_FLAG_VALID_MEDIA|CD_FLAG_VALID_TOC|CD_FLAG_OPEN);
 
+	cam_periph_unhold(periph);
 	cam_periph_unlock(periph);
 	cam_periph_release(periph);
 
@@ -1077,9 +1080,6 @@ static void
 cdshorttimeout(void *arg)
 {
 	struct cdchanger *changer;
-	int s;
-
-	s = splsoftcam();
 
 	changer = (struct cdchanger *)arg;
 
@@ -1095,8 +1095,6 @@ cdshorttimeout(void *arg)
 		changer->flags |= CHANGER_MANUAL_CALL;
 		cdrunchangerqueue(changer);
 	}
-
-	splx(s);
 }
 
 /*
@@ -1106,9 +1104,6 @@ static void
 cdschedule(struct cam_periph *periph, int priority)
 {
 	struct cd_softc *softc;
-	int s;
-
-	s = splsoftcam();
 
 	softc = (struct cd_softc *)periph->softc;
 
@@ -1147,9 +1142,6 @@ cdschedule(struct cam_periph *periph, int priority)
 	} else if ((softc->flags & CD_FLAG_ACTIVE)
 		&& ((softc->flags & CD_FLAG_SCHED_ON_COMP) == 0))
 		xpt_schedule(periph, priority);
-
-	splx(s);
-
 }
 
 static void
@@ -1158,9 +1150,6 @@ cdrunchangerqueue(void *arg)
 	struct cd_softc *softc;
 	struct cdchanger *changer;
 	int called_from_timeout;
-	int s;
-
-	s = splsoftcam();
 
 	changer = (struct cdchanger *)arg;
 
@@ -1180,7 +1169,6 @@ cdrunchangerqueue(void *arg)
 
 	/* nothing to do if the queue is empty */
 	if (changer->devq.entries <= 0) {
-		splx(s);
 		return;
 	}
 
@@ -1190,29 +1178,28 @@ cdrunchangerqueue(void *arg)
 	 */
 	if (changer->devq.qfrozen_cnt > 0) {
 
+		/*
+		 * We always need to reset the frozen count and clear the
+		 * active flag.
+		 */
+		changer->devq.qfrozen_cnt--;
+		changer->cur_device->flags &= ~CD_FLAG_ACTIVE;
+		changer->cur_device->flags &= ~CD_FLAG_SCHED_ON_COMP;
+
 		if (changer->cur_device->outstanding_cmds > 0) {
 			changer->cur_device->flags |= CD_FLAG_SCHED_ON_COMP;
 			changer->cur_device->bufs_left = 
 				changer->cur_device->outstanding_cmds;
 			if (called_from_timeout) {
-				changer->long_handle =
-					timeout(cdrunchangerqueue, changer,
-				        changer_max_busy_seconds * hz);
+				callout_reset(&changer->long_handle,
+			            changer_max_busy_seconds * hz,
+				    cdrunchangerqueue, changer);
 				changer->flags |= CHANGER_TIMEOUT_SCHED;
 			}
-			splx(s);
 			return;
 		}
 
 		/*
-		 * We always need to reset the frozen count and clear the
-		 * active flag.
-		 */
-		changer->devq.qfrozen_cnt--;
-		changer->cur_device->flags &= ~CD_FLAG_ACTIVE;
-		changer->cur_device->flags &= ~CD_FLAG_SCHED_ON_COMP;
-
-		/*
 		 * Check to see whether the current device has any I/O left
 		 * to do.  If so, requeue it at the end of the queue.  If
 		 * not, there is no need to requeue it.
@@ -1242,12 +1229,12 @@ cdrunchangerqueue(void *arg)
 	 * ones so this device gets its full time quantum.
 	 */
 	if (changer->flags & CHANGER_TIMEOUT_SCHED) {
-		untimeout(cdrunchangerqueue, changer, changer->long_handle);
+		callout_stop(&changer->long_handle);
 		changer->flags &= ~CHANGER_TIMEOUT_SCHED;
 	}
 
 	if (changer->flags & CHANGER_SHORT_TMOUT_SCHED) {
-		untimeout(cdshorttimeout, changer, changer->short_handle);
+		callout_stop(&changer->short_handle);
 		changer->flags &= ~CHANGER_SHORT_TMOUT_SCHED;
 	}
 
@@ -1257,17 +1244,12 @@ cdrunchangerqueue(void *arg)
 	 * switch time.
 	 */
 	changer->flags |= CHANGER_NEED_TIMEOUT;
-
-	splx(s);
 }
 
 static void
 cdchangerschedule(struct cd_softc *softc)
 {
 	struct cdchanger *changer;
-	int s;
-
-	s = splsoftcam();
 
 	changer = softc->changer;
 
@@ -1313,18 +1295,18 @@ cdchangerschedule(struct cd_softc *softc)
 		 * and schedule our timeouts.
 		 */
 		if ((changer->flags & CHANGER_TIMEOUT_SCHED) == 0) {
-			changer->long_handle =
-			    timeout(cdrunchangerqueue, changer,
-				    changer_max_busy_seconds * hz);
+			callout_reset(&changer->long_handle,
+			    changer_max_busy_seconds * hz,
+			    cdrunchangerqueue, changer);
 			changer->flags |= CHANGER_TIMEOUT_SCHED;
 		} else
 			printf("cdchangerschedule: already have a long"
 			       " timeout!\n");
 
 		if ((changer->flags & CHANGER_SHORT_TMOUT_SCHED) == 0) {
-			changer->short_handle =
-			    timeout(cdshorttimeout, changer,
-				    changer_min_busy_seconds * hz);
+			callout_reset(&changer->short_handle,
+			    changer_min_busy_seconds * hz,
+			    cdshorttimeout, changer);
 			changer->flags |= CHANGER_SHORT_TMOUT_SCHED;
 		} else
 			printf("cdchangerschedule: already have a short "
@@ -1337,7 +1319,6 @@ cdchangerschedule(struct cd_softc *softc)
 		changer->flags &= ~CHANGER_NEED_TIMEOUT;
 
 	}
-	splx(s);
 }
 
 static int
@@ -1366,14 +1347,10 @@ static union ccb *
 cdgetccb(struct cam_periph *periph, u_int32_t priority)
 {
 	struct cd_softc *softc;
-	int s;
 
 	softc = (struct cd_softc *)periph->softc;
 
 	if (softc->flags & CD_FLAG_CHANGER) {
-
-		s = splsoftcam();
-
 		/*
 		 * This should work the first time this device is woken up,
 		 * but just in case it doesn't, we use a while loop.
@@ -1396,9 +1373,9 @@ cdgetccb(struct cam_periph *periph, u_int32_t priority)
 				softc->changer->flags |= CHANGER_MANUAL_CALL;
 				cdrunchangerqueue(softc->changer);
 			} else
-				tsleep(&softc->changer, PRIBIO, "cgticb", 0);
+				msleep(&softc->changer, periph->sim->mtx,
+				    PRIBIO, "cgticb", 0);
 		}
-		splx(s);
 	}
 	return(cam_periph_getccb(periph, priority));
 }
@@ -1414,7 +1391,6 @@ cdstrategy(struct bio *bp)
 {
 	struct cam_periph *periph;
 	struct cd_softc *softc;
-	int    s;
 
 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
 	if (periph == NULL) {
@@ -1422,22 +1398,16 @@ cdstrategy(struct bio *bp)
 		return;
 	}
 
+	cam_periph_lock(periph);
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("entering cdstrategy\n"));
 
 	softc = (struct cd_softc *)periph->softc;
 
 	/*
-	 * Mask interrupts so that the pack cannot be invalidated until
-	 * after we are in the queue.  Otherwise, we might not properly
-	 * clean up one of the buffers.
-	 */
-	s = splbio();
-	
-	/*
 	 * If the device has been made invalid, error out
 	 */
 	if ((softc->flags & CD_FLAG_INVALID)) {
-		splx(s);
+		cam_periph_unlock(periph);
 		biofinish(bp, NULL, ENXIO);
 		return;
 	}
@@ -1451,7 +1421,7 @@ cdstrategy(struct bio *bp)
 
 		error = cdcheckmedia(periph);
 		if (error != 0) {
-			splx(s);
+			cam_periph_unlock(periph);
 			biofinish(bp, NULL, error);
 			return;
 		}
@@ -1462,8 +1432,6 @@ cdstrategy(struct bio *bp)
 	 */
 	bioq_disksort(&softc->bio_queue, bp);
 
-	splx(s);
-	
 	/*
 	 * Schedule ourselves for performing the work.  We do things
 	 * differently for changers.
@@ -1473,6 +1441,7 @@ cdstrategy(struct bio *bp)
 	else
 		cdschedule(periph, /* priority */ 1);
 
+	cam_periph_unlock(periph);
 	return;
 }
 
@@ -1483,7 +1452,6 @@ cdstart(struct cam_periph *periph, union ccb *start_ccb)
 	struct bio *bp;
 	struct ccb_scsiio *csio;
 	struct scsi_read_capacity_data *rcap;
-	int s;
 
 	softc = (struct cd_softc *)periph->softc;
 
@@ -1492,9 +1460,6 @@ cdstart(struct cam_periph *periph, union ccb *start_ccb)
 	switch (softc->state) {
 	case CD_STATE_NORMAL:
 	{
-		int oldspl;
-
-		s = splbio();
 		bp = bioq_first(&softc->bio_queue);
 		if (periph->immediate_priority <= periph->pinfo.priority) {
 			start_ccb->ccb_h.ccb_state = CD_CCB_WAITING;
@@ -1502,10 +1467,8 @@ cdstart(struct cam_periph *periph, union ccb *start_ccb)
 			SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
 					  periph_links.sle);
 			periph->immediate_priority = CAM_PRIORITY_NONE;
-			splx(s);
 			wakeup(&periph->ccb_list);
 		} else if (bp == NULL) {
-			splx(s);
 			xpt_release_ccb(start_ccb);
 		} else {
 			bioq_remove(&softc->bio_queue, bp);
@@ -1529,15 +1492,9 @@ cdstart(struct cam_periph *periph, union ccb *start_ccb)
 			start_ccb->ccb_h.ccb_state = CD_CCB_BUFFER_IO;
 
 			
-			/*
-			 * Block out any asyncronous callbacks
-			 * while we touch the pending ccb list.
-			 */
-			oldspl = splcam();
 			LIST_INSERT_HEAD(&softc->pending_ccbs,
 					 &start_ccb->ccb_h, periph_links.le);
 			softc->outstanding_cmds++;
-			splx(oldspl);
 
 			/* We expect a unit attention from this device */
 			if ((softc->flags & CD_FLAG_RETRY_UA) != 0) {
@@ -1547,7 +1504,6 @@ cdstart(struct cam_periph *periph, union ccb *start_ccb)
 
 			start_ccb->ccb_h.ccb_bp = bp;
 			bp = bioq_first(&softc->bio_queue);
-			splx(s);
 
 			xpt_action(start_ccb);
 		}
@@ -1601,7 +1557,6 @@ cddone(struct cam_periph *periph, union ccb *done_ccb)
 	{
 		struct bio	*bp;
 		int		error;
-		int		oldspl;
 
 		bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
 		error = 0;
@@ -1625,13 +1580,9 @@ cddone(struct cam_periph *periph, union ccb *done_ccb)
 		}
 
 		if (error != 0) {
-			int s;
-
 			xpt_print(periph->path,
 			    "cddone: got error %#x back\n", error);
-			s = splbio();
 			bioq_flush(&softc->bio_queue, NULL, EIO);
-			splx(s);
 			bp->bio_resid = bp->bio_bcount;
 			bp->bio_error = error;
 			bp->bio_flags |= BIO_ERROR;
@@ -1654,14 +1605,8 @@ cddone(struct cam_periph *periph, union ccb *done_ccb)
 			}
 		}
 
-		/*
-		 * Block out any asyncronous callbacks
-		 * while we touch the pending ccb list.
-		 */
-		oldspl = splcam();
 		LIST_REMOVE(&done_ccb->ccb_h, periph_links.le);
 		softc->outstanding_cmds--;
-		splx(oldspl);
 
 		if (softc->flags & CD_FLAG_CHANGER)
 			cdchangerschedule(softc);
@@ -1852,7 +1797,7 @@ cddone(struct cam_periph *periph, union ccb *done_ccb)
 		 * operation.
 		 */
 		xpt_release_ccb(done_ccb);
-		cam_periph_unlock(periph);
+		cam_periph_unhold(periph);
 		return;
 	}
 	case CD_CCB_WAITING:
@@ -1905,12 +1850,13 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 
 	struct 	cam_periph *periph;
 	struct	cd_softc *softc;
-	int	error, nocopyout;
+	int	nocopyout, error = 0;
 
 	periph = (struct cam_periph *)dp->d_drv1;
 	if (periph == NULL)
 		return(ENXIO);	
 
+	cam_periph_lock(periph);
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("entering cdioctl\n"));
 
 	softc = (struct cd_softc *)periph->softc;
@@ -1918,10 +1864,12 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, 
 		  ("trying to do ioctl %#lx\n", cmd));
 
-	error = cam_periph_lock(periph, PRIBIO | PCATCH);
+	if ((error = cam_periph_hold(periph, PRIBIO | PCATCH)) != 0) {
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
+		return (error);
+	}
 
-	if (error != 0)
-		return(error);
 	/*
 	 * If we don't have media loaded, check for it.  If still don't
 	 * have media loaded, we can only do a load or eject.
@@ -1936,11 +1884,14 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 	  && (cmd != CDIOCEJECT))
 	 && (IOCGROUP(cmd) == 'c')) {
 		error = cdcheckmedia(periph);
-		if (error != 0) {
-			cam_periph_unlock(periph);
-			return (error);
-		}
 	}
+	/*
+	 * Drop the lock here so later mallocs can use WAITOK.  The periph
+	 * is essentially locked still with the cam_periph_hold call above.
+	 */
+	cam_periph_unlock(periph);
+	if (error != 0)
+		return (error);
 
 	nocopyout = 0;
 	switch (cmd) {
@@ -1956,12 +1907,14 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			params.mode_buf = malloc(params.alloc_len, M_TEMP,
 						 M_WAITOK | M_ZERO);
 
+			cam_periph_lock(periph);
 			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
 				  ("trying to do CDIOCPLAYTRACKS\n"));
 
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -1970,8 +1923,10 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.flags |= CD_PA_IMMED;
 			error = cdsetmode(periph, &params);
 			free(params.mode_buf, M_TEMP);
-			if (error)
+			if (error) {
+				cam_periph_unlock(periph);
 				break;
+			}
 
 			/*
 			 * This was originally implemented with the PLAY
@@ -2033,6 +1988,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 						     args->end_track,
 						     args->end_index);
 			}
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCPLAYMSF:
@@ -2046,12 +2002,14 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			params.mode_buf = malloc(params.alloc_len, M_TEMP,
 						 M_WAITOK | M_ZERO);
 
+			cam_periph_lock(periph);
 			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
 				  ("trying to do CDIOCPLAYMSF\n"));
 
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2060,8 +2018,10 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.flags |= CD_PA_IMMED;
 			error = cdsetmode(periph, &params);
 			free(params.mode_buf, M_TEMP);
-			if (error)
+			if (error) {
+				cam_periph_unlock(periph);
 				break;
+			}
 			error = cdplaymsf(periph,
 					  args->start_m,
 					  args->start_s,
@@ -2069,6 +2029,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 					  args->end_m,
 					  args->end_s,
 					  args->end_f);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCPLAYBLOCKS:
@@ -2078,16 +2039,19 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_mode_params params;
 			union cd_pages *page;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCPLAYBLOCKS\n"));
-
 			params.alloc_len = sizeof(union cd_mode_data_6_10);
 			params.mode_buf = malloc(params.alloc_len, M_TEMP,
 						 M_WAITOK | M_ZERO);
 
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCPLAYBLOCKS\n"));
+
+
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2096,9 +2060,12 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.flags |= CD_PA_IMMED;
 			error = cdsetmode(periph, &params);
 			free(params.mode_buf, M_TEMP);
-			if (error)
+			if (error) {
+				cam_periph_unlock(periph);
 				break;
+			}
 			error = cdplay(periph, args->blk, args->len);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCREADSUBCHANNEL_SYSSPACE:
@@ -2111,12 +2078,13 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_sub_channel_info *data;
 			u_int32_t len = args->data_len;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCREADSUBCHANNEL\n"));
-
 			data = malloc(sizeof(struct cd_sub_channel_info), 
 				      M_TEMP, M_WAITOK);
 
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCREADSUBCHANNEL\n"));
+
 			if ((len > sizeof(struct cd_sub_channel_info)) ||
 			    (len < sizeof(struct cd_sub_channel_header))) {
 				printf(
@@ -2125,6 +2093,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 					len);
 				error = EINVAL;
 				free(data, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 
@@ -2136,6 +2105,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 
 			if (error) {
 				free(data, M_TEMP);
+				cam_periph_unlock(periph);
 	 			break;
 			}
 			if (softc->quirks & CD_Q_BCD_TRACKS)
@@ -2144,6 +2114,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			len = min(len, ((data->header.data_len[0] << 8) +
 				data->header.data_len[1] +
 				sizeof(struct cd_sub_channel_header)));
+			cam_periph_unlock(periph);
 			if (nocopyout == 0) {
 				if (copyout(data, args->data, len) != 0) {
 					error = EFAULT;
@@ -2159,15 +2130,18 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 		{
 			struct ioc_toc_header *th;
 
+			th = malloc(sizeof(struct ioc_toc_header), M_TEMP,
+				    M_WAITOK);
+
+			cam_periph_lock(periph);
 			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
 				  ("trying to do CDIOREADTOCHEADER\n"));
 
-			th = malloc(sizeof(struct ioc_toc_header), M_TEMP,
-				    M_WAITOK);
 			error = cdreadtoc(periph, 0, 0, (u_int8_t *)th, 
 				          sizeof (*th), /*sense_flags*/0);
 			if (error) {
 				free(th, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			if (softc->quirks & CD_Q_BCD_TRACKS) {
@@ -2181,6 +2155,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			th->len = ntohs(th->len);
 			bcopy(th, addr, sizeof(*th));
 			free(th, M_TEMP);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOREADTOCENTRYS:
@@ -2193,12 +2168,13 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			u_int32_t len, readlen, idx, num;
 			u_int32_t starting_track = te->starting_track;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOREADTOCENTRYS\n"));
-
 			data = malloc(sizeof(*data), M_TEMP, M_WAITOK);
 			lead = malloc(sizeof(*lead), M_TEMP, M_WAITOK);
 
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOREADTOCENTRYS\n"));
+
 			if (te->data_len < sizeof(struct cd_toc_entry)
 			 || (te->data_len % sizeof(struct cd_toc_entry)) != 0
 			 || (te->address_format != CD_MSF_FORMAT
@@ -2208,6 +2184,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 				       "returning EINVAL\n");
 				free(data, M_TEMP);
 				free(lead, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 
@@ -2217,6 +2194,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			if (error) {
 				free(data, M_TEMP);
 				free(lead, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 
@@ -2239,6 +2217,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 				       "returning EINVAL\n");
 				free(data, M_TEMP);
 				free(lead, M_TEMP);
+				cam_periph_unlock(periph);
 				error = EINVAL;
 				break;
 			}
@@ -2260,6 +2239,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 				error = EINVAL;
 				free(data, M_TEMP);
 				free(lead, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			num = len / sizeof(struct cd_toc_entry);
@@ -2273,6 +2253,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 				if (error) {
 					free(data, M_TEMP);
 					free(lead, M_TEMP);
+					cam_periph_unlock(periph);
 					break;
 				}
 			}
@@ -2289,6 +2270,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 				if (error) {
 					free(data, M_TEMP);
 					free(lead, M_TEMP);
+					cam_periph_unlock(periph);
 					break;
 				}
 				data->entries[idx - starting_track] = 
@@ -2301,6 +2283,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 				}
 			}
 
+			cam_periph_unlock(periph);
 			error = copyout(data->entries, te->data, len);
 			free(data, M_TEMP);
 			free(lead, M_TEMP);
@@ -2314,17 +2297,19 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct ioc_toc_header *th;
 			u_int32_t track;
 
+			data = malloc(sizeof(*data), M_TEMP, M_WAITOK);
+
+			cam_periph_lock(periph);
 			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
 				  ("trying to do CDIOREADTOCENTRY\n"));
 
-			data = malloc(sizeof(*data), M_TEMP, M_WAITOK);
-
 			if (te->address_format != CD_MSF_FORMAT
 			    && te->address_format != CD_LBA_FORMAT) {
 				printf("error in readtocentry, "
 				       " returning EINVAL\n");
 				free(data, M_TEMP);
 				error = EINVAL;
+				cam_periph_unlock(periph);
 				break;
 			}
 
@@ -2333,6 +2318,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 					  sizeof (*th), /*sense_flags*/0);
 			if (error) {
 				free(data, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 
@@ -2355,6 +2341,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 				       " returning EINVAL\n");
 				free(data, M_TEMP);
 				error = EINVAL;
+				cam_periph_unlock(periph);
 				break;
 			}
 
@@ -2363,6 +2350,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 					  /*sense_flags*/0);
 			if (error) {
 				free(data, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 
@@ -2371,6 +2359,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			bcopy(&data->entry, &te->entry,
 			      sizeof(struct cd_toc_entry));
 			free(data, M_TEMP);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCSETPATCH:
@@ -2379,15 +2368,18 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_mode_params params;
 			union cd_pages *page;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCSETPATCH\n"));
-
 			params.alloc_len = sizeof(union cd_mode_data_6_10);
 			params.mode_buf = malloc(params.alloc_len, M_TEMP, 
 						 M_WAITOK | M_ZERO);
+
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCSETPATCH\n"));
+
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2400,6 +2392,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.port[3].channels = arg->patch[3];
 			error = cdsetmode(periph, &params);
 			free(params.mode_buf, M_TEMP);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCGETVOL:
@@ -2408,15 +2401,18 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_mode_params params;
 			union cd_pages *page;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCGETVOL\n"));
-
 			params.alloc_len = sizeof(union cd_mode_data_6_10);
 			params.mode_buf = malloc(params.alloc_len, M_TEMP, 
 						 M_WAITOK | M_ZERO);
+
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCGETVOL\n"));
+
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2428,6 +2424,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			arg->vol[2] = page->audio.port[2].volume;
 			arg->vol[3] = page->audio.port[3].volume;
 			free(params.mode_buf, M_TEMP);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCSETVOL:
@@ -2436,15 +2433,18 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_mode_params params;
 			union cd_pages *page;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCSETVOL\n"));
-
 			params.alloc_len = sizeof(union cd_mode_data_6_10);
 			params.mode_buf = malloc(params.alloc_len, M_TEMP, 
 						 M_WAITOK | M_ZERO);
+
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCSETVOL\n"));
+
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2458,6 +2458,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.port[2].volume = arg->vol[2];
 			page->audio.port[3].volume = arg->vol[3];
 			error = cdsetmode(periph, &params);
+			cam_periph_unlock(periph);
 			free(params.mode_buf, M_TEMP);
 		}
 		break;
@@ -2466,15 +2467,18 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_mode_params params;
 			union cd_pages *page;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCSETMONO\n"));
-
 			params.alloc_len = sizeof(union cd_mode_data_6_10);
 			params.mode_buf = malloc(params.alloc_len, M_TEMP,
 						 M_WAITOK | M_ZERO);
+
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCSETMONO\n"));
+
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2486,6 +2490,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.port[2].channels = 0;
 			page->audio.port[3].channels = 0;
 			error = cdsetmode(periph, &params);
+			cam_periph_unlock(periph);
 			free(params.mode_buf, M_TEMP);
 		}
 		break;
@@ -2494,15 +2499,18 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_mode_params params;
 			union cd_pages *page;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCSETSTEREO\n"));
-
 			params.alloc_len = sizeof(union cd_mode_data_6_10);
 			params.mode_buf = malloc(params.alloc_len, M_TEMP,
 						 M_WAITOK | M_ZERO);
+
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCSETSTEREO\n"));
+
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2515,6 +2523,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.port[3].channels = 0;
 			error = cdsetmode(periph, &params);
 			free(params.mode_buf, M_TEMP);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCSETMUTE:
@@ -2522,15 +2531,18 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_mode_params params;
 			union cd_pages *page;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCSETMUTE\n"));
-
 			params.alloc_len = sizeof(union cd_mode_data_6_10);
 			params.mode_buf = malloc(params.alloc_len, M_TEMP,
 						 M_WAITOK | M_ZERO);
+
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCSETMUTE\n"));
+
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(&params, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2541,6 +2553,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.port[3].channels = 0;
 			error = cdsetmode(periph, &params);
 			free(params.mode_buf, M_TEMP);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCSETLEFT:
@@ -2548,16 +2561,18 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_mode_params params;
 			union cd_pages *page;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCSETLEFT\n"));
-
 			params.alloc_len = sizeof(union cd_mode_data_6_10);
 			params.mode_buf = malloc(params.alloc_len, M_TEMP,
 						 M_WAITOK | M_ZERO);
-			
+
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCSETLEFT\n"));
+
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2568,6 +2583,7 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.port[3].channels = 0;
 			error = cdsetmode(periph, &params);
 			free(params.mode_buf, M_TEMP);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCSETRIGHT:
@@ -2575,16 +2591,18 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			struct cd_mode_params params;
 			union cd_pages *page;
 
-			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
-				  ("trying to do CDIOCSETRIGHT\n"));
-
 			params.alloc_len = sizeof(union cd_mode_data_6_10);
 			params.mode_buf = malloc(params.alloc_len, M_TEMP,
 						 M_WAITOK | M_ZERO);
 
+			cam_periph_lock(periph);
+			CAM_DEBUG(periph->path, CAM_DEBUG_SUBTRACE, 
+				  ("trying to do CDIOCSETRIGHT\n"));
+
 			error = cdgetmode(periph, &params, AUDIO_PAGE);
 			if (error) {
 				free(params.mode_buf, M_TEMP);
+				cam_periph_unlock(periph);
 				break;
 			}
 			page = cdgetpage(&params);
@@ -2595,31 +2613,48 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 			page->audio.port[3].channels = 0;
 			error = cdsetmode(periph, &params);
 			free(params.mode_buf, M_TEMP);
+			cam_periph_unlock(periph);
 		}
 		break;
 	case CDIOCRESUME:
+		cam_periph_lock(periph);
 		error = cdpause(periph, 1);
+		cam_periph_unlock(periph);
 		break;
 	case CDIOCPAUSE:
+		cam_periph_lock(periph);
 		error = cdpause(periph, 0);
+		cam_periph_unlock(periph);
 		break;
 	case CDIOCSTART:
+		cam_periph_lock(periph);
 		error = cdstartunit(periph, 0);
+		cam_periph_unlock(periph);
 		break;
 	case CDIOCCLOSE:
+		cam_periph_lock(periph);
 		error = cdstartunit(periph, 1);
+		cam_periph_unlock(periph);
 		break;
 	case CDIOCSTOP:
+		cam_periph_lock(periph);
 		error = cdstopunit(periph, 0);
+		cam_periph_unlock(periph);
 		break;
 	case CDIOCEJECT:
+		cam_periph_lock(periph);
 		error = cdstopunit(periph, 1);
+		cam_periph_unlock(periph);
 		break;
 	case CDIOCALLOW:
+		cam_periph_lock(periph);
 		cdprevent(periph, PR_ALLOW);
+		cam_periph_unlock(periph);
 		break;
 	case CDIOCPREVENT:
+		cam_periph_lock(periph);
 		cdprevent(periph, PR_PREVENT);
+		cam_periph_unlock(periph);
 		break;
 	case CDIOCSETDEBUG:
 		/* sc_link->flags |= (SDEV_DB1 | SDEV_DB2); */
@@ -2634,10 +2669,14 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 		error = ENOTTY;
 		break;
 	case CDRIOCREADSPEED:
+		cam_periph_lock(periph);
 		error = cdsetspeed(periph, *(u_int32_t *)addr, CDR_MAX_SPEED);
+		cam_periph_unlock(periph);
 		break;
 	case CDRIOCWRITESPEED:
+		cam_periph_lock(periph);
 		error = cdsetspeed(periph, CDR_MAX_SPEED, *(u_int32_t *)addr);
+		cam_periph_unlock(periph);
 		break;
 	case DVDIOCSENDKEY:
 	case DVDIOCREPORTKEY: {
@@ -2645,10 +2684,12 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 
 		authinfo = (struct dvd_authinfo *)addr;
 
+		cam_periph_lock(periph);
 		if (cmd == DVDIOCREPORTKEY)
 			error = cdreportkey(periph, authinfo);
 		else
 			error = cdsendkey(periph, authinfo);
+		cam_periph_unlock(periph);
 		break;
 		}
 	case DVDIOCREADSTRUCTURE: {
@@ -2656,21 +2697,27 @@ cdioctl(struct disk *dp, u_long cmd, void *addr, int flag, struct thread *td)
 
 		dvdstruct = (struct dvd_struct *)addr;
 
+		cam_periph_lock(periph);
 		error = cdreaddvdstructure(periph, dvdstruct);
+		cam_periph_unlock(periph);
 
 		break;
 	}
 	default:
+		cam_periph_lock(periph);
 		error = cam_periph_ioctl(periph, cmd, addr, cderror);
+		cam_periph_unlock(periph);
 		break;
 	}
 
-	cam_periph_unlock(periph);
-
+	cam_periph_lock(periph);
+	cam_periph_unhold(periph);
+	
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("leaving cdioctl\n"));
 	if (error && bootverbose) {
 		printf("scsi_cd.c::ioctl cmd=%08lx error=%d\n", cmd, error);
 	}
+	cam_periph_unlock(periph);
 
 	return (error);
 }
@@ -2869,8 +2916,11 @@ cdsize(struct cam_periph *periph, u_int32_t *size)
              
 	ccb = cdgetccb(periph, /* priority */ 1);
 
+	/* XXX Should be M_WAITOK */
 	rcap_buf = malloc(sizeof(struct scsi_read_capacity_data), 
-			  M_TEMP, M_WAITOK);
+			  M_TEMP, M_NOWAIT);
+	if (rcap_buf == NULL)
+		return (ENOMEM);
 
 	scsi_read_capacity(&ccb->csio, 
 			   /*retries*/ 1,
diff --git a/sys/cam/scsi/scsi_ch.c b/sys/cam/scsi/scsi_ch.c
index 7ca7391..337cea3 100644
--- a/sys/cam/scsi/scsi_ch.c
+++ b/sys/cam/scsi/scsi_ch.c
@@ -173,8 +173,6 @@ struct ch_softc {
 	int		sc_settledelay;	/* delay for settle */
 };
 
-#define CHUNIT(x)       (minor((x)))
-
 static	d_open_t	chopen;
 static	d_close_t	chclose;
 static	d_ioctl_t	chioctl;
@@ -213,7 +211,7 @@ PERIPHDRIVER_DECLARE(ch, chdriver);
 
 static struct cdevsw ch_cdevsw = {
 	.d_version =	D_VERSION,
-	.d_flags =	D_NEEDGIANT,
+	.d_flags =	0,
 	.d_open =	chopen,
 	.d_close =	chclose,
 	.d_ioctl =	chioctl,
@@ -376,9 +374,11 @@ chregister(struct cam_periph *periph, void *arg)
 			  DEVSTAT_PRIORITY_OTHER);
 
 	/* Register the device */
+	cam_periph_unlock(periph);
 	softc->dev = make_dev(&ch_cdevsw, periph->unit_number, UID_ROOT,
 			      GID_OPERATOR, 0600, "%s%d", periph->periph_name,
 			      periph->unit_number);
+	cam_periph_lock(periph);
 	softc->dev->si_drv1 = periph;
 
 	/*
@@ -393,10 +393,10 @@ chregister(struct cam_periph *periph, void *arg)
 	xpt_action((union ccb *)&csa);
 
 	/*
-	 * Lock this peripheral until we are setup.
+	 * Lock this periph until we are setup.
 	 * This first call can't block
 	 */
-	(void)cam_periph_lock(periph, PRIBIO);
+	(void)cam_periph_hold(periph, PRIBIO);
 	xpt_schedule(periph, /*priority*/5);
 
 	return(CAM_REQ_CMP);
@@ -408,31 +408,30 @@ chopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	struct cam_periph *periph;
 	struct ch_softc *softc;
 	int error;
-	int s;
 
 	periph = (struct cam_periph *)dev->si_drv1;
-	if (periph == NULL)
-		return(ENXIO);
+	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
+		return (ENXIO);
 
 	softc = (struct ch_softc *)periph->softc;
 
-	s = splsoftcam();
+	cam_periph_lock(periph);
+	
 	if (softc->flags & CH_FLAG_INVALID) {
-		splx(s);
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return(ENXIO);
 	}
 
-	if ((error = cam_periph_lock(periph, PRIBIO | PCATCH)) != 0) {
-		splx(s);
-		return (error);
-	}
-	
-	splx(s);
-
-	if ((softc->flags & CH_FLAG_OPEN) == 0) {
-		if (cam_periph_acquire(periph) != CAM_REQ_CMP)
-			return(ENXIO);
+	if ((softc->flags & CH_FLAG_OPEN) == 0)
 		softc->flags |= CH_FLAG_OPEN;
+	else
+		cam_periph_release(periph);
+
+	if ((error = cam_periph_hold(periph, PRIBIO | PCATCH)) != 0) {
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
+		return (error);
 	}
 
 	/*
@@ -445,6 +444,7 @@ chopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 		return(error);
 	}
 
+	cam_periph_unhold(periph);
 	cam_periph_unlock(periph);
 
 	return(error);
@@ -465,8 +465,7 @@ chclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 
 	softc = (struct ch_softc *)periph->softc;
 
-	if ((error = cam_periph_lock(periph, PRIBIO)) != 0)
-		return(error);
+	cam_periph_lock(periph);
 
 	softc->flags &= ~CH_FLAG_OPEN;
 
@@ -480,24 +479,20 @@ static void
 chstart(struct cam_periph *periph, union ccb *start_ccb)
 {
 	struct ch_softc *softc;
-	int s;
 
 	softc = (struct ch_softc *)periph->softc;
 
 	switch (softc->state) {
 	case CH_STATE_NORMAL:
 	{
-		s = splbio();
 		if (periph->immediate_priority <= periph->pinfo.priority){
 			start_ccb->ccb_h.ccb_state = CH_CCB_WAITING;
 
 			SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
 					  periph_links.sle);
 			periph->immediate_priority = CAM_PRIORITY_NONE;
-			splx(s);
 			wakeup(&periph->ccb_list);
-		} else
-			splx(s);
+		}
 		break;
 	}
 	case CH_STATE_PROBE:
@@ -670,7 +665,7 @@ chdone(struct cam_periph *periph, union ccb *done_ccb)
 		 * operation.
 		 */
 		xpt_release_ccb(done_ccb);
-		cam_periph_unlock(periph);
+		cam_periph_unhold(periph);
 		return;
 	}
 	case CH_CCB_WAITING:
@@ -709,6 +704,7 @@ chioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 	if (periph == NULL)
 		return(ENXIO);
 
+	cam_periph_lock(periph);
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("entering chioctl\n"));
 
 	softc = (struct ch_softc *)periph->softc;
@@ -729,8 +725,10 @@ chioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 		break;
 
 	default:
-		if ((flag & FWRITE) == 0)
+		if ((flag & FWRITE) == 0) {
+			cam_periph_unlock(periph);
 			return (EBADF);
+		}
 	}
 
 	switch (cmd) {
@@ -754,8 +752,10 @@ chioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 	{
 		int new_picker = *(int *)addr;
 
-		if (new_picker > (softc->sc_counts[CHET_MT] - 1))
-			return (EINVAL);
+		if (new_picker > (softc->sc_counts[CHET_MT] - 1)) {
+			error = EINVAL;
+			break;
+		}
 		softc->sc_picker = softc->sc_firsts[CHET_MT] + new_picker;
 		break;
 	}
@@ -794,6 +794,7 @@ chioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 		break;
 	}
 
+	cam_periph_unlock(periph);
 	return (error);
 }
 
@@ -1091,8 +1092,10 @@ chgetelemstatus(struct cam_periph *periph,
 	 * we can allocate enough storage for all of them.  We assume
 	 * that the first one can fit into 1k.
 	 */
+	cam_periph_unlock(periph);
 	data = (caddr_t)malloc(1024, M_DEVBUF, M_WAITOK);
 
+	cam_periph_lock(periph);
 	ccb = cam_periph_getccb(periph, /*priority*/ 1);
 
 	scsi_read_element_status(&ccb->csio,
@@ -1113,6 +1116,7 @@ chgetelemstatus(struct cam_periph *periph,
 
 	if (error)
 		goto done;
+	cam_periph_unlock(periph);
 
 	st_hdr = (struct read_element_status_header *)data;
 	pg_hdr = (struct read_element_status_page_header *)((uintptr_t)st_hdr +
@@ -1130,6 +1134,7 @@ chgetelemstatus(struct cam_periph *periph,
 	free(data, M_DEVBUF);
 	data = (caddr_t)malloc(size, M_DEVBUF, M_WAITOK);
 
+	cam_periph_lock(periph);
 	scsi_read_element_status(&ccb->csio,
 				 /* retries */ 1,
 				 /* cbfcnp */ chdone,
@@ -1149,6 +1154,7 @@ chgetelemstatus(struct cam_periph *periph,
 
 	if (error)
 		goto done;
+	cam_periph_unlock(periph);
 
 	/*
 	 * Fill in the user status array.
@@ -1186,6 +1192,7 @@ chgetelemstatus(struct cam_periph *periph,
 	error = copyout(user_data,
 			cesr->cesr_element_status,
 			avail * sizeof(struct changer_element_status));
+	cam_periph_lock(periph);
 
  done:
 	xpt_release_ccb(ccb);
diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index b4863f6..f799e44 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -37,6 +37,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/bio.h>
 #include <sys/sysctl.h>
 #include <sys/taskqueue.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #endif /* _KERNEL */
 
 #include <sys/devicestat.h>
@@ -61,6 +63,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/cam_ccb.h>
 #include <cam/cam_periph.h>
 #include <cam/cam_xpt_periph.h>
+#include <cam/cam_sim.h>
 
 #include <cam/scsi/scsi_message.h>
 
@@ -133,6 +136,7 @@ struct da_softc {
 	struct task		sysctl_task;
 	struct sysctl_ctx_list	sysctl_ctx;
 	struct sysctl_oid	*sysctl_tree;
+	struct callout		sendordered_c;
 };
 
 struct da_quirk_entry {
@@ -551,8 +555,6 @@ static struct periph_driver dadriver =
 
 PERIPHDRIVER_DECLARE(da, dadriver);
 
-static SLIST_HEAD(,da_softc) softc_list;
-
 static int
 daopen(struct disk *dp)
 {
@@ -560,34 +562,35 @@ daopen(struct disk *dp)
 	struct da_softc *softc;
 	int unit;
 	int error;
-	int s;
 
-	s = splsoftcam();
 	periph = (struct cam_periph *)dp->d_drv1;
 	if (periph == NULL) {
-		splx(s);
 		return (ENXIO);	
 	}
-	unit = periph->unit_number;
 
+	if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
+		return(ENXIO);
+	}
+
+	cam_periph_lock(periph);
+	if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
+		return (error);
+	}
+
+	unit = periph->unit_number;
 	softc = (struct da_softc *)periph->softc;
+	softc->flags |= DA_FLAG_OPEN;
 
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE,
 	    ("daopen: disk=%s%d (unit %d)\n", dp->d_name, dp->d_unit,
 	     unit));
 
-	if ((error = cam_periph_lock(periph, PRIBIO|PCATCH)) != 0)
-		return (error); /* error code from tsleep */
-
-	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
-		return(ENXIO);
-	softc->flags |= DA_FLAG_OPEN;
-
 	if ((softc->flags & DA_FLAG_PACK_INVALID) != 0) {
 		/* Invalidate our pack information. */
 		softc->flags &= ~DA_FLAG_PACK_INVALID;
 	}
-	splx(s);
 
 	error = dagetcapacity(periph);
 
@@ -610,6 +613,7 @@ daopen(struct disk *dp)
 		softc->flags &= ~DA_FLAG_OPEN;
 		cam_periph_release(periph);
 	}
+	cam_periph_unhold(periph);
 	cam_periph_unlock(periph);
 	return (error);
 }
@@ -619,18 +623,21 @@ daclose(struct disk *dp)
 {
 	struct	cam_periph *periph;
 	struct	da_softc *softc;
-	int	error;
+	int error;
 
 	periph = (struct cam_periph *)dp->d_drv1;
 	if (periph == NULL)
 		return (ENXIO);	
 
-	softc = (struct da_softc *)periph->softc;
-
-	if ((error = cam_periph_lock(periph, PRIBIO)) != 0) {
-		return (error); /* error code from tsleep */
+	cam_periph_lock(periph);
+	if ((error = cam_periph_hold(periph, PRIBIO)) != 0) {
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
+		return (error);
 	}
 
+	softc = (struct da_softc *)periph->softc;
+
 	if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0) {
 		union	ccb *ccb;
 
@@ -692,6 +699,7 @@ daclose(struct disk *dp)
 	}
 
 	softc->flags &= ~DA_FLAG_OPEN;
+	cam_periph_unhold(periph);
 	cam_periph_unlock(periph);
 	cam_periph_release(periph);
 	return (0);	
@@ -707,7 +715,6 @@ dastrategy(struct bio *bp)
 {
 	struct cam_periph *periph;
 	struct da_softc *softc;
-	int    s;
 	
 	periph = (struct cam_periph *)bp->bio_disk->d_drv1;
 	if (periph == NULL) {
@@ -715,6 +722,9 @@ dastrategy(struct bio *bp)
 		return;
 	}
 	softc = (struct da_softc *)periph->softc;
+
+	cam_periph_lock(periph);
+
 #if 0
 	/*
 	 * check it's not too big a transfer for our adapter
@@ -727,13 +737,12 @@ dastrategy(struct bio *bp)
 	 * after we are in the queue.  Otherwise, we might not properly
 	 * clean up one of the buffers.
 	 */
-	s = splbio();
 	
 	/*
 	 * If the device has been made invalid, error out
 	 */
 	if ((softc->flags & DA_FLAG_PACK_INVALID)) {
-		splx(s);
+		cam_periph_unlock(periph);
 		biofinish(bp, NULL, ENXIO);
 		return;
 	}
@@ -743,12 +752,11 @@ dastrategy(struct bio *bp)
 	 */
 	bioq_disksort(&softc->bio_queue, bp);
 
-	splx(s);
-	
 	/*
 	 * Schedule ourselves for performing the work.
 	 */
 	xpt_schedule(periph, /* XXX priority */1);
+	cam_periph_unlock(periph);
 
 	return;
 }
@@ -773,6 +781,7 @@ dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t leng
 		return (ENXIO);
 
 	if (length > 0) {
+		periph->flags |= CAM_PERIPH_POLLED;
 		xpt_setup_ccb(&csio.ccb_h, periph->path, /*priority*/1);
 		csio.ccb_h.ccb_state = DA_CCB_DUMP;
 		scsi_read_write(&csio,
@@ -798,10 +807,11 @@ dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t leng
 			else
 				printf("status == 0x%x, scsi status == 0x%x\n",
 				       csio.ccb_h.status, csio.scsi_status);
+			periph->flags |= CAM_PERIPH_POLLED;
 			return(EIO);
 		}
 		return(0);
-	} 
+	}
 		
 	/*
 	 * Sync the disk cache contents to the physical media.
@@ -840,6 +850,7 @@ dadump(void *arg, void *virtual, vm_offset_t physical, off_t offset, size_t leng
 			}
 		}
 	}
+	periph->flags &= ~CAM_PERIPH_POLLED;
 	return (0);
 }
 
@@ -849,8 +860,6 @@ dainit(void)
 	cam_status status;
 	struct cam_path *path;
 
-	SLIST_INIT(&softc_list);
-	
 	/*
 	 * Install a global async callback.  This callback will
 	 * receive async callbacks like "new device found".
@@ -876,13 +885,6 @@ dainit(void)
 		       "due to status 0x%x!\n", status);
 	} else if (da_send_ordered) {
 
-		/*
-		 * Schedule a periodic event to occasionally send an
-		 * ordered tag to a device.
-		 */
-		timeout(dasendorderedtag, NULL,
-			(DA_DEFAULT_TIMEOUT * hz) / DA_ORDEREDTAG_INTERVAL);
-
 		/* Register our shutdown event handler */
 		if ((EVENTHANDLER_REGISTER(shutdown_post_sync, dashutdown, 
 					   NULL, SHUTDOWN_PRI_DEFAULT)) == NULL)
@@ -893,7 +895,6 @@ dainit(void)
 static void
 daoninvalidate(struct cam_periph *periph)
 {
-	int s;
 	struct da_softc *softc;
 	struct ccb_setasync csa;
 
@@ -913,21 +914,11 @@ daoninvalidate(struct cam_periph *periph)
 	softc->flags |= DA_FLAG_PACK_INVALID;
 
 	/*
-	 * Although the oninvalidate() routines are always called at
-	 * splsoftcam, we need to be at splbio() here to keep the buffer
-	 * queue from being modified while we traverse it.
-	 */
-	s = splbio();
-
-	/*
 	 * Return all queued I/O with ENXIO.
 	 * XXX Handle any transactions queued to the card
 	 *     with XPT_ABORT_CCB.
 	 */
 	bioq_flush(&softc->bio_queue, NULL, ENXIO);
-	splx(s);
-
-	SLIST_REMOVE(&softc_list, softc, da_softc, links);
 
 	disk_gone(softc->disk);
 	xpt_print(periph->path, "lost device\n");
@@ -949,6 +940,14 @@ dacleanup(struct cam_periph *periph)
 		xpt_print(periph->path, "can't remove sysctl context\n");
 	}
 	disk_destroy(softc->disk);
+
+	/*
+	 * XXX Gotta drop the periph lock so that the drain can complete with
+	 * deadlocking on the lock.  Hopefully dropping here is safe.
+	 */
+	cam_periph_unlock(periph);
+	callout_drain(&softc->sendordered_c);
+	cam_periph_lock(periph);
 	free(softc, M_DEVBUF);
 }
 
@@ -963,6 +962,7 @@ daasync(void *callback_arg, u_int32_t code,
 	case AC_FOUND_DEVICE:
 	{
 		struct ccb_getdev *cgd;
+		struct cam_sim *sim;
 		cam_status status;
  
 		cgd = (struct ccb_getdev *)arg;
@@ -979,6 +979,7 @@ daasync(void *callback_arg, u_int32_t code,
 		 * this device and start the probe
 		 * process.
 		 */
+		sim = xpt_path_sim(cgd->ccb_h.path);
 		status = cam_periph_alloc(daregister, daoninvalidate,
 					  dacleanup, dastart,
 					  "da", CAM_PERIPH_BIO,
@@ -996,10 +997,8 @@ daasync(void *callback_arg, u_int32_t code,
 	{
 		struct da_softc *softc;
 		struct ccb_hdr *ccbh;
-		int s;
 
 		softc = (struct da_softc *)periph->softc;
-		s = splsoftcam();
 		/*
 		 * Don't fail on the expected unit attention
 		 * that will occur.
@@ -1007,7 +1006,6 @@ daasync(void *callback_arg, u_int32_t code,
 		softc->flags |= DA_FLAG_RETRY_UA;
 		LIST_FOREACH(ccbh, &softc->pending_ccbs, periph_links.le)
 			ccbh->ccb_state |= DA_CCB_RETRY_UA;
-		splx(s);
 		/* FALLTHROUGH*/
 	}
 	default:
@@ -1024,8 +1022,10 @@ dasysctlinit(void *context, int pending)
 	char tmpstr[80], tmpstr2[80];
 
 	periph = (struct cam_periph *)context;
-	softc = (struct da_softc *)periph->softc;
+	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
+		return;
 
+	softc = (struct da_softc *)periph->softc;
 	snprintf(tmpstr, sizeof(tmpstr), "CAM DA unit %d", periph->unit_number);
 	snprintf(tmpstr2, sizeof(tmpstr2), "%d", periph->unit_number);
 
@@ -1038,6 +1038,7 @@ dasysctlinit(void *context, int pending)
 	if (softc->sysctl_tree == NULL) {
 		printf("dasysctlinit: unable to allocate sysctl tree\n");
 		mtx_unlock(&Giant);
+		cam_periph_release(periph);
 		return;
 	}
 
@@ -1051,6 +1052,7 @@ dasysctlinit(void *context, int pending)
 		"Minimum CDB size");
 
 	mtx_unlock(&Giant);
+	cam_periph_release(periph);
 }
 
 static int
@@ -1088,7 +1090,6 @@ dacmdsizesysctl(SYSCTL_HANDLER_ARGS)
 static cam_status
 daregister(struct cam_periph *periph, void *arg)
 {
-	int s;
 	struct da_softc *softc;
 	struct ccb_setasync csa;
 	struct ccb_pathinq cpi;
@@ -1178,17 +1179,10 @@ daregister(struct cam_periph *periph, void *arg)
 		softc->minimum_cmd_size = 16;
 
 	/*
-	 * Block our timeout handler while we
-	 * add this softc to the dev list.
-	 */
-	s = splsoftclock();
-	SLIST_INSERT_HEAD(&softc_list, softc, links);
-	splx(s);
-
-	/*
 	 * Register this media as a disk
 	 */
 
+	mtx_unlock(periph->sim->mtx);
 	softc->disk = disk_alloc();
 	softc->disk->d_open = daopen;
 	softc->disk->d_close = daclose;
@@ -1198,10 +1192,11 @@ daregister(struct cam_periph *periph, void *arg)
 	softc->disk->d_drv1 = periph;
 	softc->disk->d_maxsize = DFLTPHYS; /* XXX: probably not arbitrary */
 	softc->disk->d_unit = periph->unit_number;
-	softc->disk->d_flags = DISKFLAG_NEEDSGIANT;
+	softc->disk->d_flags = 0;
 	if ((softc->quirks & DA_Q_NO_SYNC_CACHE) == 0)
 		softc->disk->d_flags |= DISKFLAG_CANFLUSHCACHE;
 	disk_create(softc->disk, DISK_VERSION);
+	mtx_lock(periph->sim->mtx);
 
 	/*
 	 * Add async callbacks for bus reset and
@@ -1217,13 +1212,24 @@ daregister(struct cam_periph *periph, void *arg)
 	csa.callback = daasync;
 	csa.callback_arg = periph;
 	xpt_action((union ccb *)&csa);
+
 	/*
-	 * Lock this peripheral until we are setup.
-	 * This first call can't block
+	 * Take an exclusive refcount on the periph while dastart is called
+	 * to finish the probe.  The reference will be dropped in dadone at
+	 * the end of probe.
 	 */
-	(void)cam_periph_lock(periph, PRIBIO);
+	(void)cam_periph_hold(periph, PRIBIO);
 	xpt_schedule(periph, /*priority*/5);
 
+	/*
+	 * Schedule a periodic event to occasionally send an
+	 * ordered tag to a device.
+	 */
+	callout_init_mtx(&softc->sendordered_c, periph->sim->mtx, 0);
+	callout_reset(&softc->sendordered_c,
+	    (DA_DEFAULT_TIMEOUT * hz) / DA_ORDEREDTAG_INTERVAL,
+	    dasendorderedtag, softc);
+
 	return(CAM_REQ_CMP);
 }
 
@@ -1234,18 +1240,15 @@ dastart(struct cam_periph *periph, union ccb *start_ccb)
 
 	softc = (struct da_softc *)periph->softc;
 
-	
 	switch (softc->state) {
 	case DA_STATE_NORMAL:
 	{
 		/* Pull a buffer from the queue and get going on it */		
 		struct bio *bp;
-		int s;
 
 		/*
 		 * See if there is a buf with work for us to do..
 		 */
-		s = splbio();
 		bp = bioq_first(&softc->bio_queue);
 		if (periph->immediate_priority <= periph->pinfo.priority) {
 			CAM_DEBUG_PRINT(CAM_DEBUG_SUBTRACE,
@@ -1254,13 +1257,10 @@ dastart(struct cam_periph *periph, union ccb *start_ccb)
 			SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
 					  periph_links.sle);
 			periph->immediate_priority = CAM_PRIORITY_NONE;
-			splx(s);
 			wakeup(&periph->ccb_list);
 		} else if (bp == NULL) {
-			splx(s);
 			xpt_release_ccb(start_ccb);
 		} else {
-			int oldspl;
 			u_int8_t tag_code;
 
 			bioq_remove(&softc->bio_queue, bp);
@@ -1307,11 +1307,9 @@ dastart(struct cam_periph *periph, union ccb *start_ccb)
 			 * Block out any asyncronous callbacks
 			 * while we touch the pending ccb list.
 			 */
-			oldspl = splcam();
 			LIST_INSERT_HEAD(&softc->pending_ccbs,
 					 &start_ccb->ccb_h, periph_links.le);
 			softc->outstanding_cmds++;
-			splx(oldspl);
 
 			/* We expect a unit attention from this device */
 			if ((softc->flags & DA_FLAG_RETRY_UA) != 0) {
@@ -1321,7 +1319,6 @@ dastart(struct cam_periph *periph, union ccb *start_ccb)
 
 			start_ccb->ccb_h.ccb_bp = bp;
 			bp = bioq_first(&softc->bio_queue);
-			splx(s);
 
 			xpt_action(start_ccb);
 		}
@@ -1446,12 +1443,10 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 	case DA_CCB_BUFFER_IO:
 	{
 		struct bio *bp;
-		int    oldspl;
 
 		bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
 			int error;
-			int s;
 			int sf;
 			
 			if ((csio->ccb_h.ccb_state & DA_CCB_RETRY_UA) != 0)
@@ -1469,8 +1464,6 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 			}
 			if (error != 0) {
 
-				s = splbio();
-
 				if (error == ENXIO) {
 					/*
 					 * Catastrophic error.  Mark our pack as
@@ -1491,7 +1484,6 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 				 * proper order should it attempt to recover.
 				 */
 				bioq_flush(&softc->bio_queue, NULL, EIO);
-				splx(s);
 				bp->bio_error = error;
 				bp->bio_resid = bp->bio_bcount;
 				bp->bio_flags |= BIO_ERROR;
@@ -1519,12 +1511,10 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 		 * Block out any asyncronous callbacks
 		 * while we touch the pending ccb list.
 		 */
-		oldspl = splcam();
 		LIST_REMOVE(&done_ccb->ccb_h, periph_links.le);
 		softc->outstanding_cmds--;
 		if (softc->outstanding_cmds == 0)
 			softc->flags |= DA_FLAG_WENT_IDLE;
-		splx(oldspl);
 
 		biodone(bp);
 		break;
@@ -1710,7 +1700,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 		 * operation.
 		 */
 		xpt_release_ccb(done_ccb);
-		cam_periph_unlock(periph);
+		cam_periph_unhold(periph);
 		return;
 	}
 	case DA_CCB_WAITING:
@@ -1833,7 +1823,9 @@ dagetcapacity(struct cam_periph *periph)
 	/* Do a read capacity */
 	rcap = (struct scsi_read_capacity_data *)malloc(sizeof(*rcaplong),
 							M_TEMP,
-							M_WAITOK);
+							M_NOWAIT);
+	if (rcap == NULL)
+		return (ENOMEM);
 		
 	ccb = cam_periph_getccb(periph, /*priority*/1);
 	scsi_read_capacity(&ccb->csio,
@@ -1959,27 +1951,22 @@ dasetgeom(struct cam_periph *periph, uint32_t block_len, uint64_t maxsector)
 static void
 dasendorderedtag(void *arg)
 {
-	struct da_softc *softc;
-	int s;
-	if (da_send_ordered) {
-		for (softc = SLIST_FIRST(&softc_list);
-		     softc != NULL;
-		     softc = SLIST_NEXT(softc, links)) {
-			s = splsoftcam();
-			if ((softc->ordered_tag_count == 0) 
-			 && ((softc->flags & DA_FLAG_WENT_IDLE) == 0)) {
-				softc->flags |= DA_FLAG_NEED_OTAG;
-			}
-			if (softc->outstanding_cmds > 0)
-				softc->flags &= ~DA_FLAG_WENT_IDLE;
+	struct da_softc *softc = arg;
 
-			softc->ordered_tag_count = 0;
-			splx(s);
+	if (da_send_ordered) {
+		if ((softc->ordered_tag_count == 0) 
+		 && ((softc->flags & DA_FLAG_WENT_IDLE) == 0)) {
+			softc->flags |= DA_FLAG_NEED_OTAG;
 		}
-		/* Queue us up again */
-		timeout(dasendorderedtag, NULL,
-			(da_default_timeout * hz) / DA_ORDEREDTAG_INTERVAL);
+		if (softc->outstanding_cmds > 0)
+			softc->flags &= ~DA_FLAG_WENT_IDLE;
+
+		softc->ordered_tag_count = 0;
 	}
+	/* Queue us up again */
+	callout_reset(&softc->sendordered_c,
+	    (DA_DEFAULT_TIMEOUT * hz) / DA_ORDEREDTAG_INTERVAL,
+	    dasendorderedtag, softc);
 }
 
 /*
diff --git a/sys/cam/scsi/scsi_low.c b/sys/cam/scsi/scsi_low.c
index 73aa505..296af9b 100644
--- a/sys/cam/scsi/scsi_low.c
+++ b/sys/cam/scsi/scsi_low.c
@@ -966,7 +966,7 @@ scsi_low_rescan_bus_cam(slp)
 	struct scsi_low_softc *slp;
 {
   	struct cam_path *path;
-	union ccb *ccb = xpt_alloc_ccb();
+	union ccb *ccb = xpt_alloc_ccb(NULL);
 	cam_status status;
 
 	bzero(ccb, sizeof(union ccb));
@@ -1343,7 +1343,7 @@ scsi_low_attach_cam(slp)
 	slp->sl_si.sim = cam_sim_alloc(scsi_low_scsi_action_cam,
 				scsi_low_poll_cam,
 				DEVPORT_DEVNAME(slp->sl_dev), slp,
-				DEVPORT_DEVUNIT(slp->sl_dev), 
+				DEVPORT_DEVUNIT(slp->sl_dev), &Giant,
 				slp->sl_openings, tagged_openings, devq);
 
 	if (slp->sl_si.sim == NULL) {
diff --git a/sys/cam/scsi/scsi_pass.c b/sys/cam/scsi/scsi_pass.c
index f08a7ed..62aaccf 100644
--- a/sys/cam/scsi/scsi_pass.c
+++ b/sys/cam/scsi/scsi_pass.c
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/cam_queue.h>
 #include <cam/cam_xpt_periph.h>
 #include <cam/cam_debug.h>
+#include <cam/cam_sim.h>
 
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_pass.h>
@@ -106,7 +107,7 @@ PERIPHDRIVER_DECLARE(pass, passdriver);
 
 static struct cdevsw pass_cdevsw = {
 	.d_version =	D_VERSION,
-	.d_flags =	D_NEEDGIANT,
+	.d_flags =	0,
 	.d_open =	passopen,
 	.d_close =	passclose,
 	.d_ioctl =	passioctl,
@@ -201,6 +202,7 @@ passasync(void *callback_arg, u_int32_t code,
 	  struct cam_path *path, void *arg)
 {
 	struct cam_periph *periph;
+	struct cam_sim *sim;
 
 	periph = (struct cam_periph *)callback_arg;
 
@@ -219,6 +221,7 @@ passasync(void *callback_arg, u_int32_t code,
 		 * this device and start the probe
 		 * process.
 		 */
+		sim = xpt_path_sim(cgd->ccb_h.path);
 		status = cam_periph_alloc(passregister, passoninvalidate,
 					  passcleanup, passstart, "pass",
 					  CAM_PERIPH_BIO, cgd->ccb_h.path,
@@ -293,9 +296,11 @@ passregister(struct cam_periph *periph, void *arg)
 			  DEVSTAT_PRIORITY_PASS);
 
 	/* Register the device */
+	mtx_unlock(periph->sim->mtx);
 	softc->dev = make_dev(&pass_cdevsw, unit2minor(periph->unit_number),
 			      UID_ROOT, GID_OPERATOR, 0600, "%s%d",
 			      periph->periph_name, periph->unit_number);
+	mtx_lock(periph->sim->mtx);
 	softc->dev->si_drv1 = periph;
 
 	/*
@@ -321,19 +326,20 @@ passopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	struct cam_periph *periph;
 	struct pass_softc *softc;
 	int error;
-	int s;
 
 	error = 0; /* default to no error */
 
 	periph = (struct cam_periph *)dev->si_drv1;
-	if (periph == NULL)
+	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
 		return (ENXIO);
 
+	cam_periph_lock(periph);
+
 	softc = (struct pass_softc *)periph->softc;
 
-	s = splsoftcam();
 	if (softc->flags & PASS_FLAG_INVALID) {
-		splx(s);
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return(ENXIO);
 	}
 
@@ -342,7 +348,8 @@ passopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	 */
 	error = securelevel_gt(td->td_ucred, 1);
 	if (error) {
-		splx(s);
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return(error);
 	}
 
@@ -350,7 +357,8 @@ passopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	 * Only allow read-write access.
 	 */
 	if (((flags & FWRITE) == 0) || ((flags & FREAD) == 0)) {
-		splx(s);
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return(EPERM);
 	}
 
@@ -359,21 +367,16 @@ passopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	 */
 	if ((flags & O_NONBLOCK) != 0) {
 		xpt_print(periph->path, "can't do nonblocking access\n");
-		splx(s);
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return(EINVAL);
 	}
 
-	if ((error = cam_periph_lock(periph, PRIBIO | PCATCH)) != 0) {
-		splx(s);
-		return (error);
-	}
-
-	splx(s);
-
 	if ((softc->flags & PASS_FLAG_OPEN) == 0) {
-		if (cam_periph_acquire(periph) != CAM_REQ_CMP)
-			return(ENXIO);
 		softc->flags |= PASS_FLAG_OPEN;
+	} else {
+		/* Device closes aren't symmertical, so fix up the refcount */
+		cam_periph_release(periph);
 	}
 
 	cam_periph_unlock(periph);
@@ -386,17 +389,14 @@ passclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 {
 	struct 	cam_periph *periph;
 	struct	pass_softc *softc;
-	int	error;
 
 	periph = (struct cam_periph *)dev->si_drv1;
 	if (periph == NULL)
 		return (ENXIO);	
 
-	softc = (struct pass_softc *)periph->softc;
-
-	if ((error = cam_periph_lock(periph, PRIBIO)) != 0)
-		return (error);
+	cam_periph_lock(periph);
 
+	softc = (struct pass_softc *)periph->softc;
 	softc->flags &= ~PASS_FLAG_OPEN;
 
 	cam_periph_unlock(periph);
@@ -409,18 +409,15 @@ static void
 passstart(struct cam_periph *periph, union ccb *start_ccb)
 {
 	struct pass_softc *softc;
-	int s;
 
 	softc = (struct pass_softc *)periph->softc;
 
 	switch (softc->state) {
 	case PASS_STATE_NORMAL:
-		s = splbio();
 		start_ccb->ccb_h.ccb_type = PASS_CCB_WAITING;			
 		SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
 				  periph_links.sle);
 		periph->immediate_priority = CAM_PRIORITY_NONE;
-		splx(s);
 		wakeup(&periph->ccb_list);
 		break;
 	}
@@ -454,6 +451,7 @@ passioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *t
 	if (periph == NULL)
 		return(ENXIO);
 
+	cam_periph_lock(periph);
 	softc = (struct pass_softc *)periph->softc;
 
 	error = 0;
@@ -492,7 +490,7 @@ passioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *t
 						inccb->ccb_h.pinfo.priority);
 			ccb_malloced = 0;
 		} else {
-			ccb = xpt_alloc_ccb();
+			ccb = xpt_alloc_ccb_nowait(periph->sim);
 
 			if (ccb != NULL)
 				xpt_setup_ccb(&ccb->ccb_h, periph->path,
@@ -520,6 +518,7 @@ passioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *t
 		break;
 	}
 
+	cam_periph_unlock(periph);
 	return(error);
 }
 
@@ -568,7 +567,14 @@ passsendccb(struct cam_periph *periph, union ccb *ccb, union ccb *inccb)
 
 		bzero(&mapinfo, sizeof(mapinfo));
 
+		/*
+		 * cam_periph_mapmem calls into proc and vm functions that can
+		 * sleep as well as trigger I/O, so we can't hold the lock.
+		 * Dropping it here is reasonably safe.
+		 */
+		cam_periph_unlock(periph);
 		error = cam_periph_mapmem(ccb, &mapinfo); 
+		cam_periph_lock(periph);
 
 		/*
 		 * cam_periph_mapmem returned an error, we can't continue.
diff --git a/sys/cam/scsi/scsi_pt.c b/sys/cam/scsi/scsi_pt.c
index 752ddbd..9f474d9 100644
--- a/sys/cam/scsi/scsi_pt.c
+++ b/sys/cam/scsi/scsi_pt.c
@@ -119,7 +119,7 @@ PERIPHDRIVER_DECLARE(pt, ptdriver);
 
 static struct cdevsw pt_cdevsw = {
 	.d_version =	D_VERSION,
-	.d_flags =	D_NEEDGIANT,
+	.d_flags =	0,
 	.d_open =	ptopen,
 	.d_close =	ptclose,
 	.d_read =	physread,
@@ -138,40 +138,30 @@ ptopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 {
 	struct cam_periph *periph;
 	struct pt_softc *softc;
-	int unit;
-	int error;
-	int s;
+	int error = 0;
 
-	unit = minor(dev);
 	periph = (struct cam_periph *)dev->si_drv1;
-	if (periph == NULL)
+	if (cam_periph_acquire(periph) != CAM_REQ_CMP)
 		return (ENXIO);	
 
 	softc = (struct pt_softc *)periph->softc;
 
-	s = splsoftcam();
+	cam_periph_lock(periph);
 	if (softc->flags & PT_FLAG_DEVICE_INVALID) {
-		splx(s);
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return(ENXIO);
 	}
 
-	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE,
-	    ("ptopen: dev=%s (unit %d)\n", devtoname(dev), unit));
-
-	if ((error = cam_periph_lock(periph, PRIBIO|PCATCH)) != 0) {
-		splx(s);
-		return (error); /* error code from tsleep */
+	if ((softc->flags & PT_FLAG_OPEN) == 0)
+		softc->flags |= PT_FLAG_OPEN;
+	else {
+		error = EBUSY;
+		cam_periph_release(periph);
 	}
 
-	splx(s);
-
-	if ((softc->flags & PT_FLAG_OPEN) == 0) {
-		if (cam_periph_acquire(periph) != CAM_REQ_CMP)
-			error = ENXIO;
-		else
-			softc->flags |= PT_FLAG_OPEN;
-	} else
-		error = EBUSY;
+	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE,
+	    ("ptopen: dev=%s\n", devtoname(dev)));
 
 	cam_periph_unlock(periph);
 	return (error);
@@ -182,7 +172,6 @@ ptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 {
 	struct	cam_periph *periph;
 	struct	pt_softc *softc;
-	int	error;
 
 	periph = (struct cam_periph *)dev->si_drv1;
 	if (periph == NULL)
@@ -190,8 +179,7 @@ ptclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 
 	softc = (struct pt_softc *)periph->softc;
 
-	if ((error = cam_periph_lock(periph, PRIBIO)) != 0)
-		return (error); /* error code from tsleep */
+	cam_periph_lock(periph);
 
 	softc->flags &= ~PT_FLAG_OPEN;
 	cam_periph_unlock(periph);
@@ -209,7 +197,6 @@ ptstrategy(struct bio *bp)
 {
 	struct cam_periph *periph;
 	struct pt_softc *softc;
-	int    s;
 	
 	periph = (struct cam_periph *)bp->bio_dev->si_drv1;
 	bp->bio_resid = bp->bio_bcount;
@@ -217,20 +204,14 @@ ptstrategy(struct bio *bp)
 		biofinish(bp, NULL, ENXIO);
 		return;
 	}
+	cam_periph_lock(periph);
 	softc = (struct pt_softc *)periph->softc;
 
 	/*
-	 * Mask interrupts so that the pack cannot be invalidated until
-	 * after we are in the queue.  Otherwise, we might not properly
-	 * clean up one of the buffers.
-	 */
-	s = splbio();
-	
-	/*
 	 * If the device has been made invalid, error out
 	 */
 	if ((softc->flags & PT_FLAG_DEVICE_INVALID)) {
-		splx(s);
+		cam_periph_unlock(periph);
 		biofinish(bp, NULL, ENXIO);
 		return;
 	}
@@ -240,12 +221,11 @@ ptstrategy(struct bio *bp)
 	 */
 	bioq_insert_tail(&softc->bio_queue, bp);
 
-	splx(s);
-	
 	/*
 	 * Schedule ourselves for performing the work.
 	 */
 	xpt_schedule(periph, /* XXX priority */1);
+	cam_periph_unlock(periph);
 
 	return;
 }
@@ -352,7 +332,6 @@ ptctor(struct cam_periph *periph, void *arg)
 static void
 ptoninvalidate(struct cam_periph *periph)
 {
-	int s;
 	struct pt_softc *softc;
 	struct ccb_setasync csa;
 
@@ -372,21 +351,12 @@ ptoninvalidate(struct cam_periph *periph)
 	softc->flags |= PT_FLAG_DEVICE_INVALID;
 
 	/*
-	 * Although the oninvalidate() routines are always called at
-	 * splsoftcam, we need to be at splbio() here to keep the buffer
-	 * queue from being modified while we traverse it.
-	 */
-	s = splbio();
-
-	/*
 	 * Return all queued I/O with ENXIO.
 	 * XXX Handle any transactions queued to the card
 	 *     with XPT_ABORT_CCB.
 	 */
 	bioq_flush(&softc->bio_queue, NULL, ENXIO);
 
-	splx(s);
-
 	xpt_print(periph->path, "lost device\n");
 }
 
@@ -445,10 +415,8 @@ ptasync(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg)
 	{
 		struct pt_softc *softc;
 		struct ccb_hdr *ccbh;
-		int s;
 
 		softc = (struct pt_softc *)periph->softc;
-		s = splsoftcam();
 		/*
 		 * Don't fail on the expected unit attention
 		 * that will occur.
@@ -456,7 +424,6 @@ ptasync(void *callback_arg, u_int32_t code, struct cam_path *path, void *arg)
 		softc->flags |= PT_FLAG_RETRY_UA;
 		LIST_FOREACH(ccbh, &softc->pending_ccbs, periph_links.le)
 			ccbh->ccb_state |= PT_CCB_RETRY_UA;
-		splx(s);
 	}
 	/* FALLTHROUGH */
 	default:
@@ -470,14 +437,12 @@ ptstart(struct cam_periph *periph, union ccb *start_ccb)
 {
 	struct pt_softc *softc;
 	struct bio *bp;
-	int s;
 
 	softc = (struct pt_softc *)periph->softc;
 
 	/*
 	 * See if there is a buf with work for us to do..
 	 */
-	s = splbio();
 	bp = bioq_first(&softc->bio_queue);
 	if (periph->immediate_priority <= periph->pinfo.priority) {
 		CAM_DEBUG_PRINT(CAM_DEBUG_SUBTRACE,
@@ -486,14 +451,10 @@ ptstart(struct cam_periph *periph, union ccb *start_ccb)
 		SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
 				  periph_links.sle);
 		periph->immediate_priority = CAM_PRIORITY_NONE;
-		splx(s);
 		wakeup(&periph->ccb_list);
 	} else if (bp == NULL) {
-		splx(s);
 		xpt_release_ccb(start_ccb);
 	} else {
-		int oldspl;
-
 		bioq_remove(&softc->bio_queue, bp);
 
 		devstat_start_transaction_bio(softc->device_stats, bp);
@@ -515,14 +476,11 @@ ptstart(struct cam_periph *periph, union ccb *start_ccb)
 		 * Block out any asyncronous callbacks
 		 * while we touch the pending ccb list.
 		 */
-		oldspl = splcam();
 		LIST_INSERT_HEAD(&softc->pending_ccbs, &start_ccb->ccb_h,
 				 periph_links.le);
-		splx(oldspl);
 
 		start_ccb->ccb_h.ccb_bp = bp;
 		bp = bioq_first(&softc->bio_queue);
-		splx(s);
 
 		xpt_action(start_ccb);
 		
@@ -546,12 +504,10 @@ ptdone(struct cam_periph *periph, union ccb *done_ccb)
 	case PT_CCB_BUFFER_IO_UA:
 	{
 		struct bio *bp;
-		int    oldspl;
 
 		bp = (struct bio *)done_ccb->ccb_h.ccb_bp;
 		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
 			int error;
-			int s;
 			int sf;
 			
 			if ((csio->ccb_h.ccb_state & PT_CCB_RETRY_UA) != 0)
@@ -568,8 +524,6 @@ ptdone(struct cam_periph *periph, union ccb *done_ccb)
 				return;
 			}
 			if (error != 0) {
-				s = splbio();
-
 				if (error == ENXIO) {
 					/*
 					 * Catastrophic error.  Mark our device
@@ -586,7 +540,6 @@ ptdone(struct cam_periph *periph, union ccb *done_ccb)
 				 * proper order should it attempt to recover.
 				 */
 				bioq_flush(&softc->bio_queue, NULL, EIO);
-				splx(s);
 				bp->bio_error = error;
 				bp->bio_resid = bp->bio_bcount;
 				bp->bio_flags |= BIO_ERROR;
@@ -614,9 +567,7 @@ ptdone(struct cam_periph *periph, union ccb *done_ccb)
 		 * Block out any asyncronous callbacks
 		 * while we touch the pending ccb list.
 		 */
-		oldspl = splcam();
 		LIST_REMOVE(&done_ccb->ccb_h, periph_links.le);
-		splx(oldspl);
 
 		biofinish(bp, softc->device_stats, 0);
 		break;
@@ -647,7 +598,7 @@ ptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 {
 	struct cam_periph *periph;
 	struct pt_softc *softc;
-	int error;
+	int error = 0;
 
 	periph = (struct cam_periph *)dev->si_drv1;
 	if (periph == NULL)
@@ -655,9 +606,7 @@ ptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 
 	softc = (struct pt_softc *)periph->softc;
 
-	if ((error = cam_periph_lock(periph, PRIBIO|PCATCH)) != 0) {
-		return (error); /* error code from tsleep */
-	}	
+	cam_periph_lock(periph);
 
 	switch(cmd) {
 	case PTIOCGETTIMEOUT:
@@ -667,20 +616,14 @@ ptioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *td)
 			*(int *)addr = 0;
 		break;
 	case PTIOCSETTIMEOUT:
-	{
-		int s;
-
 		if (*(int *)addr < 1) {
 			error = EINVAL;
 			break;
 		}
 
-		s = splsoftcam();
 		softc->io_timeout = *(int *)addr * 1000;
-		splx(s);
 
 		break;
-	}
 	default:
 		error = cam_periph_ioctl(periph, cmd, addr, pterror);
 		break;
diff --git a/sys/cam/scsi/scsi_sa.c b/sys/cam/scsi/scsi_sa.c
index 013e421..170d35c 100644
--- a/sys/cam/scsi/scsi_sa.c
+++ b/sys/cam/scsi/scsi_sa.c
@@ -447,37 +447,33 @@ saopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	struct sa_softc *softc;
 	int unit;
 	int error;
-	int s;
 
 	unit = SAUNIT(dev);
 
-	s = splsoftcam();
 	periph = (struct cam_periph *)dev->si_drv1;
-	if (periph == NULL) {
-		(void) splx(s);
-		return (ENXIO);	
+	if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
+		return (ENXIO);
 	}
+
+	cam_periph_lock(periph);
+
 	softc = (struct sa_softc *)periph->softc;
-	if ((error = cam_periph_lock(periph, PRIBIO|PCATCH)) != 0) {
-		splx(s);
-		return (error);
-	}
-	splx(s);
 
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE|CAM_DEBUG_INFO,
 	    ("saopen(%d): dev=0x%x softc=0x%x\n", unit, unit, softc->flags));
 
-	if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
-		cam_periph_unlock(periph);
-		return (ENXIO);
-	}
-
 	if (SA_IS_CTRL(dev)) {
 		softc->ctrl_mode = 1;
 		cam_periph_unlock(periph);
 		return (0);
 	}
 
+	if ((error = cam_periph_hold(periph, PRIBIO|PCATCH)) != 0) {
+		cam_periph_unlock(periph);
+		cam_periph_release(periph);
+		return (error);
+	}
+
 	if (softc->flags & SA_FLAG_OPEN) {
 		error = EBUSY;
 	} else if (softc->flags & SA_FLAG_INVALID) {
@@ -499,17 +495,23 @@ saopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 		if (error && (flags & O_NONBLOCK)) {
 			softc->flags |= SA_FLAG_OPEN;
 			softc->open_pending_mount = 1;
+			cam_periph_unhold(periph);
 			cam_periph_unlock(periph);
 			return (0);
 		}
 	}
 
 	if (error) {
+		cam_periph_unhold(periph);
+		cam_periph_unlock(periph);
 		cam_periph_release(periph);
-	} else {
-		saprevent(periph, PR_PREVENT);
-		softc->flags |= SA_FLAG_OPEN;
+		return (error);
 	}
+
+	saprevent(periph, PR_PREVENT);
+	softc->flags |= SA_FLAG_OPEN;
+
+	cam_periph_unhold(periph);
 	cam_periph_unlock(periph);
 	return (error);
 }
@@ -528,32 +530,35 @@ saclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 	if (periph == NULL)
 		return (ENXIO);	
 
+	cam_periph_lock(periph);
+
 	softc = (struct sa_softc *)periph->softc;
 
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE|CAM_DEBUG_INFO,
 	    ("saclose(%d): dev=0x%x softc=0x%x\n", unit, unit, softc->flags));
 
 
-	if ((error = cam_periph_lock(periph, PRIBIO)) != 0) {
-		return (error);
-	}
-
 	softc->open_rdonly = 0; 
 	if (SA_IS_CTRL(dev)) {
 		softc->ctrl_mode = 0;
-		cam_periph_release(periph);
 		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return (0);
 	}
 
 	if (softc->open_pending_mount) {
 		softc->flags &= ~SA_FLAG_OPEN;
 		softc->open_pending_mount = 0; 
-		cam_periph_release(periph);
 		cam_periph_unlock(periph);
+		cam_periph_release(periph);
 		return (0);
 	}
 
+	if ((error = cam_periph_hold(periph, PRIBIO)) != 0) {
+		cam_periph_unlock(periph);
+		return (error);
+	}
+
 	/*
 	 * Were we writing the tape?
 	 */
@@ -661,6 +666,7 @@ saclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 	if ((softc->flags & SA_FLAG_TAPE_MOUNTED) == 0)
 		sareservereleaseunit(periph, FALSE);
 
+	cam_periph_unhold(periph);
 	cam_periph_unlock(periph);
 	cam_periph_release(periph);
 
@@ -677,7 +683,6 @@ sastrategy(struct bio *bp)
 {
 	struct cam_periph *periph;
 	struct sa_softc *softc;
-	int    s;
 	
 	bp->bio_resid = bp->bio_bcount;
 	if (SA_IS_CTRL(bp->bio_dev)) {
@@ -689,18 +694,18 @@ sastrategy(struct bio *bp)
 		biofinish(bp, NULL, ENXIO);
 		return;
 	}
-	softc = (struct sa_softc *)periph->softc;
+	cam_periph_lock(periph);
 
-	s = splsoftcam();
+	softc = (struct sa_softc *)periph->softc;
 
 	if (softc->flags & SA_FLAG_INVALID) {
-		splx(s);
+		cam_periph_unlock(periph);
 		biofinish(bp, NULL, ENXIO);
 		return;
 	}
 
 	if (softc->flags & SA_FLAG_TAPE_FROZEN) {
-		splx(s);
+		cam_periph_unlock(periph);
 		biofinish(bp, NULL, EPERM);
 		return;
 	}
@@ -711,16 +716,15 @@ sastrategy(struct bio *bp)
 	 * file descriptor.
 	 */
 	if (bp->bio_cmd == BIO_WRITE && softc->open_rdonly) {
-		splx(s);
+		cam_periph_unlock(periph);
 		biofinish(bp, NULL, EBADF);
 		return;
 	}
 
-	splx(s);
-
 	if (softc->open_pending_mount) {
 		int error = samount(periph, 0, bp->bio_dev);
 		if (error) {
+			cam_periph_unlock(periph);
 			biofinish(bp, NULL, ENXIO);
 			return;
 		}
@@ -733,6 +737,7 @@ sastrategy(struct bio *bp)
 	 * If it's a null transfer, return immediately
 	 */
 	if (bp->bio_bcount == 0) {
+		cam_periph_unlock(periph);
 		biodone(bp);
 		return;
 	}
@@ -750,6 +755,7 @@ sastrategy(struct bio *bp)
 			xpt_print(periph->path, "Invalid request.  Fixed block "
 			    "device requests must be a multiple of %d bytes\n",
 			    softc->min_blk);
+			cam_periph_unlock(periph);
 			biofinish(bp, NULL, EINVAL);
 			return;
 		}
@@ -765,18 +771,12 @@ sastrategy(struct bio *bp)
 		}
 		printf("between %d and %d bytes\n", softc->min_blk,
 		    softc->max_blk);
+		cam_periph_unlock(periph);
 		biofinish(bp, NULL, EINVAL);
 		return;
         }
 	
 	/*
-	 * Mask interrupts so that the device cannot be invalidated until
-	 * after we are in the queue.  Otherwise, we might not properly
-	 * clean up one of the buffers.
-	 */
-	s = splbio();
-	
-	/*
 	 * Place it at the end of the queue.
 	 */
 	bioq_insert_tail(&softc->bio_queue, bp);
@@ -791,12 +791,12 @@ sastrategy(struct bio *bp)
 		CAM_DEBUG(periph->path, CAM_DEBUG_INFO,
 		    ("sastrategy: queue count now %d\n", softc->queue_count));
 	}
-	splx(s);
 	
 	/*
 	 * Schedule ourselves for performing the work.
 	 */
 	xpt_schedule(periph, 1);
+	cam_periph_unlock(periph);
 
 	return;
 }
@@ -819,7 +819,6 @@ saioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 	struct sa_softc *softc;
 	scsi_space_code spaceop;
 	int didlockperiph = 0;
-	int s;
 	int mode;
 	int error = 0;
 
@@ -831,6 +830,7 @@ saioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 	if (periph == NULL)
 		return (ENXIO);	
 
+	cam_periph_lock(periph);
 	softc = (struct sa_softc *)periph->softc;
 
 	/*
@@ -856,13 +856,10 @@ saioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 			 * other thread that has this device open to do
 			 * an MTIOCERRSTAT that would clear latched status.
 			 */
-			s = splsoftcam();
 			if ((periph->flags & CAM_PERIPH_LOCKED) == 0) {
-				error = cam_periph_lock(periph, PRIBIO|PCATCH);
-				if (error != 0) {
-					splx(s);
+				error = cam_periph_hold(periph, PRIBIO|PCATCH);
+				 if (error != 0)
 					return (error);
-				}
 				didlockperiph = 1;
 			}
 			break;
@@ -895,12 +892,9 @@ saioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 			 * than at open time because we are sharing writable
 			 * access to data structures.
 			 */
-			s = splsoftcam();
-			error = cam_periph_lock(periph, PRIBIO|PCATCH);
-			if (error != 0) {
-				splx(s);
+			error = cam_periph_hold(periph, PRIBIO|PCATCH);
+			if (error != 0)
 				return (error);
-			}
 			didlockperiph = 1;
 			break;
 
@@ -1327,8 +1321,9 @@ saioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 		}
 	}
 	if (didlockperiph) {
-		cam_periph_unlock(periph);
+		cam_periph_unhold(periph);
 	}
+	cam_periph_unlock(periph);
 	return (error);
 }
 
@@ -1371,7 +1366,6 @@ saoninvalidate(struct cam_periph *periph)
 {
 	struct sa_softc *softc;
 	struct ccb_setasync csa;
-	int s;
 
 	softc = (struct sa_softc *)periph->softc;
 
@@ -1389,20 +1383,12 @@ saoninvalidate(struct cam_periph *periph)
 	softc->flags |= SA_FLAG_INVALID;
 
 	/*
-	 * Although the oninvalidate() routines are always called at
-	 * splsoftcam, we need to be at splbio() here to keep the buffer
-	 * queue from being modified while we traverse it.
-	 */
-	s = splbio();
-
-	/*
 	 * Return all queued I/O with ENXIO.
 	 * XXX Handle any transactions queued to the card
 	 *     with XPT_ABORT_CCB.
 	 */
 	bioq_flush(&softc->bio_queue, NULL, ENXIO);
 	softc->queue_count = 0;
-	splx(s);
 
 	xpt_print(periph->path, "lost device\n");
 
@@ -1609,12 +1595,10 @@ sastart(struct cam_periph *periph, union ccb *start_ccb)
 	{
 		/* Pull a buffer from the queue and get going on it */		
 		struct bio *bp;
-		int s;
 
 		/*
 		 * See if there is a buf with work for us to do..
 		 */
-		s = splbio();
 		bp = bioq_first(&softc->bio_queue);
 		if (periph->immediate_priority <= periph->pinfo.priority) {
 			CAM_DEBUG_PRINT(CAM_DEBUG_SUBTRACE,
@@ -1623,10 +1607,8 @@ sastart(struct cam_periph *periph, union ccb *start_ccb)
 			SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
 					  periph_links.sle);
 			periph->immediate_priority = CAM_PRIORITY_NONE;
-			splx(s);
 			wakeup(&periph->ccb_list);
 		} else if (bp == NULL) {
-			splx(s);
 			xpt_release_ccb(start_ccb);
 		} else if ((softc->flags & SA_FLAG_ERR_PENDING) != 0) {
 			struct bio *done_bp;
@@ -1669,7 +1651,6 @@ again:
 			    "%d more buffers queued up\n",
 			    (softc->flags & SA_FLAG_ERR_PENDING),
 			    (bp != NULL)? "not " : " ", softc->queue_count));
-			splx(s);
 			xpt_release_ccb(start_ccb);
 			biodone(done_bp);
 		} else {
@@ -1689,7 +1670,6 @@ again:
 					bp->bio_error = EIO;
 					xpt_print(periph->path, "zero blocksize"
 					    " for FIXED length writes?\n");
-					splx(s);
 					biodone(bp);
 					break;
 				}
@@ -1740,7 +1720,6 @@ again:
 			Set_CCB_Type(start_ccb, SA_CCB_BUFFER_IO);
 			start_ccb->ccb_h.ccb_bp = bp;
 			bp = bioq_first(&softc->bio_queue);
-			splx(s);
 			xpt_action(start_ccb);
 		}
 		
@@ -1785,7 +1764,6 @@ sadone(struct cam_periph *periph, union ccb *done_ccb)
 		}
 
 		if (error == EIO) {
-			int s;			
 
 			/*
 			 * Catastrophic error. Mark the tape as frozen
@@ -1798,10 +1776,8 @@ sadone(struct cam_periph *periph, union ccb *done_ccb)
 			 *
 			 */
 
-			s = splbio();
 			softc->flags |= SA_FLAG_TAPE_FROZEN;
 			bioq_flush(&softc->bio_queue, NULL, EIO);
-			splx(s);
 		}
 		if (error != 0) {
 			bp->bio_resid = bp->bio_bcount;
diff --git a/sys/cam/scsi/scsi_ses.c b/sys/cam/scsi/scsi_ses.c
index 2af83ce..8e0b572 100644
--- a/sys/cam/scsi/scsi_ses.c
+++ b/sys/cam/scsi/scsi_ses.c
@@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/cam_periph.h>
 #include <cam/cam_xpt_periph.h>
 #include <cam/cam_debug.h>
+#include <cam/cam_sim.h>
 
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_message.h>
@@ -182,7 +183,7 @@ static struct cdevsw ses_cdevsw = {
 	.d_close =	sesclose,
 	.d_ioctl =	sesioctl,
 	.d_name =	"ses",
-	.d_flags =	D_NEEDGIANT,
+	.d_flags =	0,
 };
 
 static void
@@ -363,9 +364,11 @@ sesregister(struct cam_periph *periph, void *arg)
 		return (CAM_REQ_CMP_ERR);
 	}
 
+	cam_periph_unlock(periph);
 	softc->ses_dev = make_dev(&ses_cdevsw, unit2minor(periph->unit_number),
 	    UID_ROOT, GID_OPERATOR, 0600, "%s%d",
 	    periph->periph_name, periph->unit_number);
+	cam_periph_lock(periph);
 	softc->ses_dev->si_drv1 = periph;
 
 	/*
@@ -409,25 +412,20 @@ sesopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 {
 	struct cam_periph *periph;
 	struct ses_softc *softc;
-	int error, s;
+	int error = 0;
 
-	s = splsoftcam();
 	periph = (struct cam_periph *)dev->si_drv1;
 	if (periph == NULL) {
-		splx(s);
 		return (ENXIO);
 	}
-	if ((error = cam_periph_lock(periph, PRIBIO | PCATCH)) != 0) {
-		splx(s);
-		return (error);
-	}
-	splx(s);
 
 	if (cam_periph_acquire(periph) != CAM_REQ_CMP) {
 		cam_periph_unlock(periph);
 		return (ENXIO);
 	}
 
+	cam_periph_lock(periph);
+
 	softc = (struct ses_softc *)periph->softc;
 
 	if (softc->ses_flags & SES_FLAG_INVALID) {
@@ -453,10 +451,10 @@ sesopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	}
 
 out:
+	cam_periph_unlock(periph);
 	if (error) {
 		cam_periph_release(periph);
 	}
-	cam_periph_unlock(periph);
 	return (error);
 }
 
@@ -473,11 +471,9 @@ sesclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 	if (periph == NULL)
 		return (ENXIO);
 
-	softc = (struct ses_softc *)periph->softc;
-
-	if ((error = cam_periph_lock(periph, PRIBIO)) != 0)
-		return (error);
+	cam_periph_lock(periph);
 
+	softc = (struct ses_softc *)periph->softc;
 	softc->ses_flags &= ~SES_FLAG_OPEN;
 
 	cam_periph_unlock(periph);
@@ -489,13 +485,11 @@ sesclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 static void
 sesstart(struct cam_periph *p, union ccb *sccb)
 {
-	int s = splbio();
 	if (p->immediate_priority <= p->pinfo.priority) {
 		SLIST_INSERT_HEAD(&p->ccb_list, &sccb->ccb_h, periph_links.sle);
 		p->immediate_priority = CAM_PRIORITY_NONE;
 		wakeup(&p->ccb_list);
 	}
-	splx(s);
 }
 
 static void
@@ -539,14 +533,17 @@ sesioctl(struct cdev *dev, u_long cmd, caddr_t arg_addr, int flag, struct thread
 
 	CAM_DEBUG(periph->path, CAM_DEBUG_TRACE, ("entering sesioctl\n"));
 
+	cam_periph_lock(periph);
 	ssc = (struct ses_softc *)periph->softc;
 
 	/*
 	 * Now check to see whether we're initialized or not.
 	 */
 	if ((ssc->ses_flags & SES_FLAG_INITIALIZED) == 0) {
+		cam_periph_unlock(periph);
 		return (ENXIO);
 	}
+	cam_periph_lock(periph);
 
 	error = 0;
 
@@ -576,22 +573,34 @@ sesioctl(struct cdev *dev, u_long cmd, caddr_t arg_addr, int flag, struct thread
 		break;
 		
 	case SESIOC_GETOBJMAP:
+		/*
+		 * XXX Dropping the lock while copying multiple segments is
+		 * bogus.
+		 */
+		cam_periph_lock(periph);
 		for (uobj = addr, i = 0; i != ssc->ses_nobjects; i++, uobj++) {
 			obj.obj_id = i;
 			obj.subencid = ssc->ses_objmap[i].subenclosure;
 			obj.object_type = ssc->ses_objmap[i].enctype;
+			cam_periph_lock(periph);
 			error = copyout(&obj, uobj, sizeof (ses_object));
+			cam_periph_lock(periph);
 			if (error) {
 				break;
 			}
 		}
+		cam_periph_lock(periph);
 		break;
 
 	case SESIOC_GETENCSTAT:
+		cam_periph_lock(periph);
 		error = (*ssc->ses_vec.get_encstat)(ssc, 1);
-		if (error)
+		if (error) {
+			cam_periph_unlock(periph);
 			break;
+		}
 		tmp = ssc->ses_encstat & ~ENCI_SVALID;
+		cam_periph_unlock(periph);
 		error = copyout(&tmp, addr, sizeof (ses_encstat));
 		ssc->ses_encstat = tmp;
 		break;
@@ -600,7 +609,9 @@ sesioctl(struct cdev *dev, u_long cmd, caddr_t arg_addr, int flag, struct thread
 		error = copyin(addr, &tmp, sizeof (ses_encstat));
 		if (error)
 			break;
+		cam_periph_lock(periph);
 		error = (*ssc->ses_vec.set_encstat)(ssc, tmp, 1);
+		cam_periph_unlock(periph);
 		break;
 
 	case SESIOC_GETOBJSTAT:
@@ -611,7 +622,9 @@ sesioctl(struct cdev *dev, u_long cmd, caddr_t arg_addr, int flag, struct thread
 			error = EINVAL;
 			break;
 		}
+		cam_periph_lock(periph);
 		error = (*ssc->ses_vec.get_objstat)(ssc, &objs, 1);
+		cam_periph_unlock(periph);
 		if (error)
 			break;
 		error = copyout(&objs, addr, sizeof (ses_objstat));
@@ -630,7 +643,9 @@ sesioctl(struct cdev *dev, u_long cmd, caddr_t arg_addr, int flag, struct thread
 			error = EINVAL;
 			break;
 		}
+		cam_periph_lock(periph);
 		error = (*ssc->ses_vec.set_objstat)(ssc, &objs, 1);
+		cam_periph_unlock(periph);
 
 		/*
 		 * Always (for now) invalidate entry.
@@ -640,11 +655,15 @@ sesioctl(struct cdev *dev, u_long cmd, caddr_t arg_addr, int flag, struct thread
 
 	case SESIOC_INIT:
 
+		cam_periph_lock(periph);
 		error = (*ssc->ses_vec.init_enc)(ssc);
+		cam_periph_unlock(periph);
 		break;
 
 	default:
+		cam_periph_lock(periph);
 		error = cam_periph_ioctl(periph, cmd, arg_addr, seserror);
+		cam_periph_unlock(periph);
 		break;
 	}
 	return (error);
diff --git a/sys/cam/scsi/scsi_sg.c b/sys/cam/scsi/scsi_sg.c
index 38942eb..7645abd 100644
--- a/sys/cam/scsi/scsi_sg.c
+++ b/sys/cam/scsi/scsi_sg.c
@@ -319,11 +319,13 @@ sgregister(struct cam_periph *periph, void *arg)
 			DEVSTAT_PRIORITY_PASS);
 
 	/* Register the device */
+	cam_periph_unlock(periph);
 	softc->dev = make_dev(&sg_cdevsw, unit2minor(periph->unit_number),
 			      UID_ROOT, GID_OPERATOR, 0600, "%s%d",
 			      periph->periph_name, periph->unit_number);
 	softc->devalias = make_dev_alias(softc->dev, "sg%c",
 					 'a' + periph->unit_number);
+	cam_periph_lock(periph);
 	softc->dev->si_drv1 = periph;
 
 	/*
@@ -410,10 +412,6 @@ sgopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	if (periph == NULL)
 		return (ENXIO);
 
-	softc = (struct sg_softc *)periph->softc;
-	if (softc->flags & SG_FLAG_INVALID)
-		return (ENXIO);
-
 	/*
 	 * Don't allow access when we're running at a high securelevel.
 	 */
@@ -421,13 +419,19 @@ sgopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 	if (error)
 		return (error);
 
-	if ((error = cam_periph_lock(periph, PRIBIO | PCATCH)) != 0)
-		return (error);
+	cam_periph_lock(periph);
+
+	softc = (struct sg_softc *)periph->softc;
+	if (softc->flags & SG_FLAG_INVALID) {
+		cam_periph_unlock(periph);
+		return (ENXIO);
+	}
 
 	if ((softc->flags & SG_FLAG_OPEN) == 0) {
-		if (cam_periph_acquire(periph) != CAM_REQ_CMP)
-			return (ENXIO);
 		softc->flags |= SG_FLAG_OPEN;
+	} else {
+		/* Device closes aren't symmetrical, fix up the refcount. */
+		cam_periph_release(periph);
 	}
 
 	cam_periph_unlock(periph);
@@ -440,17 +444,14 @@ sgclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 {
 	struct cam_periph *periph;
 	struct sg_softc *softc;
-	int error;
 
 	periph = (struct cam_periph *)dev->si_drv1;
 	if (periph == NULL)
 		return (ENXIO);
 
-	softc = (struct sg_softc *)periph->softc;
-
-	if ((error = cam_periph_lock(periph, PRIBIO)) != 0)
-		return (error);
+	cam_periph_lock(periph);
 
+	softc = (struct sg_softc *)periph->softc;
 	softc->flags &= ~SG_FLAG_OPEN;
 
 	cam_periph_unlock(periph);
@@ -473,6 +474,8 @@ sgioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 	if (periph == NULL)
 		return (ENXIO);
 
+	cam_periph_lock(periph);
+
 	softc = (struct sg_softc *)periph->softc;
 	error = 0;
 
@@ -669,6 +672,7 @@ sgioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 		break;
 	}
 
+	cam_periph_unlock(periph);
 	return (error);
 }
 
@@ -686,7 +690,6 @@ sgwrite(struct cdev *dev, struct uio *uio, int ioflag)
 	int error = 0, cdb_len, buf_len, dir;
 
 	periph = dev->si_drv1;
-	sc = periph->softc;
 	rdwr = malloc(sizeof(*rdwr), M_DEVBUF, M_WAITOK | M_ZERO);
 	hdr = &rdwr->hdr.hdr;
 
@@ -699,12 +702,11 @@ sgwrite(struct cdev *dev, struct uio *uio, int ioflag)
 	if (error)
 		goto out_hdr;
 
-	ccb = xpt_alloc_ccb();
+	ccb = xpt_alloc_ccb(periph->sim);
 	if (ccb == NULL) {
 		error = ENOMEM;
 		goto out_hdr;
 	}
-	xpt_setup_ccb(&ccb->ccb_h, periph->path, /*priority*/5);
 	csio = &ccb->csio;
 
 	/*
@@ -751,6 +753,9 @@ sgwrite(struct cdev *dev, struct uio *uio, int ioflag)
 		dir = CAM_DIR_NONE;
 	}
 
+	cam_periph_lock(periph);
+	sc = periph->softc;
+	xpt_setup_ccb(&ccb->ccb_h, periph->path, /*priority*/5);
 	cam_fill_csio(csio,
 		      /*retries*/1,
 		      sgdone,
@@ -774,7 +779,9 @@ sgwrite(struct cdev *dev, struct uio *uio, int ioflag)
 	ccb->ccb_h.ccb_rdwr = rdwr;
 	ccb->ccb_h.ccb_type = SG_CCB_RDWR_IO;
 	TAILQ_INSERT_TAIL(&sc->rdwr_done, rdwr, rdwr_link);
-	return (sgsendrdwr(periph, ccb));
+	error = sgsendrdwr(periph, ccb);
+	cam_periph_unlock(periph);
+	return (error);
 
 out_buf:
 	free(buf, M_DEVBUF);
@@ -797,7 +804,6 @@ sgread(struct cdev *dev, struct uio *uio, int ioflag)
 	int error, pack_len, reply_len, pack_id;
 
 	periph = dev->si_drv1;
-	sc = periph->softc;
 
 	/* XXX The pack len field needs to be updated and written out instead
 	 * of discarded.  Not sure how to do that.
@@ -811,17 +817,20 @@ sgread(struct cdev *dev, struct uio *uio, int ioflag)
 		return (error);
 	uio->uio_rw = UIO_READ;
 
+	cam_periph_lock(periph);
+	sc = periph->softc;
 search:
 	TAILQ_FOREACH(rdwr, &sc->rdwr_done, rdwr_link) {
 		if (rdwr->tag == pack_id)
 			break;
 	}
 	if ((rdwr == NULL) || (rdwr->state != SG_RDWR_DONE)) {
-		if (tsleep(rdwr, PCATCH, "sgread", 0) == ERESTART)
+		if (msleep(rdwr, periph->sim->mtx, PCATCH, "sgread", 0) == ERESTART)
 			return (EAGAIN);
 		goto search;
 	}
 	TAILQ_REMOVE(&sc->rdwr_done, rdwr, rdwr_link);
+	cam_periph_unlock(periph);
 
 	hdr = &rdwr->hdr.hdr;
 	csio = &rdwr->ccb->csio;
@@ -865,7 +874,9 @@ search:
 	if ((error == 0) && (hdr->result == 0))
 		error = uiomove(rdwr->buf, rdwr->buf_len, uio);
 
+	cam_periph_lock(periph);
 	xpt_free_ccb(rdwr->ccb);
+	cam_periph_unlock(periph);
 	free(rdwr->buf, M_DEVBUF);
 	free(rdwr, M_DEVBUF);
 	return (error);
@@ -882,7 +893,15 @@ sgsendccb(struct cam_periph *periph, union ccb *ccb)
 	if (((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE)
 	    && (ccb->csio.data_ptr != NULL)) {
 		bzero(&mapinfo, sizeof(mapinfo));
+
+		/*
+		 * cam_periph_mapmem calls into proc and vm functions that can
+		 * sleep as well as trigger I/O, so we can't hold the lock.
+		 * Dropping it here is reasonably safe.
+		 */
+		cam_periph_unlock(periph);
 		error = cam_periph_mapmem(ccb, &mapinfo);
+		cam_periph_lock(periph);
 		if (error)
 			return (error);
 		need_unmap = 1;
diff --git a/sys/cam/scsi/scsi_targ_bh.c b/sys/cam/scsi/scsi_targ_bh.c
index b61152e..984baf4 100644
--- a/sys/cam/scsi/scsi_targ_bh.c
+++ b/sys/cam/scsi/scsi_targ_bh.c
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/cam_queue.h>
 #include <cam/cam_xpt_periph.h>
 #include <cam/cam_debug.h>
+#include <cam/cam_sim.h>
 
 #include <cam/scsi/scsi_all.h>
 #include <cam/scsi/scsi_message.h>
@@ -447,7 +448,7 @@ targbhdtor(struct cam_periph *periph)
 		/* FALLTHROUGH */
 	default:
 		/* XXX Wait for callback of targbhdislun() */
-		tsleep(softc, PRIBIO, "targbh", hz/2);
+		msleep(softc, periph->sim->mtx, PRIBIO, "targbh", hz/2);
 		free(softc, M_SCSIBH);
 		break;
 	}
@@ -462,27 +463,22 @@ targbhstart(struct cam_periph *periph, union ccb *start_ccb)
 	struct targbh_cmd_desc *desc;
 	struct ccb_scsiio *csio;
 	ccb_flags flags;
-	int    s;
 
 	softc = (struct targbh_softc *)periph->softc;
 	
-	s = splbio();
 	ccbh = TAILQ_FIRST(&softc->work_queue);
 	if (periph->immediate_priority <= periph->pinfo.priority) {
 		start_ccb->ccb_h.ccb_type = TARGBH_CCB_WAITING;			
 		SLIST_INSERT_HEAD(&periph->ccb_list, &start_ccb->ccb_h,
 				  periph_links.sle);
 		periph->immediate_priority = CAM_PRIORITY_NONE;
-		splx(s);
 		wakeup(&periph->ccb_list);
 	} else if (ccbh == NULL) {
-		splx(s);
 		xpt_release_ccb(start_ccb);	
 	} else {
 		TAILQ_REMOVE(&softc->work_queue, ccbh, periph_links.tqe);
 		TAILQ_INSERT_HEAD(&softc->pending_queue, ccbh,
 				  periph_links.tqe);
-		splx(s);	
 		atio = (struct ccb_accept_tio*)ccbh;
 		desc = (struct targbh_cmd_desc *)atio->ccb_h.ccb_descr;
 
@@ -543,9 +539,7 @@ targbhstart(struct cam_periph *periph, union ccb *start_ccb)
 					 /*getcount_only*/0); 
 			atio->ccb_h.status &= ~CAM_DEV_QFRZN;
 		}
-		s = splbio();
 		ccbh = TAILQ_FIRST(&softc->work_queue);
-		splx(s);
 	}
 	if (ccbh != NULL)
 		xpt_schedule(periph, /*priority*/1);
diff --git a/sys/cam/scsi/scsi_target.c b/sys/cam/scsi/scsi_target.c
index 670fb0e..c6086f9 100644
--- a/sys/cam/scsi/scsi_target.c
+++ b/sys/cam/scsi/scsi_target.c
@@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$");
 #include <cam/cam_ccb.h>
 #include <cam/cam_periph.h>
 #include <cam/cam_xpt_periph.h>
+#include <cam/cam_sim.h>
 #include <cam/scsi/scsi_targetio.h>
 
 /* Transaction information attached to each CCB sent by the user */
@@ -160,7 +161,11 @@ PERIPHDRIVER_DECLARE(targ, targdriver);
 
 static MALLOC_DEFINE(M_TARG, "TARG", "TARG data");
 
-/* Create softc and initialize it. Only one proc can open each targ device. */
+/*
+ * Create softc and initialize it. Only one proc can open each targ device.
+ * There is no locking here because a periph doesn't get created until an
+ * ioctl is issued to do so, and that can't happen until this method returns.
+ */
 static int
 targopen(struct cdev *dev, int flags, int fmt, struct thread *td)
 {
@@ -199,9 +204,24 @@ static int
 targclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 {
 	struct targ_softc     *softc;
+	struct cam_periph     *periph;
 	int    error;
 
 	softc = (struct targ_softc *)dev->si_drv1;
+	if ((softc->periph == NULL) ||
+	    (softc->state & TARG_STATE_LUN_ENABLED) == 0) {
+		destroy_dev(dev);
+		FREE(softc, M_TARG);
+		return (0);
+	}
+
+	/*
+	 * Acquire a hold on the periph so that it doesn't go away before
+	 * we are ready at the end of the function.
+	 */
+	periph = softc->periph;
+	cam_periph_acquire(periph);
+	cam_periph_lock(periph);
 	error = targdisable(softc);
 	if (error == CAM_REQ_CMP) {
 		dev->si_drv1 = 0;
@@ -212,6 +232,9 @@ targclose(struct cdev *dev, int flag, int fmt, struct thread *td)
 		destroy_dev(dev);
 		FREE(softc, M_TARG);
 	}
+	cam_periph_unlock(periph);
+	cam_periph_release(periph);
+
 	return (error);
 }
 
@@ -229,44 +252,56 @@ targioctl(struct cdev *dev, u_long cmd, caddr_t addr, int flag, struct thread *t
 	{
 		struct ioc_enable_lun	*new_lun;
 		struct cam_path		*path;
+		struct cam_sim		*sim;
 
 		new_lun = (struct ioc_enable_lun *)addr;
-		status = xpt_create_path(&path, /*periph*/NULL,
-					 new_lun->path_id,
-					 new_lun->target_id,
-					 new_lun->lun_id);
+		status = xpt_create_path_unlocked(&path, /*periph*/NULL,
+						  new_lun->path_id,
+						  new_lun->target_id,
+						  new_lun->lun_id);
 		if (status != CAM_REQ_CMP) {
 			printf("Couldn't create path, status %#x\n", status);
 			break;
 		}
+		sim = xpt_path_sim(path);
+		mtx_lock(sim->mtx);
 		status = targenable(softc, path, new_lun->grp6_len,
 				    new_lun->grp7_len);
 		xpt_free_path(path);
+		mtx_unlock(sim->mtx);
 		break;
 	}
 	case TARGIOCDISABLE:
+		if (softc->periph == NULL) {
+			status = CAM_DEV_NOT_THERE;
+			break;
+		}
+		cam_periph_lock(softc->periph);
 		status = targdisable(softc);
+		cam_periph_unlock(softc->periph);
 		break;
 	case TARGIOCDEBUG:
 	{
 #ifdef	CAMDEBUG
 		struct ccb_debug cdbg;
 
+		/* If no periph available, disallow debugging changes */
+		if ((softc->state & TARG_STATE_LUN_ENABLED) == 0) {
+			status = CAM_DEV_NOT_THERE;
+			break;
+		}
 		bzero(&cdbg, sizeof cdbg);
 		if (*((int *)addr) != 0)
 			cdbg.flags = CAM_DEBUG_PERIPH;
 		else
 			cdbg.flags = CAM_DEBUG_NONE;
+		cam_periph_lock(softc->periph);
 		xpt_setup_ccb(&cdbg.ccb_h, softc->path, /*priority*/0);
 		cdbg.ccb_h.func_code = XPT_DEBUG;
 		cdbg.ccb_h.cbfcnp = targdone;
 
-		/* If no periph available, disallow debugging changes */
-		if ((softc->state & TARG_STATE_LUN_ENABLED) == 0) {
-			status = CAM_DEV_NOT_THERE;
-			break;
-		}
 		xpt_action((union ccb *)&cdbg);
+		cam_periph_unlock(softc->periph);
 		status = cdbg.ccb_h.status & CAM_STATUS_MASK;
 #else
 		status = CAM_FUNC_NOTAVAIL;
@@ -294,10 +329,12 @@ targpoll(struct cdev *dev, int poll_events, struct thread *td)
 	revents = poll_events & (POLLOUT | POLLWRNORM);
 	if ((poll_events & (POLLIN | POLLRDNORM)) != 0) {
 		/* Poll for read() depends on user and abort queues. */
+		cam_periph_lock(softc->periph);
 		if (!TAILQ_EMPTY(&softc->user_ccb_queue) ||
 		    !TAILQ_EMPTY(&softc->abort_queue)) {
 			revents |= poll_events & (POLLIN | POLLRDNORM);
 		}
+		cam_periph_unlock(softc->periph);
 		/* Only sleep if the user didn't poll for write. */
 		if (revents == 0)
 			selrecord(td, &softc->read_select);
@@ -335,8 +372,10 @@ targreadfilt(struct knote *kn, long hint)
 	int	retval;
 
 	softc = (struct targ_softc *)kn->kn_hook;
+	cam_periph_lock(softc->periph);
 	retval = !TAILQ_EMPTY(&softc->user_ccb_queue) ||
 		 !TAILQ_EMPTY(&softc->abort_queue);
+	cam_periph_unlock(softc->periph);
 	return (retval);
 }
 
@@ -532,6 +571,7 @@ targwrite(struct cdev *dev, struct uio *uio, int ioflag)
 		switch (func_code) {
 		case XPT_ACCEPT_TARGET_IO:
 		case XPT_IMMED_NOTIFY:
+			cam_periph_lock(softc->periph);
 			ccb = targgetccb(softc, func_code, priority);
 			descr = (struct targ_cmd_descr *)ccb->ccb_h.targ_descr;
 			descr->user_ccb = user_ccb;
@@ -542,8 +582,10 @@ targwrite(struct cdev *dev, struct uio *uio, int ioflag)
 			TAILQ_INSERT_TAIL(&softc->pending_ccb_queue,
 					  &ccb->ccb_h,
 					  periph_links.tqe);
+			cam_periph_unlock(softc->periph);
 			break;
 		default:
+			cam_periph_lock(softc->periph);
 			if ((func_code & XPT_FC_QUEUED) != 0) {
 				CAM_DEBUG(softc->path, CAM_DEBUG_PERIPH,
 					  ("Sending queued ccb %#x (%p)\n",
@@ -569,6 +611,7 @@ targwrite(struct cdev *dev, struct uio *uio, int ioflag)
 					targsendccb(softc, ccb, descr);
 				targreturnccb(softc, ccb);
 			}
+			cam_periph_unlock(softc->periph);
 			break;
 		}
 		write_len += sizeof(user_ccb);
@@ -796,8 +839,6 @@ targread(struct cdev *dev, struct uio *uio, int ioflag)
 	union  ccb	  *user_ccb;
 	int		   read_len, error;
 
-	mtx_lock(&Giant);
-
 	error = 0;
 	read_len = 0;
 	softc = (struct targ_softc *)dev->si_drv1;
@@ -806,11 +847,12 @@ targread(struct cdev *dev, struct uio *uio, int ioflag)
 	CAM_DEBUG(softc->path, CAM_DEBUG_PERIPH, ("targread\n"));
 
 	/* If no data is available, wait or return immediately */
+	cam_periph_lock(softc->periph);
 	ccb_h = TAILQ_FIRST(user_queue);
 	user_descr = TAILQ_FIRST(abort_queue);
 	while (ccb_h == NULL && user_descr == NULL) {
 		if ((ioflag & IO_NDELAY) == 0) {
-			error = tsleep(user_queue,
+			error = msleep(user_queue, softc->periph->sim->mtx,
 			    PRIBIO | PCATCH, "targrd", 0);
 			ccb_h = TAILQ_FIRST(user_queue);
 			user_descr = TAILQ_FIRST(abort_queue);
@@ -822,7 +864,7 @@ targread(struct cdev *dev, struct uio *uio, int ioflag)
 				}
 			}
 		} else {
-			mtx_unlock(&Giant);
+			cam_periph_unlock(softc->periph);
 			return (EAGAIN);
 		}
 	}
@@ -841,7 +883,9 @@ targread(struct cdev *dev, struct uio *uio, int ioflag)
 		error = targreturnccb(softc, (union ccb *)ccb_h);
 		if (error != 0)
 			goto read_fail;
+		cam_periph_unlock(softc->periph);
 		error = uiomove((caddr_t)&user_ccb, sizeof(user_ccb), uio);
+		cam_periph_lock(softc->periph);
 		if (error != 0)
 			goto read_fail;
 		read_len += sizeof(user_ccb);
@@ -859,7 +903,9 @@ targread(struct cdev *dev, struct uio *uio, int ioflag)
 			  ("targread aborted descr %p (%p)\n",
 			  user_descr, user_ccb));
 		suword(&user_ccb->ccb_h.status, CAM_REQ_ABORTED);
+		cam_periph_unlock(softc->periph);
 		error = uiomove((caddr_t)&user_ccb, sizeof(user_ccb), uio);
+		cam_periph_lock(softc->periph);
 		if (error != 0)
 			goto read_fail;
 		read_len += sizeof(user_ccb);
@@ -876,7 +922,7 @@ targread(struct cdev *dev, struct uio *uio, int ioflag)
 		error = ENOSPC;
 
 read_fail:
-	mtx_unlock(&Giant);
+	cam_periph_unlock(softc->periph);
 	return (error);
 }
 
@@ -1005,6 +1051,7 @@ abort_all_pending(struct targ_softc *softc)
 	struct targ_cmd_descr   *descr;
 	struct ccb_abort	 cab;
 	struct ccb_hdr		*ccb_h;
+	struct cam_sim		*sim;
 
 	CAM_DEBUG(softc->path, CAM_DEBUG_PERIPH, ("abort_all_pending\n"));
 
@@ -1037,7 +1084,8 @@ abort_all_pending(struct targ_softc *softc)
 
 	/* If we aborted at least one pending CCB ok, wait for it. */
 	if (cab.ccb_h.status == CAM_REQ_CMP) {
-		tsleep(&softc->pending_ccb_queue,
+		sim = xpt_path_sim(softc->path);
+		msleep(&softc->pending_ccb_queue, sim->mtx,
 		       PRIBIO | PCATCH, "tgabrt", 0);
 	}
 
diff --git a/sys/dev/aac/aac_cam.c b/sys/dev/aac/aac_cam.c
index 1b5690a..88112b5 100644
--- a/sys/dev/aac/aac_cam.c
+++ b/sys/dev/aac/aac_cam.c
@@ -171,7 +171,7 @@ aac_cam_attach(device_t dev)
 		return (EIO);
 
 	sim = cam_sim_alloc(aac_cam_action, aac_cam_poll, "aacp", camsc,
-	    device_get_unit(dev), 1, 1, devq);
+	    device_get_unit(dev), &Giant, 1, 1, devq);
 	if (sim == NULL) {
 		cam_simq_free(devq);
 		return (EIO);
diff --git a/sys/dev/advansys/advansys.c b/sys/dev/advansys/advansys.c
index 5a37af7..79c1f06 100644
--- a/sys/dev/advansys/advansys.c
+++ b/sys/dev/advansys/advansys.c
@@ -1414,7 +1414,7 @@ adv_attach(adv)
 	 * Construct our SIM entry.
 	 */
 	adv->sim = cam_sim_alloc(adv_action, adv_poll, "adv", adv, adv->unit,
-				 1, adv->max_openings, devq);
+				 &Giant, 1, adv->max_openings, devq);
 	if (adv->sim == NULL)
 		return (ENOMEM);
 
diff --git a/sys/dev/advansys/adwcam.c b/sys/dev/advansys/adwcam.c
index 78b8fdd..2ba42da 100644
--- a/sys/dev/advansys/adwcam.c
+++ b/sys/dev/advansys/adwcam.c
@@ -1234,7 +1234,7 @@ adw_attach(struct adw_softc *adw)
 	 * Construct our SIM entry.
 	 */
 	adw->sim = cam_sim_alloc(adw_action, adw_poll, "adw", adw, adw->unit,
-				 1, adw->max_acbs, devq);
+				 &Giant, 1, adw->max_acbs, devq);
 	if (adw->sim == NULL) {
 		error = ENOMEM;
 		goto fail;
diff --git a/sys/dev/aha/aha.c b/sys/dev/aha/aha.c
index 714c1bb..b52b201 100644
--- a/sys/dev/aha/aha.c
+++ b/sys/dev/aha/aha.c
@@ -605,8 +605,8 @@ aha_attach(struct aha_softc *aha)
 	/*
 	 * Construct our SIM entry
 	 */
-	aha->sim = cam_sim_alloc(ahaaction, ahapoll, "aha", aha, aha->unit, 2,
-	    tagged_dev_openings, devq);
+	aha->sim = cam_sim_alloc(ahaaction, ahapoll, "aha", aha, aha->unit,
+	     &Giant, 2, tagged_dev_openings, devq);
 	if (aha->sim == NULL) {
 		cam_simq_free(devq);
 		return (ENOMEM);
diff --git a/sys/dev/ahb/ahb.c b/sys/dev/ahb/ahb.c
index 8887c80..3f4a588 100644
--- a/sys/dev/ahb/ahb.c
+++ b/sys/dev/ahb/ahb.c
@@ -553,7 +553,7 @@ ahbxptattach(struct ahb_softc *ahb)
 	 * Construct our SIM entry
 	 */
 	ahb->sim = cam_sim_alloc(ahbaction, ahbpoll, "ahb", ahb, ahb->unit,
-				 2, ahb->num_ecbs, devq);
+				 &Giant, 2, ahb->num_ecbs, devq);
 	if (ahb->sim == NULL) {
 		cam_simq_free(devq);
 		return (ENOMEM);
diff --git a/sys/dev/aic/aic.c b/sys/dev/aic/aic.c
index cd9fabc..b647cd4 100644
--- a/sys/dev/aic/aic.c
+++ b/sys/dev/aic/aic.c
@@ -30,6 +30,9 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/malloc.h>
 #include <sys/bus.h>
 
 #include <machine/bus.h>
@@ -1540,7 +1543,7 @@ aic_attach(struct aic_softc *aic)
 	 * Construct our SIM entry
 	 */
 	aic->sim = cam_sim_alloc(aic_action, aic_poll, "aic", aic,
-				 aic->unit, 2, 256, devq);
+				 aic->unit, &Giant, 2, 256, devq);
 	if (aic->sim == NULL) {
 		cam_simq_free(devq);
 		return (ENOMEM);
diff --git a/sys/dev/aic7xxx/aic79xx_osm.c b/sys/dev/aic7xxx/aic79xx_osm.c
index e16b421..aef520d 100644
--- a/sys/dev/aic7xxx/aic79xx_osm.c
+++ b/sys/dev/aic7xxx/aic79xx_osm.c
@@ -143,7 +143,7 @@ ahd_attach(struct ahd_softc *ahd)
 	 */
 	sim = cam_sim_alloc(ahd_action, ahd_poll, "ahd", ahd,
 			    device_get_unit(ahd->dev_softc),
-			    1, /*XXX*/256, devq);
+			    &Giant, 1, /*XXX*/256, devq);
 	if (sim == NULL) {
 		cam_simq_free(devq);
 		goto fail;
diff --git a/sys/dev/aic7xxx/aic7xxx_osm.c b/sys/dev/aic7xxx/aic7xxx_osm.c
index f3b106c..b53f96a 100644
--- a/sys/dev/aic7xxx/aic7xxx_osm.c
+++ b/sys/dev/aic7xxx/aic7xxx_osm.c
@@ -196,7 +196,7 @@ ahc_attach(struct ahc_softc *ahc)
 	 */
 	sim = cam_sim_alloc(ahc_action, ahc_poll, "ahc", ahc,
 			    device_get_unit(ahc->dev_softc),
-			    1, AHC_MAX_QUEUE, devq);
+			    &Giant, 1, AHC_MAX_QUEUE, devq);
 	if (sim == NULL) {
 		cam_simq_free(devq);
 		goto fail;
@@ -227,7 +227,8 @@ ahc_attach(struct ahc_softc *ahc)
 
 	if (ahc->features & AHC_TWIN) {
 		sim2 = cam_sim_alloc(ahc_action, ahc_poll, "ahc",
-				    ahc, device_get_unit(ahc->dev_softc), 1,
+				    ahc, device_get_unit(ahc->dev_softc),
+				    &Giant, 1,
 				    AHC_MAX_QUEUE, devq);
 
 		if (sim2 == NULL) {
diff --git a/sys/dev/amd/amd.c b/sys/dev/amd/amd.c
index 4c8ed41..7e5c4df 100644
--- a/sys/dev/amd/amd.c
+++ b/sys/dev/amd/amd.c
@@ -58,6 +58,7 @@
 #include <sys/module.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
+#include <sys/malloc.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>
@@ -2481,8 +2482,8 @@ amd_attach(device_t dev)
 	}
 
 	amd->psim = cam_sim_alloc(amd_action, amd_poll, "amd",
-				  amd, amd->unit, 1, MAX_TAGS_CMD_QUEUE,
-				  devq);
+				  amd, amd->unit, &Giant,
+				  1, MAX_TAGS_CMD_QUEUE, devq);
 	if (amd->psim == NULL) {
 		cam_simq_free(devq);
 		if (bootverbose)
diff --git a/sys/dev/amr/amr_cam.c b/sys/dev/amr/amr_cam.c
index 3553d43..78cd4ec 100644
--- a/sys/dev/amr/amr_cam.c
+++ b/sys/dev/amr/amr_cam.c
@@ -148,6 +148,7 @@ amr_cam_attach(struct amr_softc *sc)
 						  "amr",
 						  sc,
 						  device_get_unit(sc->amr_dev),
+						  &Giant,
 						  1,
 						  AMR_MAX_SCSI_CMDS,
 						  devq)) == NULL) {
diff --git a/sys/dev/arcmsr/arcmsr.c b/sys/dev/arcmsr/arcmsr.c
index e6441b2..f4e79b5 100644
--- a/sys/dev/arcmsr/arcmsr.c
+++ b/sys/dev/arcmsr/arcmsr.c
@@ -2145,8 +2145,9 @@ static u_int32_t arcmsr_attach(device_t dev)
 		printf("arcmsr%d: cam_simq_alloc failure!\n", unit);
 		return ENXIO;
 	}
-	acb->psim=cam_sim_alloc(arcmsr_action, arcmsr_poll
-		, "arcmsr", acb, unit, 1, ARCMSR_MAX_OUTSTANDING_CMD, devq);
+	acb->psim=cam_sim_alloc(arcmsr_action, arcmsr_poll,
+		"arcmsr", acb, unit, &Giant, 1,
+		ARCMSR_MAX_OUTSTANDING_CMD, devq);
 	if(acb->psim == NULL) {
 		arcmsr_free_resource(acb);
 		bus_release_resource(dev, SYS_RES_IRQ, 0, acb->irqres);
diff --git a/sys/dev/asr/asr.c b/sys/dev/asr/asr.c
index 1de0bfd..260cbbd 100644
--- a/sys/dev/asr/asr.c
+++ b/sys/dev/asr/asr.c
@@ -2651,7 +2651,8 @@ asr_attach(device_t dev)
 		 *	Construct our first channel SIM entry
 		 */
 		sc->ha_sim[bus] = cam_sim_alloc(asr_action, asr_poll, "asr", sc,
-						unit, 1, QueueSize, devq);
+						unit, &Giant,
+						1, QueueSize, devq);
 		if (sc->ha_sim[bus] == NULL) {
 			continue;
 		}
diff --git a/sys/dev/ata/atapi-cam.c b/sys/dev/ata/atapi-cam.c
index ebabf03..4f2bd34 100644
--- a/sys/dev/ata/atapi-cam.c
+++ b/sys/dev/ata/atapi-cam.c
@@ -210,7 +210,7 @@ atapi_cam_attach(device_t dev)
     }
 
     if ((sim = cam_sim_alloc(atapi_action, atapi_poll, "ata",
-		 (void *)scp, unit, 1, 1, devq)) == NULL) {
+		 (void *)scp, unit, &Giant, 1, 1, devq)) == NULL) {
 	error = ENOMEM;
 	goto out;
     }
diff --git a/sys/dev/buslogic/bt.c b/sys/dev/buslogic/bt.c
index 7eee35b..21579ea 100644
--- a/sys/dev/buslogic/bt.c
+++ b/sys/dev/buslogic/bt.c
@@ -874,7 +874,7 @@ bt_attach(device_t dev)
 	 * Construct our SIM entry
 	 */
 	bt->sim = cam_sim_alloc(btaction, btpoll, "bt", bt, bt->unit,
-				2, tagged_dev_openings, devq);
+				&Giant, 2, tagged_dev_openings, devq);
 	if (bt->sim == NULL) {
 		cam_simq_free(devq);
 		return (ENOMEM);
diff --git a/sys/dev/ciss/ciss.c b/sys/dev/ciss/ciss.c
index f23017b..6d71334 100644
--- a/sys/dev/ciss/ciss.c
+++ b/sys/dev/ciss/ciss.c
@@ -2476,8 +2476,8 @@ ciss_cam_init(struct ciss_softc *sc)
 	if ((sc->ciss_cam_sim[i] = cam_sim_alloc(ciss_cam_action, ciss_cam_poll,
 						 "ciss", sc,
 						 device_get_unit(sc->ciss_dev),
+						 &Giant, 1,
 						 sc->ciss_max_requests - 2,
-						 1,
 						 sc->ciss_cam_devq)) == NULL) {
 	    ciss_printf(sc, "can't allocate CAM SIM for controller %d\n", i);
 	    return(ENOMEM);
@@ -2499,8 +2499,8 @@ ciss_cam_init(struct ciss_softc *sc)
 	if ((sc->ciss_cam_sim[i] = cam_sim_alloc(ciss_cam_action, ciss_cam_poll,
 						 "ciss", sc,
 						 device_get_unit(sc->ciss_dev),
+						 &Giant, 1,
 						 sc->ciss_max_requests - 2,
-						 1,
 						 sc->ciss_cam_devq)) == NULL) {
 	    ciss_printf(sc, "can't allocate CAM SIM for controller %d\n", i);
 	    return (ENOMEM);
diff --git a/sys/dev/dpt/dpt_scsi.c b/sys/dev/dpt/dpt_scsi.c
index 9238292..276f0c9 100644
--- a/sys/dev/dpt/dpt_scsi.c
+++ b/sys/dev/dpt/dpt_scsi.c
@@ -1566,7 +1566,8 @@ dpt_attach(dpt_softc_t *dpt)
 		 * Construct our SIM entry
 		 */
 		dpt->sims[i] = cam_sim_alloc(dpt_action, dpt_poll, "dpt",
-					     dpt, dpt->unit, /*untagged*/2,
+					     dpt, dpt->unit, &Giant,
+					     /*untagged*/2,
 					     /*tagged*/dpt->max_dccbs, devq);
 		if (dpt->sims[i] == NULL) {
 			if (i == 0)
diff --git a/sys/dev/esp/ncr53c9x.c b/sys/dev/esp/ncr53c9x.c
index 50c6295..cba58d5 100644
--- a/sys/dev/esp/ncr53c9x.c
+++ b/sys/dev/esp/ncr53c9x.c
@@ -325,7 +325,7 @@ ncr53c9x_attach(struct ncr53c9x_softc *sc)
 	}
 
 	sim = cam_sim_alloc(ncr53c9x_action, ncr53c9x_poll, "esp", sc,
-			    device_get_unit(sc->sc_dev), 1,
+			    device_get_unit(sc->sc_dev), &Giant, 1,
 			    NCR_TAG_DEPTH, devq);
 	if (sim == NULL) {
 		device_printf(sc->sc_dev, "cannot allocate SIM entry\n");
diff --git a/sys/dev/firewire/sbp.c b/sys/dev/firewire/sbp.c
index c796d68..fb88553 100644
--- a/sys/dev/firewire/sbp.c
+++ b/sys/dev/firewire/sbp.c
@@ -1964,6 +1964,7 @@ END_DEBUG
 
 	sbp->sim = cam_sim_alloc(sbp_action, sbp_poll, "sbp", sbp,
 				 device_get_unit(dev),
+				 &Giant,
 				 /*untagged*/ 1,
 				 /*tagged*/ SBP_QUEUE_LEN - 1,
 				 devq);
diff --git a/sys/dev/firewire/sbp_targ.c b/sys/dev/firewire/sbp_targ.c
index eb23412..f943772 100644
--- a/sys/dev/firewire/sbp_targ.c
+++ b/sys/dev/firewire/sbp_targ.c
@@ -1626,7 +1626,7 @@ sbp_targ_attach(device_t dev)
 		return (ENXIO);
 
 	sc->sim = cam_sim_alloc(sbp_targ_action, sbp_targ_poll,
-	    "sbp_targ", sc, device_get_unit(dev),
+	    "sbp_targ", sc, device_get_unit(dev), &Giant,
 	    /*untagged*/ 1, /*tagged*/ 1, devq);
 	if (sc->sim == NULL) {
 		cam_simq_free(devq);
diff --git a/sys/dev/hptmv/entry.c b/sys/dev/hptmv/entry.c
index bab49a8..f58329d 100644
--- a/sys/dev/hptmv/entry.c
+++ b/sys/dev/hptmv/entry.c
@@ -1960,7 +1960,8 @@ hpt_attach(device_t dev)
 	 * Construct our SIM entry
 	 */
 	if ((hpt_vsim = cam_sim_alloc(hpt_action, hpt_poll, __str(PROC_DIR_NAME),
-			pAdapter, device_get_unit(pAdapter->hpt_dev), /*untagged*/1, /*tagged*/8,  devq)) == NULL)	{
+			pAdapter, device_get_unit(pAdapter->hpt_dev),
+			&Giant, /*untagged*/1, /*tagged*/8,  devq)) == NULL)	{
 		cam_simq_free(devq);
 		return ENOMEM;
 	}
diff --git a/sys/dev/iir/iir.c b/sys/dev/iir/iir.c
index 51a636c..b58306b 100644
--- a/sys/dev/iir/iir.c
+++ b/sys/dev/iir/iir.c
@@ -502,7 +502,8 @@ iir_attach(struct gdt_softc *gdt)
          * Construct our SIM entry
          */
         gdt->sims[i] = cam_sim_alloc(iir_action, iir_poll, "iir",
-                                     gdt, gdt->sc_hanum, /*untagged*/1,
+                                     gdt, gdt->sc_hanum, &Giant,
+				     /*untagged*/1,
                                      /*tagged*/GDT_MAXCMDS, devq);
         if (xpt_bus_register(gdt->sims[i], i) != CAM_SUCCESS) {
             cam_sim_free(gdt->sims[i], /*free_devq*/i == 0);
diff --git a/sys/dev/isp/isp_freebsd.c b/sys/dev/isp/isp_freebsd.c
index f422659..6c4fd06 100644
--- a/sys/dev/isp/isp_freebsd.c
+++ b/sys/dev/isp/isp_freebsd.c
@@ -137,7 +137,7 @@ isp_attach(ispsoftc_t *isp)
 	 */
 	ISPLOCK_2_CAMLOCK(isp);
 	sim = cam_sim_alloc(isp_action, isp_poll, "isp", isp,
-	    device_get_unit(isp->isp_dev), 1, isp->isp_maxcmds, devq);
+	    device_get_unit(isp->isp_dev), &Giant, 1, isp->isp_maxcmds, devq);
 	if (sim == NULL) {
 		cam_simq_free(devq);
 		CAMLOCK_2_ISPLOCK(isp);
@@ -224,7 +224,8 @@ isp_attach(ispsoftc_t *isp)
 	if (IS_DUALBUS(isp)) {
 		ISPLOCK_2_CAMLOCK(isp);
 		sim = cam_sim_alloc(isp_action, isp_poll, "isp", isp,
-		    device_get_unit(isp->isp_dev), 1, isp->isp_maxcmds, devq);
+		    device_get_unit(isp->isp_dev), &Giant, 1,
+		    isp->isp_maxcmds, devq);
 		if (sim == NULL) {
 			xpt_bus_deregister(cam_sim_path(isp->isp_sim));
 			xpt_free_path(isp->isp_path);
@@ -2147,7 +2148,7 @@ isp_make_here(ispsoftc_t *isp, int tgt)
 	 * Allocate a CCB, create a wildcard path for this bus,
 	 * and schedule a rescan.
 	 */
-	ccb = xpt_alloc_ccb_nowait();
+	ccb = xpt_alloc_ccb_nowait(isp->isp_osinfo.sim);
 	if (ccb == NULL) {
 		isp_prt(isp, ISP_LOGWARN, "unable to alloc CCB for rescan");
 		CAMLOCK_2_ISPLOCK(mpt);
diff --git a/sys/dev/mly/mly.c b/sys/dev/mly/mly.c
index 62052b0..f9ca34b 100644
--- a/sys/dev/mly/mly.c
+++ b/sys/dev/mly/mly.c
@@ -1945,6 +1945,7 @@ mly_cam_attach(struct mly_softc *sc)
 
 	    if ((sc->mly_cam_sim[chn] = cam_sim_alloc(mly_cam_action, mly_cam_poll, "mly", sc,
 						      device_get_unit(sc->mly_dev),
+						      &Giant,
 						      sc->mly_controllerinfo->maximum_parallel_commands,
 						      1, devq)) == NULL) {
 		return(ENOMEM);
@@ -1964,6 +1965,7 @@ mly_cam_attach(struct mly_softc *sc)
     for (i = 0; i < sc->mly_controllerinfo->virtual_channels_present; i++, chn++) {
 	if ((sc->mly_cam_sim[chn] = cam_sim_alloc(mly_cam_action, mly_cam_poll, "mly", sc,
 						  device_get_unit(sc->mly_dev),
+						  &Giant,
 						  sc->mly_controllerinfo->maximum_parallel_commands,
 						  0, devq)) == NULL) {
 	    return(ENOMEM);
diff --git a/sys/dev/mpt/mpt_cam.c b/sys/dev/mpt/mpt_cam.c
index ac5dffa..5d7c5a5 100644
--- a/sys/dev/mpt/mpt_cam.c
+++ b/sys/dev/mpt/mpt_cam.c
@@ -311,7 +311,7 @@ mpt_cam_attach(struct mpt_softc *mpt)
 	 * Construct our SIM entry.
 	 */
 	mpt->sim = cam_sim_alloc(mpt_action, mpt_poll, "mpt", mpt,
-	    mpt->unit, 1, maxq, devq);
+	    mpt->unit, &Giant, 1, maxq, devq);
 	if (mpt->sim == NULL) {
 		mpt_prt(mpt, "Unable to allocate CAM SIM!\n");
 		cam_simq_free(devq);
@@ -348,7 +348,7 @@ mpt_cam_attach(struct mpt_softc *mpt)
 	 * Create a "bus" to export all hidden disks to CAM.
 	 */
 	mpt->phydisk_sim = cam_sim_alloc(mpt_action, mpt_poll, "mpt", mpt,
-	    mpt->unit, 1, maxq, devq);
+	    mpt->unit, &Giant, 1, maxq, devq);
 	if (mpt->phydisk_sim == NULL) {
 		mpt_prt(mpt, "Unable to allocate Physical Disk CAM SIM!\n");
 		error = ENOMEM;
@@ -2087,6 +2087,7 @@ mpt_cam_event(struct mpt_softc *mpt, request_t *req,
 	{
 		union ccb *ccb;
 		uint32_t pathid;
+		struct cam_sim *sim;
 		/*
 		 * In general this means a device has been added to the loop.
 		 */
@@ -2095,16 +2096,17 @@ mpt_cam_event(struct mpt_softc *mpt, request_t *req,
 			break;
 		}
 		if (mpt->phydisk_sim) {
-			pathid = cam_sim_path(mpt->phydisk_sim);;
+			sim = mpt->phydisk_sim;
 		} else {
-			pathid = cam_sim_path(mpt->sim);
+			sim = mpt->sim;
 		}
+		pathid = cam_sim_path(sim);
 		MPTLOCK_2_CAMLOCK(mpt);
 		/*
 		 * Allocate a CCB, create a wildcard path for this bus,
 		 * and schedule a rescan.
 		 */
-		ccb = xpt_alloc_ccb_nowait();
+		ccb = xpt_alloc_ccb_nowait(sim);
 		if (ccb == NULL) {
 			mpt_prt(mpt, "unable to alloc CCB for rescan\n");
 			CAMLOCK_2_MPTLOCK(mpt);
diff --git a/sys/dev/ppbus/vpo.c b/sys/dev/ppbus/vpo.c
index abb2ef8..c9510b2 100644
--- a/sys/dev/ppbus/vpo.c
+++ b/sys/dev/ppbus/vpo.c
@@ -32,6 +32,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/systm.h>
 #include <sys/module.h>
 #include <sys/bus.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #include <sys/malloc.h>
 
 #include <cam/cam.h>
@@ -160,7 +162,7 @@ vpo_attach(device_t dev)
 		return (ENXIO);
 
 	vpo->sim = cam_sim_alloc(vpo_action, vpo_poll, "vpo", vpo,
-				 device_get_unit(dev),
+				 device_get_unit(dev), &Giant,
 				 /*untagged*/1, /*tagged*/0, devq);
 	if (vpo->sim == NULL) {
 		cam_simq_free(devq);
diff --git a/sys/dev/rr232x/osm_bsd.c b/sys/dev/rr232x/osm_bsd.c
index 1e77fbf..ae394f4 100644
--- a/sys/dev/rr232x/osm_bsd.c
+++ b/sys/dev/rr232x/osm_bsd.c
@@ -1088,7 +1088,8 @@ static void hpt_final_init(void *dummy)
 		}
 	
 		vbus_ext->sim = cam_sim_alloc(hpt_action, hpt_poll, driver_name,
-				vbus_ext, 0, os_max_queue_comm, /*tagged*/8,  devq);
+				vbus_ext, 0, &Giant,
+				os_max_queue_comm, /*tagged*/8,  devq);
 				
 		if (!vbus_ext->sim) {
 			os_printk("cam_sim_alloc failed");
diff --git a/sys/dev/sym/sym_hipd.c b/sys/dev/sym/sym_hipd.c
index e4357b1..679b98b 100644
--- a/sys/dev/sym/sym_hipd.c
+++ b/sys/dev/sym/sym_hipd.c
@@ -8973,7 +8973,7 @@ static int sym_cam_attach(hcb_p np)
 	 *  Construct our SIM entry.
 	 */
 	sim = cam_sim_alloc(sym_action, sym_poll, "sym", np, np->unit,
-			    1, SYM_SETUP_MAX_TAG, devq);
+			    &Giant, 1, SYM_SETUP_MAX_TAG, devq);
 	if (!sim)
 		goto fail;
 	devq = 0;
diff --git a/sys/dev/trm/trm.c b/sys/dev/trm/trm.c
index 31cad43..b3690ac 100644
--- a/sys/dev/trm/trm.c
+++ b/sys/dev/trm/trm.c
@@ -3636,6 +3636,7 @@ trm_attach(device_t dev)
 	    "trm",
 	    pACB,
 	    unit,
+	    &Giant,
 	    1,
 	    TRM_MAX_TAGS_CMD_QUEUE,
 	    device_Q);
diff --git a/sys/dev/twa/tw_osl_cam.c b/sys/dev/twa/tw_osl_cam.c
index 6b6afeb..e5a1143 100644
--- a/sys/dev/twa/tw_osl_cam.c
+++ b/sys/dev/twa/tw_osl_cam.c
@@ -102,7 +102,7 @@ tw_osli_cam_attach(struct twa_softc *sc)
 	 */
 	tw_osli_dbg_dprintf(3, sc, "Calling cam_sim_alloc");
 	sc->sim = cam_sim_alloc(twa_action, twa_poll, "twa", sc,
-			device_get_unit(sc->bus_dev),
+			device_get_unit(sc->bus_dev), &Giant,
 			TW_OSLI_MAX_NUM_IOS - 1, 1, devq);
 	if (sc->sim == NULL) {
 		cam_simq_free(devq);
diff --git a/sys/dev/usb/umass.c b/sys/dev/usb/umass.c
index de15d47..5c071b3 100644
--- a/sys/dev/usb/umass.c
+++ b/sys/dev/usb/umass.c
@@ -110,6 +110,8 @@
 #include <sys/systm.h>
 #include <sys/kernel.h>
 #include <sys/module.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #include <sys/bus.h>
 #include <sys/sysctl.h>
 
@@ -2257,6 +2259,7 @@ umass_cam_attach_sim(struct umass_softc *sc)
 				DEVNAME_SIM,
 				sc /*priv*/,
 				device_get_unit(sc->sc_dev) /*unit number*/,
+				&Giant,
 				1 /*maximum device openings*/,
 				0 /*maximum tagged device openings*/,
 				devq);
diff --git a/sys/dev/wds/wd7000.c b/sys/dev/wds/wd7000.c
index 8c91f75..05695c7 100644
--- a/sys/dev/wds/wd7000.c
+++ b/sys/dev/wds/wd7000.c
@@ -132,6 +132,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/errno.h>
 #include <sys/kernel.h>
 #include <sys/assym.h>
+#include <sys/malloc.h>
 
 #include <sys/bio.h>
 #include <sys/buf.h>
@@ -606,7 +607,7 @@ wds_attach(device_t dev)
 		goto bad;
 
 	sim = cam_sim_alloc(wds_action, wds_poll, "wds", (void *) wp,
-			    wp->unit, 1, 1, devq);
+			    wp->unit, &Giant, 1, 1, devq);
 	if (sim == NULL) {
 		cam_simq_free(devq);
 		goto bad;
diff --git a/sys/pci/ncr.c b/sys/pci/ncr.c
index 21f7f85..bcd200c 100644
--- a/sys/pci/ncr.c
+++ b/sys/pci/ncr.c
@@ -184,6 +184,8 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/module.h>
 #include <sys/sysctl.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
 #include <sys/bus.h>
 #include <machine/md_var.h>
 #include <machine/bus.h>
@@ -3781,7 +3783,7 @@ ncr_attach (device_t dev)
 	**	about our bus.
 	*/
 	np->sim = cam_sim_alloc(ncr_action, ncr_poll, "ncr", np, np->unit,
-				1, MAX_TAGS, devq);
+				&Giant, 1, MAX_TAGS, devq);
 	if (np->sim == NULL) {
 		cam_simq_free(devq);
 		return ENOMEM;