Lock the IPS driver and bring it out from under Giant. Also do some

significant clean up and optimizations:
- don't call bioq_disksort() on every command, the hardware will do that for
  us.
- remove all of the complicated bio deferral code.  bio's that can't be
  serviced immediately can just wait on the bioq.
- Only reserve one command object for doing control commands to the card.
  This simplifies a lot of code and significantly reduces the size of the
  command struct.
- Allocate commands out of a slab instead of embedding them into the softc.
- Call the command action method directly instead of having ips_get_free_cmd()
  call it indirectly.

MFC After: 1 week

6 files changed, 149 insertions, 234 deletions

diff --git a/sys/dev/ips/ips.c b/sys/dev/ips/ips.c
index b1a8297..d1aa459 100644
--- a/sys/dev/ips/ips.c
+++ b/sys/dev/ips/ips.c
@@ -101,7 +101,7 @@ static void ips_cmd_dmaload(void *cmdptr, bus_dma_segment_t *segments,int segnum
 }
 
 /* is locking needed? what locking guarentees are there on removal? */
-static __inline__ int ips_cmdqueue_free(ips_softc_t *sc)
+static int ips_cmdqueue_free(ips_softc_t *sc)
 {
 	int i, error = -1;
 	ips_command_t *command;
@@ -112,7 +112,6 @@ static __inline__ int ips_cmdqueue_free(ips_softc_t *sc)
 		for(i = 0; i < sc->max_cmds; i++){
 
 			command = &sc->commandarray[i];
-			sema_destroy(&command->cmd_sema);
 
 			if(command->command_phys_addr == 0)
 				continue;
@@ -121,22 +120,32 @@ static __inline__ int ips_cmdqueue_free(ips_softc_t *sc)
 			bus_dmamem_free(sc->command_dmatag, 
 					command->command_buffer,
 					command->command_dmamap);
+			if (command->data_dmamap != NULL)
+				bus_dmamap_destroy(command->data_dmatag,
+				    command->data_dmamap);
 		}
 		error = 0;
 		sc->state |= IPS_OFFLINE;
 	}
+	sc->staticcmd = NULL;
+	free(sc->commandarray, M_DEVBUF);
 	splx(mask);
 	return error;
 }
 
 /* places all ips command structs on the free command queue.  No locking as if someone else tries
  * to access this during init, we have bigger problems */
-static __inline__ int ips_cmdqueue_init(ips_softc_t *sc)
+static int ips_cmdqueue_init(ips_softc_t *sc)
 {
 	int i;
 	ips_command_t *command;
+
+	sc->commandarray = (ips_command_t *)malloc(sizeof(ips_command_t) *
+	    sc->max_cmds, M_DEVBUF, M_NOWAIT|M_ZERO);
+	if (sc->commandarray == NULL)
+		return (ENOMEM);
+
 	SLIST_INIT(&sc->free_cmd_list);
-	STAILQ_INIT(&sc->cmd_wait_list);
 	for(i = 0; i < sc->max_cmds; i++){
 		command = &sc->commandarray[i];
 		command->id = i;
@@ -154,8 +163,14 @@ static __inline__ int ips_cmdqueue_init(ips_softc_t *sc)
 			goto error;
 		}
 
-		sema_init(&command->cmd_sema, 0, "IPS Command Semaphore");
-		SLIST_INSERT_HEAD(&sc->free_cmd_list, command, next);	
+		if (i != 0) {
+			command->data_dmatag = sc->sg_dmatag;
+			if (bus_dmamap_create(command->data_dmatag, 0,
+			    &command->data_dmamap))
+				goto error;
+			SLIST_INSERT_HEAD(&sc->free_cmd_list, command, next);	
+		} else
+			sc->staticcmd = command;
 	}
 	sc->state &= ~IPS_OFFLINE;
 	return 0;
@@ -164,75 +179,12 @@ error:
 	return ENOMEM;
 }
 
-static int ips_add_waiting_command(ips_softc_t *sc, int (*callback)(ips_command_t *), void *data, unsigned long flags)
-{
-	intrmask_t mask;
-	ips_command_t *command;
-	ips_wait_list_t *waiter;
-	unsigned long memflags = 0;
-	if(IPS_NOWAIT_FLAG & flags)
-		memflags = M_NOWAIT;
-	waiter = malloc(sizeof(ips_wait_list_t), M_IPSBUF, memflags);
-	if(!waiter)
-		return ENOMEM;
-	mask = splbio();
-	if(sc->state & IPS_OFFLINE){
-		splx(mask);
-		free(waiter, M_IPSBUF);
-		return EIO;
-	}
-	command = SLIST_FIRST(&sc->free_cmd_list);
-	if(command && !(sc->state & IPS_TIMEOUT)){
-		SLIST_REMOVE_HEAD(&sc->free_cmd_list, next);
-		(sc->used_commands)++;
-		splx(mask);
-		clear_ips_command(command);
-		bzero(command->command_buffer, IPS_COMMAND_LEN);
-		free(waiter, M_IPSBUF);
-		command->arg = data;
-		return callback(command);
-	}
-	DEVICE_PRINTF(1, sc->dev, "adding command to the wait queue\n"); 
-	waiter->callback = callback; 
-	waiter->data = data;
-	STAILQ_INSERT_TAIL(&sc->cmd_wait_list, waiter, next);
-	splx(mask);
-	return 0;
-}
-
-static void ips_run_waiting_command(ips_softc_t *sc)
-{
-	ips_wait_list_t *waiter;
-	ips_command_t	*command;
-	int (*callback)(ips_command_t*);
-	intrmask_t mask;
-
-	mask = splbio();
-	waiter = STAILQ_FIRST(&sc->cmd_wait_list);
-	command = SLIST_FIRST(&sc->free_cmd_list);
-	if(!waiter || !command){
-		splx(mask);
-		return;
-	}
-	DEVICE_PRINTF(1, sc->dev, "removing command from wait queue\n");
-	SLIST_REMOVE_HEAD(&sc->free_cmd_list, next);
-	STAILQ_REMOVE_HEAD(&sc->cmd_wait_list, next);
-	(sc->used_commands)++;
-	splx(mask);
-	clear_ips_command(command);
-	bzero(command->command_buffer, IPS_COMMAND_LEN);
-	command->arg = waiter->data;
-	callback = waiter->callback;
-	free(waiter, M_IPSBUF);
-	callback(command);
-	return;	
-}
 /* returns a free command struct if one is available. 
  * It also blanks out anything that may be a wild pointer/value.
  * Also, command buffers are not freed.  They are
  * small so they are saved and kept dmamapped and loaded.
  */
-int ips_get_free_cmd(ips_softc_t *sc, int (*callback)(ips_command_t *), void *data, unsigned long flags)
+int ips_get_free_cmd(ips_softc_t *sc, ips_command_t **cmd, unsigned long flags)
 {
 	intrmask_t mask;
 	ips_command_t *command;
@@ -242,20 +194,25 @@ int ips_get_free_cmd(ips_softc_t *sc, int (*callback)(ips_command_t *), void *da
 		splx(mask);
 		return EIO;
 	}
-	command = SLIST_FIRST(&sc->free_cmd_list);
-	if(!command || (sc->state & IPS_TIMEOUT)){
-		splx(mask);
-		if(flags & IPS_NOWAIT_FLAG)
+	if ((flags & IPS_STATIC_FLAG) == 0) {
+		command = SLIST_FIRST(&sc->free_cmd_list);
+		if(!command || (sc->state & IPS_TIMEOUT)){
+			splx(mask);
+			return EBUSY;
+		}
+		SLIST_REMOVE_HEAD(&sc->free_cmd_list, next);
+		(sc->used_commands)++;
+	} else {
+		if (sc->state & IPS_STATIC_BUSY)
 			return EAGAIN;
-		return ips_add_waiting_command(sc, callback, data, flags);
+		command = sc->staticcmd;
+		sc->state |= IPS_STATIC_BUSY;
 	}
-	SLIST_REMOVE_HEAD(&sc->free_cmd_list, next);
-	(sc->used_commands)++;
 	splx(mask);
 	clear_ips_command(command);
 	bzero(command->command_buffer, IPS_COMMAND_LEN);
-	command->arg = data;
-	return callback(command);
+	*cmd = command;
+	return 0;
 }
 
 /* adds a command back to the free command queue */
@@ -264,14 +221,16 @@ void ips_insert_free_cmd(ips_softc_t *sc, ips_command_t *command)
 	intrmask_t mask;
 	mask = splbio();
 
-	if (sema_value(&command->cmd_sema) != 0)
+	if (sema_value(&sc->cmd_sema) != 0)
 		panic("ips: command returned non-zero semaphore");
 
-	SLIST_INSERT_HEAD(&sc->free_cmd_list, command, next);
-	(sc->used_commands)--;
+	if (command != sc->staticcmd) {
+		SLIST_INSERT_HEAD(&sc->free_cmd_list, command, next);
+		(sc->used_commands)--;
+	} else {
+		sc->state &= ~IPS_STATIC_BUSY;
+	}
 	splx(mask);
-	if(!(sc->state & IPS_TIMEOUT))
-		ips_run_waiting_command(sc);
 }
 static const char* ips_diskdev_statename(u_int8_t state)
 {
@@ -348,6 +307,8 @@ static void ips_timeout(void *arg)
 	ips_softc_t *sc = arg;
 	int i, state = 0;
 	ips_command_t *command;
+
+	mtx_lock(&sc->queue_mtx);
 	command = &sc->commandarray[0];
 	mask = splbio();
 	for(i = 0; i < sc->max_cmds; i++){
@@ -376,10 +337,10 @@ static void ips_timeout(void *arg)
 			   would go to the card. This sucks. */
 		} else
 			sc->state &= ~IPS_TIMEOUT;
-		ips_run_waiting_command(sc);
 	}
 	if (sc->state != IPS_OFFLINE)
 		sc->timer = timeout(ips_timeout, sc, 10*hz);
+	mtx_unlock(&sc->queue_mtx);
 	splx(mask);
 }
 
@@ -402,8 +363,8 @@ int ips_adapter_init(ips_softc_t *sc)
 				/* maxsegsize*/	IPS_COMMAND_LEN + 
 						    IPS_MAX_SG_LEN,
 				/* flags     */	0,
-				/* lockfunc  */ busdma_lock_mutex,
-				/* lockarg   */ &Giant,
+				/* lockfunc  */ NULL,
+				/* lockarg   */ NULL,
 				&sc->command_dmatag) != 0) {
                 device_printf(sc->dev, "can't alloc command dma tag\n");
 		goto error;
@@ -420,7 +381,7 @@ int ips_adapter_init(ips_softc_t *sc)
 				/* maxsegsize*/	IPS_MAX_IOBUF_SIZE,
 				/* flags     */	0,
 				/* lockfunc  */ busdma_lock_mutex,
-				/* lockarg   */ &Giant,
+				/* lockarg   */ &sc->queue_mtx,
 				&sc->sg_dmatag) != 0) {
 		device_printf(sc->dev, "can't alloc SG dma tag\n");
 		goto error;
@@ -563,11 +524,13 @@ void ips_morpheus_intr(void *void_sc)
 	int cmdnumber;
 	ips_cmd_status_t status;
 
+	mtx_lock(&sc->queue_mtx);
 	iisr =ips_read_4(sc, MORPHEUS_REG_IISR);
 	oisr =ips_read_4(sc, MORPHEUS_REG_OISR);
         PRINTF(9,"interrupt registers in:%x out:%x\n",iisr, oisr);
 	if(!(oisr & MORPHEUS_BIT_CMD_IRQ)){
 		DEVICE_PRINTF(2,sc->dev, "got a non-command irq\n");
+		mtx_unlock(&sc->queue_mtx);
 		return;	
 	}
 	while((status.value = ips_read_4(sc, MORPHEUS_REG_OQPR)) != 0xffffffff){
@@ -579,6 +542,7 @@ void ips_morpheus_intr(void *void_sc)
 
 		DEVICE_PRINTF(9,sc->dev, "got command %d\n", cmdnumber);
 	}
+	mtx_unlock(&sc->queue_mtx);
         return;
 }
 
@@ -622,8 +586,8 @@ static int ips_copperhead_queue_init(ips_softc_t *sc)
 				/* numsegs   */	1,
 				/* maxsegsize*/	sizeof(ips_copper_queue_t),
 				/* flags     */	0,
-				/* lockfunc  */ busdma_lock_mutex,
-				/* lockarg   */ &Giant,
+				/* lockfunc  */ NULL,
+				/* lockarg   */ NULL,
 				&dmatag) != 0) {
                 device_printf(sc->dev, "can't alloc dma tag for statue queue\n");
 		error = ENOMEM;
@@ -742,6 +706,7 @@ void ips_copperhead_intr(void *void_sc)
 	int cmdnumber;
 	ips_cmd_status_t status;
 
+	mtx_lock(&sc->queue_mtx);
 	while(ips_read_1(sc, COPPER_REG_HISR) & COPPER_SCE_BIT){
 		status.value = ips_copperhead_cmd_status(sc);
 		cmdnumber = status.fields.command_id;
@@ -750,6 +715,7 @@ void ips_copperhead_intr(void *void_sc)
 		sc->commandarray[cmdnumber].callback(&(sc->commandarray[cmdnumber]));
 		PRINTF(9, "ips: got command %d\n", cmdnumber);
 	}
+	mtx_unlock(&sc->queue_mtx);
         return;
 }
 
diff --git a/sys/dev/ips/ips.h b/sys/dev/ips/ips.h
index 9c26a7a..c6952fd 100644
--- a/sys/dev/ips/ips.h
+++ b/sys/dev/ips/ips.h
@@ -71,12 +71,13 @@ MALLOC_DECLARE(M_IPSBUF);
 #define IPS_MAX_SG_LEN			(sizeof(ips_sg_element_t) * IPS_MAX_SG_ELEMENTS)
 #define IPS_NVRAM_PAGE_SIZE		128
 /* various flags */
-#define IPS_NOWAIT_FLAG			1
+#define IPS_STATIC_FLAG			0x01
 
 /* states for the card to be in */
 #define IPS_DEV_OPEN			0x01
 #define IPS_TIMEOUT			0x02 /* command time out, need reset */
 #define IPS_OFFLINE			0x04 /* can't reset card/card failure */
+#define IPS_STATIC_BUSY			0x08
 
 /* max number of commands set to something low for now */
 #define IPS_MAX_CMD_NUM			128	
@@ -379,25 +380,18 @@ typedef struct ips_command{
 	u_int8_t 		id;
 	u_int8_t		timeout;
 	struct ips_softc *	sc;
+	bus_dma_tag_t		data_dmatag;
+	bus_dmamap_t		data_dmamap;
 	bus_dmamap_t		command_dmamap;
 	void *			command_buffer;
 	u_int32_t		command_phys_addr;/*WARNING! must be changed if 64bit addressing ever used*/	
-	struct sema		cmd_sema;
 	ips_cmd_status_t	status;
 	SLIST_ENTRY(ips_command)	next;
-	bus_dma_tag_t		data_dmatag;
-	bus_dmamap_t		data_dmamap;
 	void *			data_buffer;
-	void * 			arg;
+	void *			arg;
 	void			(* callback)(struct ips_command *command);
 }ips_command_t;
 
-typedef struct ips_wait_list{
-	STAILQ_ENTRY(ips_wait_list) next;
-	void 			*data;
-	int			(* callback)(ips_command_t *command);
-}ips_wait_list_t;
-
 typedef struct ips_softc{
         struct resource *       iores;
         struct resource *       irqres;
@@ -426,9 +420,9 @@ typedef struct ips_softc{
 	u_int8_t		next_drive;
 	u_int8_t		max_cmds;
 	volatile u_int8_t	used_commands;
-	ips_command_t		commandarray[IPS_MAX_CMD_NUM];
+	ips_command_t		*commandarray;
+	ips_command_t		*staticcmd;
 	SLIST_HEAD(command_list, ips_command) free_cmd_list;
-	STAILQ_HEAD(command_wait_list,ips_wait_list)  cmd_wait_list;
 	int			(* ips_adapter_reinit)(struct ips_softc *sc, 
 						       int force);
         void                    (* ips_adapter_intr)(void *sc);
@@ -436,6 +430,7 @@ typedef struct ips_softc{
 	ips_copper_queue_t *	copper_queue;
 	struct mtx		queue_mtx;
 	struct bio_queue_head	queue;
+	struct sema		cmd_sema;
 
 }ips_softc_t;
 
@@ -455,8 +450,7 @@ extern int ips_update_nvram(ips_softc_t *sc);
 extern int ips_clear_adapter(ips_softc_t *sc);
 
 /* function defines from ips.c */
-extern int ips_get_free_cmd(ips_softc_t *sc, int (*callback)(ips_command_t *), 
-				void *data, unsigned long flags);
+extern int ips_get_free_cmd(ips_softc_t *sc, ips_command_t **command, unsigned long flags);
 extern void ips_insert_free_cmd(ips_softc_t *sc, ips_command_t *command);
 extern int ips_adapter_init(ips_softc_t *sc);
 extern int ips_morpheus_reinit(ips_softc_t *sc, int force);
diff --git a/sys/dev/ips/ips_commands.c b/sys/dev/ips/ips_commands.c
index e2495ab..bbaed71 100644
--- a/sys/dev/ips/ips_commands.c
+++ b/sys/dev/ips/ips_commands.c
@@ -39,12 +39,9 @@ __FBSDID("$FreeBSD$");
  */
 static void ips_wakeup_callback(ips_command_t *command)
 {
-	ips_cmd_status_t *status;
-	status = command->arg;
-	status->value = command->status.value;
 	bus_dmamap_sync(command->sc->command_dmatag, command->command_dmamap, 
 			BUS_DMASYNC_POSTWRITE);
-	sema_post(&command->cmd_sema);
+	sema_post(&command->sc->cmd_sema);
 }
 /* Below are a series of functions for sending an IO request
  * to the adapter.  The flow order is: start, send, callback, finish.
@@ -62,7 +59,6 @@ static void ips_io_request_finish(ips_command_t *command)
 				BUS_DMASYNC_POSTWRITE);
 	}
 	bus_dmamap_unload(command->data_dmatag, command->data_dmamap);
-	bus_dmamap_destroy(command->data_dmatag, command->data_dmamap);
 	if(COMMAND_ERROR(&command->status)){
 		iobuf->bio_flags |=BIO_ERROR;
 		iobuf->bio_error = EIO;
@@ -85,7 +81,6 @@ static void ips_io_request_callback(void *cmdptr, bus_dma_segment_t *segments,in
 	if(error){
 		printf("ips: error = %d in ips_sg_request_callback\n", error);
 		bus_dmamap_unload(command->data_dmatag, command->data_dmamap);
-		bus_dmamap_destroy(command->data_dmatag, command->data_dmamap);
 		iobuf->bio_flags |= BIO_ERROR;
 		iobuf->bio_error = ENOMEM;
 		ips_insert_free_cmd(sc, command);
@@ -140,20 +135,10 @@ static void ips_io_request_callback(void *cmdptr, bus_dma_segment_t *segments,in
 	return;
 }
 
-static int ips_send_io_request(ips_command_t *command)
+static int ips_send_io_request(ips_command_t *command, struct bio *iobuf)
 {
-	ips_softc_t *sc = command->sc;
-	struct bio *iobuf = command->arg;
-	command->data_dmatag = sc->sg_dmatag;
-	if(bus_dmamap_create(command->data_dmatag, 0, &command->data_dmamap)){
-		device_printf(sc->dev, "dmamap failed\n");
-		iobuf->bio_flags |= BIO_ERROR;
-		iobuf->bio_error = ENOMEM;
-		ips_insert_free_cmd(sc, command);
-		ipsd_finish(iobuf);
-		return 0;
-	}
 	command->callback = ips_io_request_finish;
+	command->arg = iobuf;
 	PRINTF(10, "ips test: : bcount %ld\n", iobuf->bio_bcount);
 	bus_dmamap_load(command->data_dmatag, command->data_dmamap,
 			iobuf->bio_data, iobuf->bio_bcount,
@@ -164,21 +149,17 @@ static int ips_send_io_request(ips_command_t *command)
 void ips_start_io_request(ips_softc_t *sc)
 {
 	struct bio *iobuf;
+	ips_command_t *command;
 
-	mtx_lock(&sc->queue_mtx);
 	iobuf = bioq_first(&sc->queue);
-	if(!iobuf) {
-		mtx_unlock(&sc->queue_mtx);
+	if(!iobuf)
 		return;
-	}
 
-	if(ips_get_free_cmd(sc, ips_send_io_request, iobuf, IPS_NOWAIT_FLAG)){
-		mtx_unlock(&sc->queue_mtx);
+	if (ips_get_free_cmd(sc, &command, 0))
 		return;
-	}
 	
 	bioq_remove(&sc->queue, iobuf);
-	mtx_unlock(&sc->queue_mtx);
+	ips_send_io_request(command, iobuf);
 	return;
 }
 
@@ -193,8 +174,7 @@ static void ips_adapter_info_callback(void *cmdptr, bus_dma_segment_t *segments,
 	ips_adapter_info_cmd *command_struct;
 	sc = command->sc;
 	if(error){
-		ips_cmd_status_t * status = command->arg;
-		status->value = IPS_ERROR_STATUS; /* a lovely error value */
+		command->status.value = IPS_ERROR_STATUS; /* a lovely error value */
 		ips_insert_free_cmd(sc, command);
 		printf("ips: error = %d in ips_get_adapter_info\n", error);
 		return;
@@ -217,7 +197,6 @@ static int ips_send_adapter_info_cmd(ips_command_t *command)
 {
 	int error = 0;
 	ips_softc_t *sc = command->sc;
-	ips_cmd_status_t *status = command->arg;
 
 	if (bus_dma_tag_create(	/* parent    */	sc->adapter_dmatag,
 				/* alignemnt */	1,
@@ -230,8 +209,8 @@ static int ips_send_adapter_info_cmd(ips_command_t *command)
 				/* numsegs   */	1,
 				/* maxsegsize*/	IPS_ADAPTER_INFO_LEN,
 				/* flags     */	0,
-				/* lockfunc  */ busdma_lock_mutex,
-				/* lockarg   */ &Giant,
+				/* lockfunc  */ NULL,
+				/* lockarg   */ NULL,
 				&command->data_dmatag) != 0) {
                 printf("ips: can't alloc dma tag for adapter status\n");
 		error = ENOMEM;
@@ -247,8 +226,8 @@ static int ips_send_adapter_info_cmd(ips_command_t *command)
 			command->data_buffer,IPS_ADAPTER_INFO_LEN, 
 			ips_adapter_info_callback, command, BUS_DMA_NOWAIT);
 
-	if ((status->value == IPS_ERROR_STATUS) ||
-	    (sema_timedwait(&command->cmd_sema, 30*hz) != 0))
+	if ((command->status.value == IPS_ERROR_STATUS) ||
+	    (sema_timedwait(&sc->cmd_sema, 30*hz) != 0))
 		error = ETIMEDOUT;
 
 	if (error == 0) {
@@ -270,21 +249,17 @@ exit:
 
 int ips_get_adapter_info(ips_softc_t *sc)
 {
+	ips_command_t *command;
 	int error = 0;
-	ips_cmd_status_t *status;
-	status = malloc(sizeof(ips_cmd_status_t), M_IPSBUF, M_NOWAIT|M_ZERO);
-	if(!status)
-		return ENOMEM;
-	if(ips_get_free_cmd(sc, ips_send_adapter_info_cmd, status, 
-			    IPS_NOWAIT_FLAG) > 0){
+
+	if (ips_get_free_cmd(sc, &command, IPS_STATIC_FLAG) > 0){
 		device_printf(sc->dev, "unable to get adapter configuration\n");
-		free(status, M_IPSBUF);
 		return ENXIO;
 	}
-	if (COMMAND_ERROR(status)){
+	ips_send_adapter_info_cmd(command);
+	if (COMMAND_ERROR(&command->status)){
 		error = ENXIO;
 	}
-	free(status, M_IPSBUF);
 	return error;
 }
 
@@ -299,8 +274,7 @@ static void ips_drive_info_callback(void *cmdptr, bus_dma_segment_t *segments,in
 	ips_drive_cmd *command_struct;
 	sc = command->sc;
 	if(error){
-		ips_cmd_status_t * status = command->arg;
-                status->value = IPS_ERROR_STATUS;
+                command->status.value = IPS_ERROR_STATUS;
 		ips_insert_free_cmd(sc, command);
 		printf("ips: error = %d in ips_get_drive_info\n", error);
 		return;
@@ -321,7 +295,6 @@ static int ips_send_drive_info_cmd(ips_command_t *command)
 {
 	int error = 0;
 	ips_softc_t *sc = command->sc;
-	ips_cmd_status_t *status = command->arg;
 	ips_drive_info_t *driveinfo;
 
 	if (bus_dma_tag_create(	/* parent    */	sc->adapter_dmatag,
@@ -335,8 +308,8 @@ static int ips_send_drive_info_cmd(ips_command_t *command)
 				/* numsegs   */	1,
 				/* maxsegsize*/	IPS_DRIVE_INFO_LEN,
 				/* flags     */	0,
-				/* lockfunc  */ busdma_lock_mutex,
-				/* lockarg   */ &Giant,
+				/* lockfunc  */ NULL,
+				/* lockarg   */ NULL,
 				&command->data_dmatag) != 0) {
                 printf("ips: can't alloc dma tag for drive status\n");
 		error = ENOMEM;
@@ -351,8 +324,8 @@ static int ips_send_drive_info_cmd(ips_command_t *command)
 	bus_dmamap_load(command->data_dmatag, command->data_dmamap, 
 			command->data_buffer,IPS_DRIVE_INFO_LEN, 
 			ips_drive_info_callback, command, BUS_DMA_NOWAIT);
-	if ((status->value == IPS_ERROR_STATUS) ||
-	    (sema_timedwait(&command->cmd_sema, 10*hz) != 0))
+	if ((command->status.value == IPS_ERROR_STATUS) ||
+	    (sema_timedwait(&sc->cmd_sema, 10*hz) != 0))
 		error = ETIMEDOUT;
 
 	if (error == 0) {
@@ -377,20 +350,16 @@ exit:
 int ips_get_drive_info(ips_softc_t *sc)
 {
 	int error = 0;
-	ips_cmd_status_t *status;
-	status = malloc(sizeof(ips_cmd_status_t), M_IPSBUF, M_NOWAIT|M_ZERO);
-	if(!status)
-		return ENOMEM;
-	if(ips_get_free_cmd(sc, ips_send_drive_info_cmd, status, 
-			    IPS_NOWAIT_FLAG) > 0){
-		free(status, M_IPSBUF);
+	ips_command_t *command;
+
+	if (ips_get_free_cmd(sc, &command, IPS_STATIC_FLAG) > 0){
 		device_printf(sc->dev, "unable to get drive configuration\n");
 		return ENXIO;
 	}
-	if(COMMAND_ERROR(status)){
+	ips_send_drive_info_cmd(command);
+	if(COMMAND_ERROR(&command->status)){
 		error = ENXIO;
 	}
-	free(status, M_IPSBUF);
 	return error;
 }
 
@@ -399,7 +368,6 @@ int ips_get_drive_info(ips_softc_t *sc)
 static int ips_send_flush_cache_cmd(ips_command_t *command)
 {
 	ips_softc_t *sc = command->sc;
-	ips_cmd_status_t *status = command->arg;
 	ips_generic_cmd *command_struct;
 
 	PRINTF(10,"ips test: got a command, building flush command\n");
@@ -410,28 +378,24 @@ static int ips_send_flush_cache_cmd(ips_command_t *command)
 	bus_dmamap_sync(sc->command_dmatag, command->command_dmamap, 
 			BUS_DMASYNC_PREWRITE);
 	sc->ips_issue_cmd(command);
-	if (status->value != IPS_ERROR_STATUS)
-		sema_wait(&command->cmd_sema);
+	if (command->status.value != IPS_ERROR_STATUS)
+		sema_wait(&sc->cmd_sema);
 	ips_insert_free_cmd(sc, command);
 	return 0;
 }
 
 int ips_flush_cache(ips_softc_t *sc)
 {
-	ips_cmd_status_t *status;
-	status = malloc(sizeof(ips_cmd_status_t), M_IPSBUF, M_NOWAIT|M_ZERO);
-	if(!status)
-		return ENOMEM;
+	ips_command_t *command;
+
 	device_printf(sc->dev, "flushing cache\n");
-	if(ips_get_free_cmd(sc, ips_send_flush_cache_cmd, status, 
-			    IPS_NOWAIT_FLAG)){
-		free(status, M_IPSBUF);
+	if (ips_get_free_cmd(sc, &command, IPS_STATIC_FLAG)){
 		device_printf(sc->dev, "ERROR: unable to get a command! can't flush cache!\n");
 	}
-	if(COMMAND_ERROR(status)){
+	ips_send_flush_cache_cmd(command);
+	if(COMMAND_ERROR(&command->status)){
 		device_printf(sc->dev, "ERROR: cache flush command failed!\n");
 	}
-	free(status, M_IPSBUF);
 	return 0;
 }
 
@@ -481,7 +445,6 @@ void static ips_ffdc_settime(ips_adapter_ffdc_cmd *command, time_t sctime)
 static int ips_send_ffdc_reset_cmd(ips_command_t *command)
 {
 	ips_softc_t *sc = command->sc;
-	ips_cmd_status_t *status = command->arg;
 	ips_adapter_ffdc_cmd *command_struct;
 
 	PRINTF(10,"ips test: got a command, building ffdc reset command\n");
@@ -496,27 +459,23 @@ static int ips_send_ffdc_reset_cmd(ips_command_t *command)
 	bus_dmamap_sync(sc->command_dmatag, command->command_dmamap,
 			BUS_DMASYNC_PREWRITE);
 	sc->ips_issue_cmd(command);
-	if (status->value != IPS_ERROR_STATUS)
-		sema_wait(&command->cmd_sema);
+	if (command->status.value != IPS_ERROR_STATUS)
+		sema_wait(&sc->cmd_sema);
 	ips_insert_free_cmd(sc, command);
 	return 0;
 }
 
 int ips_ffdc_reset(ips_softc_t *sc)
 {
-	ips_cmd_status_t *status;
-	status = malloc(sizeof(ips_cmd_status_t), M_IPSBUF, M_NOWAIT|M_ZERO);
-	if(!status)
-		return ENOMEM;
-	if(ips_get_free_cmd(sc, ips_send_ffdc_reset_cmd, status,
-			    IPS_NOWAIT_FLAG)){
-		free(status, M_IPSBUF);
+	ips_command_t *command;
+
+	if (ips_get_free_cmd(sc, &command, IPS_STATIC_FLAG)){
 		device_printf(sc->dev, "ERROR: unable to get a command! can't send ffdc reset!\n");
 	}
-	if(COMMAND_ERROR(status)){
+	ips_send_ffdc_reset_cmd(command);
+	if(COMMAND_ERROR(&command->status)){
 		device_printf(sc->dev, "ERROR: ffdc reset command failed!\n");
 	}
-	free(status, M_IPSBUF);
 	return 0;
 }
 
@@ -553,8 +512,7 @@ static void ips_read_nvram_callback(void *cmdptr, bus_dma_segment_t *segments,in
 	ips_rw_nvram_cmd *command_struct;
 	sc = command->sc;
 	if(error){
-		ips_cmd_status_t * status = command->arg;
-                status->value = IPS_ERROR_STATUS;
+                command->status.value = IPS_ERROR_STATUS;
 		ips_insert_free_cmd(sc, command);
 		printf("ips: error = %d in ips_read_nvram_callback\n", error);
 		return;
@@ -573,10 +531,10 @@ static void ips_read_nvram_callback(void *cmdptr, bus_dma_segment_t *segments,in
 	sc->ips_issue_cmd(command);
 }
 
-static int ips_read_nvram(ips_command_t *command){
+static int ips_read_nvram(ips_command_t *command)
+{
 	int error = 0;
 	ips_softc_t *sc = command->sc;
-	ips_cmd_status_t *status = command->arg;
 
 	if (bus_dma_tag_create(	/* parent    */	sc->adapter_dmatag,
 				/* alignemnt */	1,
@@ -589,8 +547,8 @@ static int ips_read_nvram(ips_command_t *command){
 				/* numsegs   */	1,
 				/* maxsegsize*/	IPS_NVRAM_PAGE_SIZE,
 				/* flags     */	0,
-				/* lockfunc  */ busdma_lock_mutex,
-				/* lockarg   */ &Giant,
+				/* lockfunc  */ NULL,
+				/* lockarg   */ NULL,
 				&command->data_dmatag) != 0) {
                 printf("ips: can't alloc dma tag for nvram\n");
 		error = ENOMEM;
@@ -605,8 +563,8 @@ static int ips_read_nvram(ips_command_t *command){
 	bus_dmamap_load(command->data_dmatag, command->data_dmamap, 
 			command->data_buffer,IPS_NVRAM_PAGE_SIZE, 
 			ips_read_nvram_callback, command, BUS_DMA_NOWAIT);
-	if ((status->value == IPS_ERROR_STATUS) ||
-	    (sema_timedwait(&command->cmd_sema, 30*hz) != 0))
+	if ((command->status.value == IPS_ERROR_STATUS) ||
+	    (sema_timedwait(&sc->cmd_sema, 30*hz) != 0))
 		error = ETIMEDOUT;
 
 	if (error == 0) {
@@ -625,19 +583,16 @@ exit:
 
 int ips_update_nvram(ips_softc_t *sc)
 {
-	ips_cmd_status_t *status;
-	status = malloc(sizeof(ips_cmd_status_t), M_IPSBUF, M_NOWAIT|M_ZERO);
-	if(!status)
-		return ENOMEM;
-	if(ips_get_free_cmd(sc, ips_read_nvram, status, IPS_NOWAIT_FLAG)){
-		free(status, M_IPSBUF);
+	ips_command_t *command;
+
+	if (ips_get_free_cmd(sc, &command, IPS_STATIC_FLAG)){
 		device_printf(sc->dev, "ERROR: unable to get a command! can't update nvram\n");
 		return 1;
 	}
-	if(COMMAND_ERROR(status)){
+	ips_read_nvram(command);
+	if(COMMAND_ERROR(&command->status)){
 		device_printf(sc->dev, "ERROR: nvram update command failed!\n");
 	}
-	free(status, M_IPSBUF);
 	return 0;
 
 
@@ -647,7 +602,6 @@ int ips_update_nvram(ips_softc_t *sc)
 static int ips_send_config_sync_cmd(ips_command_t *command)
 {
 	ips_softc_t *sc = command->sc;
-	ips_cmd_status_t *status = command->arg;
 	ips_generic_cmd *command_struct;
 
 	PRINTF(10,"ips test: got a command, building flush command\n");
@@ -659,8 +613,8 @@ static int ips_send_config_sync_cmd(ips_command_t *command)
 	bus_dmamap_sync(sc->command_dmatag, command->command_dmamap, 
 			BUS_DMASYNC_PREWRITE);
 	sc->ips_issue_cmd(command);
-	if (status->value != IPS_ERROR_STATUS)
-		sema_wait(&command->cmd_sema);
+	if (command->status.value != IPS_ERROR_STATUS)
+		sema_wait(&sc->cmd_sema);
 	ips_insert_free_cmd(sc, command);
 	return 0;
 }
@@ -668,7 +622,6 @@ static int ips_send_config_sync_cmd(ips_command_t *command)
 static int ips_send_error_table_cmd(ips_command_t *command)
 {
 	ips_softc_t *sc = command->sc;
-	ips_cmd_status_t *status = command->arg;
 	ips_generic_cmd *command_struct;
 
 	PRINTF(10,"ips test: got a command, building errortable command\n");
@@ -680,8 +633,8 @@ static int ips_send_error_table_cmd(ips_command_t *command)
 	bus_dmamap_sync(sc->command_dmatag, command->command_dmamap, 
 			BUS_DMASYNC_PREWRITE);
 	sc->ips_issue_cmd(command);
-	if (status->value != IPS_ERROR_STATUS)
-		sema_wait(&command->cmd_sema);
+	if (command->status.value != IPS_ERROR_STATUS)
+		sema_wait(&sc->cmd_sema);
 	ips_insert_free_cmd(sc, command);
 	return 0;
 }
@@ -689,36 +642,29 @@ static int ips_send_error_table_cmd(ips_command_t *command)
 
 int ips_clear_adapter(ips_softc_t *sc)
 {
-	ips_cmd_status_t *status;
-	status = malloc(sizeof(ips_cmd_status_t), M_IPSBUF, M_NOWAIT|M_ZERO);
-	if(!status)
-		return ENOMEM;
+	ips_command_t *command;
+
 	device_printf(sc->dev, "syncing config\n");
-	if(ips_get_free_cmd(sc, ips_send_config_sync_cmd, status, 
-			    IPS_NOWAIT_FLAG)){
-		free(status, M_IPSBUF);
+	if (ips_get_free_cmd(sc, &command, IPS_STATIC_FLAG)){
 		device_printf(sc->dev, "ERROR: unable to get a command! can't sync cache!\n");
 		return 1;
 	}
-	if(COMMAND_ERROR(status)){
-		free(status, M_IPSBUF);
+	ips_send_config_sync_cmd(command);
+	if(COMMAND_ERROR(&command->status)){
 		device_printf(sc->dev, "ERROR: cache sync command failed!\n");
 		return 1;
 	}
 
 	device_printf(sc->dev, "clearing error table\n");
-	if(ips_get_free_cmd(sc, ips_send_error_table_cmd, status, 
-			    IPS_NOWAIT_FLAG)){
-		free(status, M_IPSBUF);
+	if(ips_get_free_cmd(sc, &command, IPS_STATIC_FLAG)){
 		device_printf(sc->dev, "ERROR: unable to get a command! can't sync cache!\n");
 		return 1;
 	}
-	if(COMMAND_ERROR(status)){
+	ips_send_error_table_cmd(command);
+	if(COMMAND_ERROR(&command->status)){
 		device_printf(sc->dev, "ERROR: etable command failed!\n");
-		free(status, M_IPSBUF);
 		return 1;
 	}
 
-	free(status, M_IPSBUF);
 	return 0;
 }
diff --git a/sys/dev/ips/ips_disk.c b/sys/dev/ips/ips_disk.c
index 87b848e..4ddcfa1 100644
--- a/sys/dev/ips/ips_disk.c
+++ b/sys/dev/ips/ips_disk.c
@@ -102,9 +102,9 @@ static void ipsd_strategy(struct bio *iobuf)
 	DEVICE_PRINTF(8,dsc->dev,"in strategy\n");
 	iobuf->bio_driver1 = (void *)(uintptr_t)dsc->sc->drives[dsc->disk_number].drivenum;
 	mtx_lock(&dsc->sc->queue_mtx);
-	bioq_disksort(&dsc->sc->queue, iobuf);
-	mtx_unlock(&dsc->sc->queue_mtx);
+	bioq_insert_tail(&dsc->sc->queue, iobuf);
 	ips_start_io_request(dsc->sc);
+	mtx_unlock(&dsc->sc->queue_mtx);
 }
 
 static int ipsd_probe(device_t dev)
@@ -149,7 +149,7 @@ static int ipsd_attach(device_t dev)
 	dsc->ipsd_disk->d_sectorsize = IPS_BLKSIZE;
 	dsc->ipsd_disk->d_mediasize = (off_t)totalsectors * IPS_BLKSIZE;
 	dsc->ipsd_disk->d_unit = dsc->unit;
-	dsc->ipsd_disk->d_flags = DISKFLAG_NEEDSGIANT;
+	dsc->ipsd_disk->d_flags = 0;
 	disk_create(dsc->ipsd_disk, DISK_VERSION);
 
 	device_printf(dev, "Logical Drive  (%dMB)\n",
diff --git a/sys/dev/ips/ips_ioctl.c b/sys/dev/ips/ips_ioctl.c
index fa4ff36..d3cee20 100644
--- a/sys/dev/ips/ips_ioctl.c
+++ b/sys/dev/ips/ips_ioctl.c
@@ -85,6 +85,7 @@ static int ips_ioctl_start(ips_command_t *command)
 
 static int ips_ioctl_cmd(ips_softc_t *sc, ips_ioctl_t *ioctl_cmd, ips_user_request *user_request)
 {
+	ips_command_t *command;
 	int error = EINVAL;
 
 	if (bus_dma_tag_create(	/* parent    */	sc->adapter_dmatag,
@@ -98,8 +99,8 @@ static int ips_ioctl_cmd(ips_softc_t *sc, ips_ioctl_t *ioctl_cmd, ips_user_reque
 			/* numsegs   */	1,
 			/* maxsegsize*/	ioctl_cmd->datasize,
 			/* flags     */	0,
-			/* lockfunc  */ busdma_lock_mutex,
-			/* lockarg   */ &Giant,
+			/* lockfunc  */ NULL,
+			/* lockarg   */ NULL,
 			&ioctl_cmd->dmatag) != 0) {
 		return ENOMEM;
         }
@@ -112,16 +113,21 @@ static int ips_ioctl_cmd(ips_softc_t *sc, ips_ioctl_t *ioctl_cmd, ips_user_reque
 	    ioctl_cmd->datasize))
 		goto exit;
 	ioctl_cmd->status.value = 0xffffffff;
-	if((error = ips_get_free_cmd(sc, ips_ioctl_start, ioctl_cmd,0)) > 0){
+	mtx_lock(&sc->queue_mtx);
+	if((error = ips_get_free_cmd(sc, &command, 0)) > 0){
 		error = ENOMEM;
+		mtx_unlock(&sc->queue_mtx);
 		goto exit;
 	}
+	command->arg = ioctl_cmd;
+	ips_ioctl_start(command);
 	while( ioctl_cmd->status.value == 0xffffffff)
-		tsleep(ioctl_cmd, 0, "ips", hz/10);
+		msleep(ioctl_cmd, &sc->queue_mtx, 0, "ips", hz/10);
 	if(COMMAND_ERROR(&ioctl_cmd->status))
 		error = EIO;
 	else
 		error = 0;
+	mtx_unlock(&sc->queue_mtx);
 	if(copyout(ioctl_cmd->data_buffer, user_request->data_buffer,
 	    ioctl_cmd->datasize))
 		error = EINVAL;
diff --git a/sys/dev/ips/ips_pci.c b/sys/dev/ips/ips_pci.c
index 5431c96..3b76c93 100644
--- a/sys/dev/ips/ips_pci.c
+++ b/sys/dev/ips/ips_pci.c
@@ -123,7 +123,7 @@ static int ips_pci_attach(device_t dev)
                 device_printf(dev, "irq allocation failed\n");
                 goto error;
         }
-	if(bus_setup_intr(dev, sc->irqres, INTR_TYPE_BIO, sc->ips_adapter_intr, sc, &sc->irqcookie)){
+	if(bus_setup_intr(dev, sc->irqres, INTR_TYPE_BIO|INTR_MPSAFE, sc->ips_adapter_intr, sc, &sc->irqcookie)){
                 device_printf(dev, "irq setup failed\n");
                 goto error;
         }
@@ -138,8 +138,8 @@ static int ips_pci_attach(device_t dev)
 				/* numsegs   */	IPS_MAX_SG_ELEMENTS,
 				/* maxsegsize*/	BUS_SPACE_MAXSIZE_32BIT,
 				/* flags     */	0,
-				/* lockfunc  */ busdma_lock_mutex,
-				/* lockarg   */ &Giant,
+				/* lockfunc  */ NULL,
+				/* lockarg   */ NULL,
 				&sc->adapter_dmatag) != 0) {
                 printf("IPS can't alloc dma tag\n");
                 goto error;
@@ -147,6 +147,7 @@ static int ips_pci_attach(device_t dev)
 	sc->ips_ich.ich_func = ips_intrhook;
 	sc->ips_ich.ich_arg = sc;
 	mtx_init(&sc->queue_mtx, "IPS bioqueue lock", MTX_DEF, 0);
+	sema_init(&sc->cmd_sema, 0, "IPS Command Semaphore");
 	bioq_init(&sc->queue);
 	if (config_intrhook_establish(&sc->ips_ich) != 0) {
 		printf("IPS can't establish configuration hook\n");
@@ -181,6 +182,8 @@ static int ips_pci_free(ips_softc_t *sc)
         if(sc->iores)
                 bus_release_resource(sc->dev, sc->iotype, sc->rid, sc->iores);
 	sc->configured = 0;
+	mtx_destroy(&sc->queue_mtx);
+	sema_destroy(&sc->cmd_sema);
 	return 0;
 }