Fix the busdma support in twe to support EINPROGRESS and enable it for

use with PAE kernels.

4 files changed, 249 insertions, 111 deletions

diff --git a/sys/dev/twe/twe.c b/sys/dev/twe/twe.c
index e2f5ff1..59c737f 100644
--- a/sys/dev/twe/twe.c
+++ b/sys/dev/twe/twe.c
@@ -66,7 +66,6 @@ static void	twe_del_unit(struct twe_softc *sc, int unit);
 /*
  * Command I/O to controller.
  */
-static int	twe_start(struct twe_request *tr);
 static void	twe_done(struct twe_softc *sc);
 static void	twe_complete(struct twe_softc *sc);
 static int	twe_wait_status(struct twe_softc *sc, u_int32_t status, int timeout);
@@ -117,6 +116,7 @@ int
 twe_setup(struct twe_softc *sc)
 {
     struct twe_request	*tr;
+    TWE_Command		*cmd;
     u_int32_t		status_reg;
     int			i;
 
@@ -136,13 +136,14 @@ twe_setup(struct twe_softc *sc)
      * Allocate request structures up front.
      */
     for (i = 0; i < TWE_Q_LENGTH; i++) {
-	if ((tr = twe_allocate_request(sc)) == NULL)
+	if ((tr = twe_allocate_request(sc, i)) == NULL)
 	    return(ENOMEM);
 	/*
 	 * Set global defaults that won't change.
 	 */
-	tr->tr_command.generic.host_id = sc->twe_host_id;	/* controller-assigned host ID */
-	tr->tr_command.generic.request_id = i;			/* our index number */
+	cmd = TWE_FIND_COMMAND(tr);
+	cmd->generic.host_id = sc->twe_host_id;		/* controller-assigned host ID */
+	cmd->generic.request_id = i;			/* our index number */
 	sc->twe_lookup[i] = tr;
 
 	/*
@@ -369,6 +370,9 @@ twe_startio(struct twe_softc *sc)
 
     debug_called(4);
 
+    if (sc->twe_state & TWE_STATE_FRZN)
+	return;
+
     /* spin until something prevents us from doing any work */
     for (;;) {
 
@@ -393,7 +397,7 @@ twe_startio(struct twe_softc *sc)
 	    tr->tr_private = bp;
 	    tr->tr_data = TWE_BIO_DATA(bp);
 	    tr->tr_length = TWE_BIO_LENGTH(bp);
-	    cmd = &tr->tr_command;
+	    cmd = TWE_FIND_COMMAND(tr);
 	    if (TWE_BIO_IS_READ(bp)) {
 		tr->tr_flags |= TWE_CMD_DATAIN;
 		cmd->io.opcode = TWE_OP_READ;
@@ -407,25 +411,22 @@ twe_startio(struct twe_softc *sc)
 	    cmd->io.unit = TWE_BIO_UNIT(bp);
 	    cmd->io.block_count = (tr->tr_length + TWE_BLOCK_SIZE - 1) / TWE_BLOCK_SIZE;
 	    cmd->io.lba = TWE_BIO_LBA(bp);
-
-	    /* map the command so the controller can work with it */
-	    twe_map_request(tr);
 	}
 	
 	/* did we find something to do? */
 	if (tr == NULL)
 	    break;
 	
-	/* try to give command to controller */
-	error = twe_start(tr);
-
+	/* map the command so the controller can work with it */
+	error = twe_map_request(tr);
 	if (error != 0) {
 	    if (error == EBUSY) {
 		twe_requeue_ready(tr);		/* try it again later */
 		break;				/* don't try anything more for now */
 	    }
+
 	    /* we don't support any other return from twe_start */
-	    twe_panic(sc, "twe_start returned nonsense");
+	    twe_panic(sc, "twe_map_request returned nonsense");
 	}
     }
 }
@@ -448,14 +449,13 @@ twe_dump_blocks(struct twe_softc *sc, int unit,	u_int32_t lba, void *data, int n
     tr->tr_length = nblks * TWE_BLOCK_SIZE;
     tr->tr_flags = TWE_CMD_DATAOUT;
 
-    cmd = &tr->tr_command;
+    cmd = TWE_FIND_COMMAND(tr);
     cmd->io.opcode = TWE_OP_WRITE;
     cmd->io.size = 3;
     cmd->io.unit = unit;
     cmd->io.block_count = nblks;
     cmd->io.lba = lba;
 
-    twe_map_request(tr);
     error = twe_immediate_request(tr);
     if (error == 0)
 	if (twe_report_request(tr))
@@ -468,13 +468,14 @@ twe_dump_blocks(struct twe_softc *sc, int unit,	u_int32_t lba, void *data, int n
  * Handle controller-specific control operations.
  */
 int
-twe_ioctl(struct twe_softc *sc, int cmd, void *addr)
+twe_ioctl(struct twe_softc *sc, int ioctlcmd, void *addr)
 {
     struct twe_usercommand	*tu = (struct twe_usercommand *)addr;
     struct twe_paramcommand	*tp = (struct twe_paramcommand *)addr;
     struct twe_drivecommand	*td = (struct twe_drivecommand *)addr;
     union twe_statrequest	*ts = (union twe_statrequest *)addr;
     TWE_Param			*param;
+    TWE_Command			*cmd;
     void			*data;
     int				*arg = (int *)addr;
     struct twe_request		*tr;
@@ -482,7 +483,7 @@ twe_ioctl(struct twe_softc *sc, int cmd, void *addr)
     int				s, error;
 
     error = 0;
-    switch(cmd) {
+    switch(ioctlcmd) {
 	/* handle a command from userspace */
     case TWEIO_COMMAND:
 	/* get a request */
@@ -493,9 +494,10 @@ twe_ioctl(struct twe_softc *sc, int cmd, void *addr)
 	 * Save the command's request ID, copy the user-supplied command in,
 	 * restore the request ID.
 	 */
-	srid = tr->tr_command.generic.request_id;
-	bcopy(&tu->tu_command, &tr->tr_command, sizeof(TWE_Command));
-	tr->tr_command.generic.request_id = srid;
+	cmd = TWE_FIND_COMMAND(tr);
+	srid = cmd->generic.request_id;
+	bcopy(&tu->tu_command, cmd, sizeof(TWE_Command));
+	cmd->generic.request_id = srid;
 
 	/*
 	 * if there's a data buffer, allocate and copy it in.
@@ -513,11 +515,10 @@ twe_ioctl(struct twe_softc *sc, int cmd, void *addr)
 	}
 
 	/* run the command */
-	twe_map_request(tr);
 	twe_wait_request(tr);
 
 	/* copy the command out again */
-	bcopy(&tr->tr_command, &tu->tu_command, sizeof(TWE_Command));
+	bcopy(cmd, &tu->tu_command, sizeof(TWE_Command));
 	
 	/* if there was a data buffer, copy it out */
 	if (tr->tr_length > 0)
@@ -716,15 +717,12 @@ twe_get_param(struct twe_softc *sc, int table_id, int param_id, size_t param_siz
     tr->tr_flags = TWE_CMD_DATAIN | TWE_CMD_DATAOUT;
 
     /* build the command for the controller */
-    cmd = &tr->tr_command;
+    cmd = TWE_FIND_COMMAND(tr);
     cmd->param.opcode = TWE_OP_GET_PARAM;
     cmd->param.size = 2;
     cmd->param.unit = 0;
     cmd->param.param_count = 1;
 
-    /* map the command/data into controller-visible space */
-    twe_map_request(tr);
-
     /* fill in the outbound parameter data */
     param->table_id = table_id;
     param->parameter_id = param_id;
@@ -742,7 +740,7 @@ twe_get_param(struct twe_softc *sc, int table_id, int param_id, size_t param_siz
 	return(param);
     } else {
 	tr->tr_complete = func;
-	error = twe_start(tr);
+	error = twe_map_request(tr);
 	if (error == 0)
 	    return(func);
     }
@@ -811,15 +809,12 @@ twe_set_param(struct twe_softc *sc, int table_id, int param_id, int param_size,
     tr->tr_flags = TWE_CMD_DATAIN | TWE_CMD_DATAOUT;
 
     /* build the command for the controller */
-    cmd = &tr->tr_command;
+    cmd = TWE_FIND_COMMAND(tr);
     cmd->param.opcode = TWE_OP_SET_PARAM;
     cmd->param.size = 2;
     cmd->param.unit = 0;
     cmd->param.param_count = 1;
 
-    /* map the command/data into controller-visible space */
-    twe_map_request(tr);
-
     /* fill in the outbound parameter data */
     param->table_id = table_id;
     param->parameter_id = param_id;
@@ -860,16 +855,13 @@ twe_init_connection(struct twe_softc *sc, int mode)
 	return(0);
 
     /* build the command */
-    cmd = &tr->tr_command;
+    cmd = TWE_FIND_COMMAND(tr);
     cmd->initconnection.opcode = TWE_OP_INIT_CONNECTION;
     cmd->initconnection.size = 3;
     cmd->initconnection.host_id = 0;
     cmd->initconnection.message_credits = mode;
     cmd->initconnection.response_queue_pointer = 0;
 
-    /* map the command into controller-visible space */
-    twe_map_request(tr);
-
     /* submit the command */
     error = twe_immediate_request(tr);
     /* XXX check command result? */
@@ -913,14 +905,13 @@ twe_wait_request(struct twe_request *tr)
 static int
 twe_immediate_request(struct twe_request *tr)
 {
-    int		error;
 
     debug_called(4);
 
-    error = 0;
+    tr->tr_flags |= TWE_CMD_IMMEDIATE;
+    tr->tr_status = TWE_CMD_BUSY;
+    twe_map_request(tr);
 
-    if ((error = twe_start(tr)) != 0)
-	return(error);
     while (tr->tr_status == TWE_CMD_BUSY){
 	twe_done(tr->tr_sc);
     }
@@ -1023,10 +1014,11 @@ out:
  *
  * Can be called at any interrupt level, with or without interrupts enabled.
  */
-static int
+int
 twe_start(struct twe_request *tr)
 {
     struct twe_softc	*sc = tr->tr_sc;
+    TWE_Command		*cmd;
     int			i, s, done;
     u_int32_t		status_reg;
 
@@ -1034,6 +1026,7 @@ twe_start(struct twe_request *tr)
 
     /* mark the command as currently being processed */
     tr->tr_status = TWE_CMD_BUSY;
+    cmd = TWE_FIND_COMMAND(tr);
 
     /* 
      * Spin briefly waiting for the controller to come ready 
@@ -1049,17 +1042,18 @@ twe_start(struct twe_request *tr)
 	twe_check_bits(sc, status_reg);
 
 	if (!(status_reg & TWE_STATUS_COMMAND_QUEUE_FULL)) {
-	    TWE_COMMAND_QUEUE(sc, tr->tr_cmdphys);
+	    twe_enqueue_busy(tr);
+
+	    TWE_COMMAND_QUEUE(sc, TWE_FIND_COMMANDPHYS(tr));
 	    done = 1;
 	    /* move command to work queue */
-	    twe_enqueue_busy(tr);
 #ifdef TWE_DEBUG
 	    if (tr->tr_complete != NULL) {
-		debug(3, "queued request %d with callback %p", tr->tr_command.generic.request_id, tr->tr_complete);
+		debug(3, "queued request %d with callback %p", cmd->generic.request_id, tr->tr_complete);
 	    } else if (tr->tr_flags & TWE_CMD_SLEEPER) {
-		debug(3, "queued request %d with wait channel %p", tr->tr_command.generic.request_id, tr);
+		debug(3, "queued request %d with wait channel %p", cmd->generic.request_id, tr);
 	    } else {
-		debug(3, "queued request %d for polling caller", tr->tr_command.generic.request_id);
+		debug(3, "queued request %d for polling caller", cmd->generic.request_id);
 	    }
 #endif
 	}
@@ -1088,6 +1082,7 @@ static void
 twe_done(struct twe_softc *sc)
 {
     TWE_Response_Queue	rq;
+    TWE_Command		*cmd;
     struct twe_request	*tr;
     int			s, found;
     u_int32_t		status_reg;
@@ -1105,11 +1100,12 @@ twe_done(struct twe_softc *sc)
 	    found = 1;
 	    rq = TWE_RESPONSE_QUEUE(sc);
 	    tr = sc->twe_lookup[rq.u.response_id];	/* find command */
+	    cmd = TWE_FIND_COMMAND(tr);
 	    if (tr->tr_status != TWE_CMD_BUSY)
 		twe_printf(sc, "completion event for nonbusy command\n");
 	    tr->tr_status = TWE_CMD_COMPLETE;
 	    debug(3, "completed request id %d with status %d", 
-		  tr->tr_command.generic.request_id, tr->tr_command.generic.status);
+		  cmd->generic.request_id, cmd->generic.status);
 	    /* move to completed queue */
 	    twe_remove_busy(tr);
 	    twe_enqueue_complete(tr);
@@ -1144,7 +1140,6 @@ twe_complete(struct twe_softc *sc)
      * Pull commands off the completed list, dispatch them appropriately
      */
     while ((tr = twe_dequeue_complete(sc)) != NULL) {
-
 	/* unmap the command's data buffer */
 	twe_unmap_request(tr);
 
@@ -1161,6 +1156,8 @@ twe_complete(struct twe_softc *sc)
 	    debug(2, "command left for owner");
 	}
     }   
+
+    sc->twe_state &= ~TWE_STATE_FRZN;
 }
 
 /********************************************************************************
@@ -1502,6 +1499,7 @@ twe_wait_aen(struct twe_softc *sc, int aen, int timeout)
 static int
 twe_get_request(struct twe_softc *sc, struct twe_request **tr)
 {
+    TWE_Command		*cmd;
     debug_called(4);
 
     /* try to reuse an old buffer */
@@ -1509,13 +1507,14 @@ twe_get_request(struct twe_softc *sc, struct twe_request **tr)
 
     /* initialise some fields to their defaults */
     if (*tr != NULL) {
+	cmd = TWE_FIND_COMMAND(*tr);
 	(*tr)->tr_data = NULL;
 	(*tr)->tr_private = NULL;
 	(*tr)->tr_status = TWE_CMD_SETUP;		/* command is in setup phase */
 	(*tr)->tr_flags = 0;
 	(*tr)->tr_complete = NULL;
-	(*tr)->tr_command.generic.status = 0;		/* before submission to controller */
-	(*tr)->tr_command.generic.flags = 0;		/* not used */
+	cmd->generic.status = 0;			/* before submission to controller */
+	cmd->generic.flags = 0;				/* not used */
     }
     return(*tr == NULL);
 }
@@ -1638,7 +1637,7 @@ twe_check_bits(struct twe_softc *sc, u_int32_t status_reg)
 	    twe_clear_pci_parity_error(sc);
 	}
 	if (status_reg & TWE_STATUS_PCI_ABORT) {
-	    twe_printf(sc, "PCI abort, clearing.");
+	    twe_printf(sc, "PCI abort, clearing.\n");
 	    twe_clear_pci_abort(sc);
 	}
     }
@@ -1705,7 +1704,7 @@ static int
 twe_report_request(struct twe_request *tr)
 {
     struct twe_softc	*sc = tr->tr_sc;
-    TWE_Command		*cmd = &tr->tr_command;
+    TWE_Command		*cmd = TWE_FIND_COMMAND(tr);
     int			result = 0;
 
     /*
@@ -1791,7 +1790,7 @@ twe_panic(struct twe_softc *sc, char *reason)
 #endif
 }
 
-#if 0
+#ifdef TWE_DEBUG
 /********************************************************************************
  * Print a request/command in human-readable format.
  */
@@ -1799,7 +1798,7 @@ static void
 twe_print_request(struct twe_request *tr)
 {
     struct twe_softc	*sc = tr->tr_sc;
-    TWE_Command	*cmd = &tr->tr_command;
+    TWE_Command	*cmd = TWE_FIND_COMMAND(tr);
     int		i;
 
     twe_printf(sc, "CMD: request_id %d  opcode <%s>  size %d  unit %d  host_id %d\n", 
@@ -1833,7 +1832,7 @@ twe_print_request(struct twe_request *tr)
 	break;
     }
     twe_printf(sc, " tr_command %p/0x%x  tr_data %p/0x%x,%d\n", 
-	       tr, tr->tr_cmdphys, tr->tr_data, tr->tr_dataphys, tr->tr_length);
+	       tr, TWE_FIND_COMMANDPHYS(tr), tr->tr_data, tr->tr_dataphys, tr->tr_length);
     twe_printf(sc, " tr_status %d  tr_flags 0x%x  tr_complete %p  tr_private %p\n", 
 	       tr->tr_status, tr->tr_flags, tr->tr_complete, tr->tr_private);
 }
diff --git a/sys/dev/twe/twe_compat.h b/sys/dev/twe/twe_compat.h
index 69db4f2..181c79f 100644
--- a/sys/dev/twe/twe_compat.h
+++ b/sys/dev/twe/twe_compat.h
@@ -37,6 +37,7 @@
 #define TWE_SUPPORTED_PLATFORM
 
 #include <sys/param.h>
+#include <sys/endian.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
 #include <sys/kernel.h>
@@ -105,6 +106,8 @@
  * FreeBSD-specific softc elements
  */
 #define TWE_PLATFORM_SOFTC								\
+    bus_dmamap_t		twe_cmdmap;	/* DMA map for command */				\
+    u_int32_t			twe_cmdphys;	/* address of command in controller space */		\
     device_t			twe_dev;		/* bus device */		\
     dev_t			twe_dev_t;		/* control device */		\
     struct resource		*twe_io;		/* register interface window */	\
@@ -112,9 +115,14 @@
     bus_space_tag_t		twe_btag;		/* bus space tag */		\
     bus_dma_tag_t		twe_parent_dmat;	/* parent DMA tag */		\
     bus_dma_tag_t		twe_buffer_dmat;	/* data buffer DMA tag */	\
+    bus_dma_tag_t		twe_cmd_dmat;		/* command buffer DMA tag */	\
+    bus_dma_tag_t		twe_immediate_dmat;	/* command buffer DMA tag */	\
     struct resource		*twe_irq;		/* interrupt */			\
     void			*twe_intr;		/* interrupt handle */		\
     struct intr_config_hook	twe_ich;		/* delayed-startup hook */	\
+    void			*twe_cmd;		/* command structures */	\
+    void			*twe_immediate;		/* immediate commands */	\
+    bus_dmamap_t		twe_immediate_map;					\
     struct sysctl_ctx_list	sysctl_ctx;						\
     struct sysctl_oid		*sysctl_tree;
 
@@ -122,8 +130,6 @@
  * FreeBSD-specific request elements
  */
 #define TWE_PLATFORM_REQUEST										\
-    bus_dmamap_t		tr_cmdmap;	/* DMA map for command */				\
-    u_int32_t			tr_cmdphys;	/* address of command in controller space */		\
     bus_dmamap_t		tr_dmamap;	/* DMA map for data */					\
     u_int32_t			tr_dataphys;	/* data buffer base address in controller space */
 
diff --git a/sys/dev/twe/twe_freebsd.c b/sys/dev/twe/twe_freebsd.c
index 75c5003..a1b871f 100644
--- a/sys/dev/twe/twe_freebsd.c
+++ b/sys/dev/twe/twe_freebsd.c
@@ -57,6 +57,9 @@ static u_int32_t	twed_bio_out;
 #define TWED_BIO_OUT
 #endif
 
+static void	twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
+static void	twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
+
 /********************************************************************************
  ********************************************************************************
                                                          Control device interface
@@ -269,18 +272,50 @@ twe_attach(device_t dev)
     }
 
     /*
+     * Create DMA tag for mapping command's into controller-addressable space.
+     */
+    if (bus_dma_tag_create(sc->twe_parent_dmat, 	/* parent */
+			   1, 0, 			/* alignment, boundary */
+			   BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
+			   BUS_SPACE_MAXADDR, 		/* highaddr */
+			   NULL, NULL, 			/* filter, filterarg */
+			   sizeof(TWE_Command) *
+			   TWE_Q_LENGTH, 1,		/* maxsize, nsegments */
+			   BUS_SPACE_MAXSIZE_32BIT,	/* maxsegsize */
+			   BUS_DMA_ALLOCNOW,		/* flags */
+			   NULL,			/* lockfunc */
+			   NULL,			/* lockarg */
+			   &sc->twe_cmd_dmat)) {
+	twe_printf(sc, "can't allocate data buffer DMA tag\n");
+	twe_free(sc);
+	return(ENOMEM);
+    }
+    /*
+     * Allocate memory and make it available for DMA.
+     */
+    if (bus_dmamem_alloc(sc->twe_cmd_dmat, (void **)&sc->twe_cmd,
+			 BUS_DMA_NOWAIT, &sc->twe_cmdmap)) {
+	twe_printf(sc, "can't allocate command memory\n");
+	return(ENOMEM);
+    }
+    bus_dmamap_load(sc->twe_cmd_dmat, sc->twe_cmdmap, sc->twe_cmd,
+		    sizeof(TWE_Command) * TWE_Q_LENGTH,
+		    twe_setup_request_dmamap, sc, 0);
+    bzero(sc->twe_cmd, sizeof(TWE_Command) * TWE_Q_LENGTH);
+
+    /*
      * Create DMA tag for mapping objects into controller-addressable space.
      */
     if (bus_dma_tag_create(sc->twe_parent_dmat, 	/* parent */
 			   1, 0, 			/* alignment, boundary */
-			   BUS_SPACE_MAXADDR,		/* lowaddr */
+			   BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
 			   BUS_SPACE_MAXADDR, 		/* highaddr */
 			   NULL, NULL, 			/* filter, filterarg */
 			   MAXBSIZE, TWE_MAX_SGL_LENGTH,/* maxsize, nsegments */
 			   BUS_SPACE_MAXSIZE_32BIT,	/* maxsegsize */
 			   0,				/* flags */
-			   busdma_lock_mutex,			/* lockfunc */
-			   &Giant,				/* lockarg */
+			   busdma_lock_mutex,		/* lockfunc */
+			   &Giant,			/* lockarg */
 			   &sc->twe_buffer_dmat)) {
 	twe_printf(sc, "can't allocate data buffer DMA tag\n");
 	twe_free(sc);
@@ -288,10 +323,39 @@ twe_attach(device_t dev)
     }
 
     /*
+     * Create DMA tag for mapping objects into controller-addressable space.
+     */
+    if (bus_dma_tag_create(sc->twe_parent_dmat, 	/* parent */
+			   1, 0, 			/* alignment, boundary */
+			   BUS_SPACE_MAXADDR_32BIT,	/* lowaddr */
+			   BUS_SPACE_MAXADDR, 		/* highaddr */
+			   NULL, NULL, 			/* filter, filterarg */
+			   MAXBSIZE, 1,			/* maxsize, nsegments */
+			   BUS_SPACE_MAXSIZE_32BIT,	/* maxsegsize */
+			   BUS_DMA_ALLOCNOW,		/* flags */
+			   NULL,			/* lockfunc */
+			   NULL,			/* lockarg */
+			   &sc->twe_immediate_dmat)) {
+	twe_printf(sc, "can't allocate data buffer DMA tag\n");
+	twe_free(sc);
+	return(ENOMEM);
+    }
+    /*
+     * Allocate memory for requests which cannot sleep or support continuation.
+     */
+     if (bus_dmamem_alloc(sc->twe_immediate_dmat, (void **)&sc->twe_immediate,
+			  BUS_DMA_NOWAIT, &sc->twe_immediate_map)) {
+	twe_printf(sc, "can't allocate memory for immediate requests\n");
+	return(ENOMEM);
+     }
+
+    /*
      * Initialise the controller and driver core.
      */
-    if ((error = twe_setup(sc)))
+    if ((error = twe_setup(sc))) {
+	twe_free(sc);
 	return(error);
+    }
 
     /*
      * Print some information about the controller and configuration.
@@ -337,6 +401,20 @@ twe_free(struct twe_softc *sc)
     while ((tr = twe_dequeue_free(sc)) != NULL)
 	twe_free_request(tr);
 
+    if (sc->twe_cmd != NULL) {
+	bus_dmamap_unload(sc->twe_cmd_dmat, sc->twe_cmdmap);
+	bus_dmamem_free(sc->twe_cmd_dmat, sc->twe_cmd, sc->twe_cmdmap);
+    }
+
+    if (sc->twe_immediate != NULL) {
+	bus_dmamap_unload(sc->twe_immediate_dmat, sc->twe_immediate_map);
+	bus_dmamem_free(sc->twe_immediate_dmat, sc->twe_immediate,
+			sc->twe_immediate_map);
+    }
+
+    if (sc->twe_immediate_dmat)
+	bus_dma_tag_destroy(sc->twe_immediate_dmat);
+
     /* destroy the data-transfer DMA tag */
     if (sc->twe_buffer_dmat)
 	bus_dma_tag_destroy(sc->twe_buffer_dmat);
@@ -486,7 +564,7 @@ twe_intrhook(void *arg)
 /********************************************************************************
  * Given a detected drive, attach it to the bio interface.
  *
- * This is called from twe_add_unit.
+ * This is called from twe_init.
  */
 void
 twe_attach_drive(struct twe_softc *sc, struct twe_drive *dr)
@@ -784,30 +862,26 @@ twed_detach(device_t dev)
  ********************************************************************************
  ********************************************************************************/
 
-static void	twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
-static void	twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
-
 /********************************************************************************
  * Allocate a command buffer
  */
 MALLOC_DEFINE(TWE_MALLOC_CLASS, "twe commands", "twe commands");
 
 struct twe_request *
-twe_allocate_request(struct twe_softc *sc)
+twe_allocate_request(struct twe_softc *sc, int tag)
 {
     struct twe_request	*tr;
 
-    if ((tr = malloc(sizeof(struct twe_request), TWE_MALLOC_CLASS, M_NOWAIT)) == NULL)
+    if ((tr = malloc(sizeof(struct twe_request), TWE_MALLOC_CLASS, M_WAITOK)) == NULL) {
+	twe_printf(sc, "unable to allocate memory for tag %d\n", tag);
 	return(NULL);
+    }
     bzero(tr, sizeof(*tr));
     tr->tr_sc = sc;
-    if (bus_dmamap_create(sc->twe_buffer_dmat, 0, &tr->tr_cmdmap)) {
-	twe_free_request(tr);
-	return(NULL);
-    }
+    tr->tr_tag = tag;
     if (bus_dmamap_create(sc->twe_buffer_dmat, 0, &tr->tr_dmamap)) {
-	bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_cmdmap);
 	twe_free_request(tr);
+	twe_printf(sc, "unable to allocate dmamap for tag %d\n", tag);
 	return(NULL);
     }    
     return(tr);
@@ -823,7 +897,6 @@ twe_free_request(struct twe_request *tr)
     
     debug_called(4);
 
-    bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_cmdmap);
     bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_dmamap);
     free(tr, TWE_MALLOC_CLASS);
 }
@@ -855,15 +928,21 @@ static void
 twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
 {
     struct twe_request	*tr = (struct twe_request *)arg;
-    TWE_Command		*cmd = &tr->tr_command;
+    struct twe_softc	*sc = tr->tr_sc;
+    TWE_Command		*cmd = TWE_FIND_COMMAND(tr);
 
     debug_called(4);
 
+    if (tr->tr_flags & TWE_CMD_MAPPED)
+	panic("already mapped command");
+
+    tr->tr_flags |= TWE_CMD_MAPPED;
+
     /* save base of first segment in command (applicable if there only one segment) */
     tr->tr_dataphys = segs[0].ds_addr;
 
     /* correct command size for s/g list size */
-    tr->tr_command.generic.size += 2 * nsegments;
+    cmd->generic.size += 2 * nsegments;
 
     /*
      * Due to the fact that parameter and I/O commands have the scatter/gather list in
@@ -904,38 +983,66 @@ twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int err
 	    break;
 	}
     }
+
+    if (tr->tr_flags & TWE_CMD_DATAIN) {
+	if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
+	    bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
+			    BUS_DMASYNC_PREREAD);
+	} else {
+	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
+			    BUS_DMASYNC_PREREAD);
+	}
+    }
+
+    if (tr->tr_flags & TWE_CMD_DATAOUT) {
+	/*
+	 * if we're using an alignment buffer, and we're writing data
+	 * copy the real data out
+	 */
+	if (tr->tr_flags & TWE_CMD_ALIGNBUF)
+	    bcopy(tr->tr_realdata, tr->tr_data, tr->tr_length);
+
+	if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
+	    bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
+			    BUS_DMASYNC_PREWRITE);
+	} else {
+	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
+			    BUS_DMASYNC_PREWRITE);
+	}
+    }
+
+    if (twe_start(tr) == EBUSY)
+	panic("EBUSY should not happen");
 }
 
 static void
 twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
 {
-    struct twe_request	*tr = (struct twe_request *)arg;
+    struct twe_softc	*sc = (struct twe_softc *)arg;
 
     debug_called(4);
 
     /* command can't cross a page boundary */
-    tr->tr_cmdphys = segs[0].ds_addr;
+    sc->twe_cmdphys = segs[0].ds_addr;
 }
 
-void
+int
 twe_map_request(struct twe_request *tr)
 {
     struct twe_softc	*sc = tr->tr_sc;
+    int			error = 0;
 
     debug_called(4);
 
+    if (sc->twe_state & TWE_STATE_FRZN)
+	return (EBUSY);
 
-    /*
-     * Map the command into bus space.
-     */
-    bus_dmamap_load(sc->twe_buffer_dmat, tr->tr_cmdmap, &tr->tr_command, sizeof(tr->tr_command), 
-		    twe_setup_request_dmamap, tr, 0);
-    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_cmdmap, BUS_DMASYNC_PREWRITE);
+    bus_dmamap_sync(sc->twe_cmd_dmat, sc->twe_cmdmap, BUS_DMASYNC_PREWRITE);
 
     /*
      * If the command involves data, map that too.
      */
-    if (tr->tr_data != NULL) {
+    if (tr->tr_data != NULL && ((tr->tr_flags & TWE_CMD_MAPPED) == 0)) {
 
 	/* 
 	 * Data must be 64-byte aligned; allocate a fixup buffer if it's not.
@@ -948,18 +1055,23 @@ twe_map_request(struct twe_request *tr)
 	
 	/*
 	 * Map the data buffer into bus space and build the s/g list.
-	 */
-	bus_dmamap_load(sc->twe_buffer_dmat, tr->tr_dmamap, tr->tr_data, tr->tr_length, 
-			twe_setup_data_dmamap, tr, 0);
-	if (tr->tr_flags & TWE_CMD_DATAIN)
-	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_PREREAD);
-	if (tr->tr_flags & TWE_CMD_DATAOUT) {
-	    /* if we're using an alignment buffer, and we're writing data, copy the real data out */
-	    if (tr->tr_flags & TWE_CMD_ALIGNBUF)
-		bcopy(tr->tr_realdata, tr->tr_data, tr->tr_length);
-	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_PREWRITE);
+	*/
+	if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
+	    bcopy(tr->tr_data, sc->twe_immediate, tr->tr_length);
+	    bus_dmamap_load(sc->twe_immediate_dmat, sc->twe_immediate_map, sc->twe_immediate,
+			    tr->tr_length, twe_setup_data_dmamap, tr, 0);
+	} else {
+	    error = bus_dmamap_load(sc->twe_buffer_dmat, tr->tr_dmamap, tr->tr_data, tr->tr_length, 
+				    twe_setup_data_dmamap, tr, 0);
 	}
-    }
+	if (error == EINPROGRESS) {
+	    sc->twe_state |= TWE_STATE_FRZN;
+	    error = 0;
+	}
+    } else
+	error = twe_start(tr);
+
+    return(error);
 }
 
 void
@@ -969,27 +1081,41 @@ twe_unmap_request(struct twe_request *tr)
 
     debug_called(4);
 
-    /*
-     * Unmap the command from bus space.
-     */
-    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_cmdmap, BUS_DMASYNC_POSTWRITE);
-    bus_dmamap_unload(sc->twe_buffer_dmat, tr->tr_cmdmap); 
+    bus_dmamap_sync(sc->twe_cmd_dmat, sc->twe_cmdmap, BUS_DMASYNC_POSTWRITE);
 
     /*
      * If the command involved data, unmap that too.
      */
     if (tr->tr_data != NULL) {
-	
 	if (tr->tr_flags & TWE_CMD_DATAIN) {
-	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_POSTREAD);
+	    if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
+		bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
+				BUS_DMASYNC_POSTREAD);
+	    } else {
+		bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
+				BUS_DMASYNC_POSTREAD);
+	    }
+
 	    /* if we're using an alignment buffer, and we're reading data, copy the real data in */
 	    if (tr->tr_flags & TWE_CMD_ALIGNBUF)
 		bcopy(tr->tr_data, tr->tr_realdata, tr->tr_length);
 	}
-	if (tr->tr_flags & TWE_CMD_DATAOUT)
-	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_POSTWRITE);
+	if (tr->tr_flags & TWE_CMD_DATAOUT) {
+	    if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
+		bus_dmamap_sync(sc->twe_immediate_dmat, sc->twe_immediate_map,
+				BUS_DMASYNC_POSTWRITE);
+	    } else {
+		bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap,
+				BUS_DMASYNC_POSTWRITE);
+	    }
+	}
 
-	bus_dmamap_unload(sc->twe_buffer_dmat, tr->tr_dmamap); 
+	if (tr->tr_flags & TWE_CMD_IMMEDIATE) {
+	    bcopy(sc->twe_immediate, tr->tr_data, tr->tr_length);
+	    bus_dmamap_unload(sc->twe_immediate_dmat, sc->twe_immediate_map);
+	} else {
+	    bus_dmamap_unload(sc->twe_buffer_dmat, tr->tr_dmamap); 
+	}
     }
 
     /* free alignment buffer if it was used */
diff --git a/sys/dev/twe/twevar.h b/sys/dev/twe/twevar.h
index d28bd35..22fb966 100644
--- a/sys/dev/twe/twevar.h
+++ b/sys/dev/twe/twevar.h
@@ -70,9 +70,7 @@ struct twe_drive
  */
 struct twe_request
 {
-    /* controller command */
-    TWE_Command			tr_command;	/* command as submitted to controller */
-
+    int				tr_tag;
     /* command payload */
     void			*tr_data;	/* data buffer */
     void			*tr_realdata;	/* copy of real data buffer pointer for alignment fixup */
@@ -89,12 +87,20 @@ struct twe_request
 #define TWE_CMD_DATAOUT		(1<<1)
 #define TWE_CMD_ALIGNBUF	(1<<2)	/* data in bio is misaligned, have to copy to/from private buffer */
 #define TWE_CMD_SLEEPER		(1<<3)	/* owner is sleeping on this command */
+#define TWE_CMD_IMMEDIATE	(1<<4)	/* immediate request */
+#define TWE_CMD_MAPPED		(1<<5)
     void			(* tr_complete)(struct twe_request *tr);	/* completion handler */
     void			*tr_private;	/* submitter-private data or wait channel */
 
     TWE_PLATFORM_REQUEST	/* platform-specific request elements */
 };
 
+#define TWE_FIND_COMMAND(tr)					\
+	(TWE_Command *)((u_int8_t *)(tr)->tr_sc->twe_cmd +	\
+			((tr)->tr_tag * sizeof(TWE_Command)))
+#define TWE_FIND_COMMANDPHYS(tr)	((tr)->tr_sc->twe_cmdphys +	\
+					 ((tr)->tr_tag * sizeof(TWE_Command)))
+	
 /*
  * Per-controller state.
  */
@@ -120,6 +126,7 @@ struct twe_softc
 #define TWE_STATE_SHUTDOWN	(1<<1)	/* controller is shut down */
 #define TWE_STATE_OPEN		(1<<2)	/* control device is open */
 #define TWE_STATE_SUSPEND	(1<<3)	/* controller is suspended */
+#define TWE_STATE_FRZN		(1<<4)
     int			twe_host_id;
     struct twe_qstat	twe_qstat[TWEQ_COUNT];	/* queue statistics */
 
@@ -134,6 +141,7 @@ extern void	twe_init(struct twe_softc *sc);			/* init controller */
 extern void	twe_deinit(struct twe_softc *sc);		/* stop controller */
 extern void	twe_intr(struct twe_softc *sc);			/* hardware interrupt signalled */
 extern void	twe_startio(struct twe_softc *sc);
+extern int	twe_start(struct twe_request *tr);
 extern int	twe_dump_blocks(struct twe_softc *sc, int unit,	/* crashdump block write */
 				u_int32_t lba, void *data, int nblks);
 extern int	twe_ioctl(struct twe_softc *sc, int cmd,
@@ -144,15 +152,15 @@ extern void	twe_enable_interrupts(struct twe_softc *sc);	/* enable controller in
 extern void	twe_disable_interrupts(struct twe_softc *sc);	/* disable controller interrupts */
 
 extern void	twe_attach_drive(struct twe_softc *sc,
-					 struct twe_drive *dr); /* attach drive when found in twe_add_unit */
+					 struct twe_drive *dr); /* attach drive when found in twe_init */
 extern void	twe_detach_drive(struct twe_softc *sc,
 					 int unit);		/* detach drive */
 extern void	twe_clear_pci_parity_error(struct twe_softc *sc);
 extern void	twe_clear_pci_abort(struct twe_softc *sc);
 extern void	twed_intr(twe_bio *bp);				/* return bio from core */
-extern struct twe_request *twe_allocate_request(struct twe_softc *sc);	/* allocate request structure */
+extern struct twe_request *twe_allocate_request(struct twe_softc *sc, int tag);	/* allocate request structure */
 extern void	twe_free_request(struct twe_request *tr);	/* free request structure */
-extern void	twe_map_request(struct twe_request *tr);	/* make request visible to controller, do s/g */
+extern int	twe_map_request(struct twe_request *tr);	/* make request visible to controller, do s/g */
 extern void	twe_unmap_request(struct twe_request *tr);	/* cleanup after transfer, unmap */
 
 /********************************************************************************
@@ -232,7 +240,6 @@ TWEQ_REQUEST_QUEUE(ready, TWEQ_READY)
 TWEQ_REQUEST_QUEUE(busy, TWEQ_BUSY)
 TWEQ_REQUEST_QUEUE(complete, TWEQ_COMPLETE)
 
-
 /*
  * outstanding bio queue
  */