Major update to the 'twe' driver.

 - Layout reorganisation to enhance portability.  The driver now has
   a relatively MI 'core' and a FreeBSD-specific layer over the top.
   Since the NetBSD people have already done their own port, this is
   largely just to help me with the BSD/OS port.

 - Request ID allocation changed to improve performance (I'd been
   considering switching to this approach after having failed to come
   up with a better way to dynamically allocate request IDs, and seeing
   Andy Doran use it in the NetBSD port of the driver convinced me
   that I was wasting my time doing it any other way).  Now we just
   allocate all the requests up front.

 - Maximum request count bumped back to 255 after characterisation
   of a firmware issue (off-by-one causing it to crash with 256
   outstanding commands).

 - Control interface implemented.  This allows 3ware's '3dm' utility to
   talk to the controller.  3dm will be available from 3ware shortly.

 - Controller soft-reset feature added; if the controller signals a
   firmware or protocol error, the controller will be reset and all
   outstanding commands will be retried.

10 files changed, 2735 insertions, 1415 deletions

diff --git a/sys/conf/files b/sys/conf/files
index 4541998..1fe5d0f 100644
--- a/sys/conf/files
+++ b/sys/conf/files
@@ -356,7 +356,7 @@ dev/sym/sym_hipd.c	optional sym					\
 	dependency	"$S/dev/sym/sym_{conf,defs}.h"
 dev/tdfx/tdfx_pci.c	optional	tdfx
 dev/twe/twe.c		optional twe
-dev/twe/twe_disk.c	optional twe
+dev/twe/twe_freebsd.c	optional twe
 #
 # USB support
 dev/usb/usb_if.m	optional usb
diff --git a/sys/dev/twe/twe.c b/sys/dev/twe/twe.c
index 067e6f9..ecd4dfa 100644
--- a/sys/dev/twe/twe.c
+++ b/sys/dev/twe/twe.c
@@ -31,114 +31,73 @@
  * Driver for the 3ware Escalade family of IDE RAID controllers.
  */
 
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/malloc.h>
-#include <sys/kernel.h>
-
-#if __FreeBSD_version < 500000
-# include <dev/twe/twe_compat.h>
-#else
-# include <sys/bio.h>
-#endif
-#include <sys/bus.h>
-#include <sys/conf.h>
-#include <sys/devicestat.h>
-#include <sys/disk.h>
-#include <sys/stat.h>
-
-#include <machine/bus_pio.h>
-#include <machine/bus.h>
-#include <machine/resource.h>
-#include <sys/rman.h>
-
-#include <pci/pcireg.h>
-#include <pci/pcivar.h>
-
+#include <dev/twe/twe_compat.h>
 #include <dev/twe/twereg.h>
+#include <dev/twe/tweio.h>
 #include <dev/twe/twevar.h>
-
-/*
- * Initialisation, bus interface.
- */
-static int			twe_probe(device_t dev);
-static int			twe_attach(device_t dev);
-static void			twe_free(struct twe_softc *sc);
-static void			twe_startup(void *arg);
-static int			twe_detach(device_t dev);
-static int			twe_shutdown(device_t dev);
-static int			twe_suspend(device_t dev);
-static int			twe_resume(device_t dev);
-static void			twe_intr(void *arg);
-
-/*
- * Control device.
- */
-static	d_open_t		twe_open;
-static	d_close_t		twe_close;
-static	d_ioctl_t		twe_ioctl;
+#define TWE_DEFINE_TABLES
+#include <dev/twe/twe_tables.h>
 
 /*
  * Command submission.
  */
-static void			*twe_get_param(struct twe_softc *sc, int table_id, int parameter_id, size_t size, 
+static int	twe_get_param_1(struct twe_softc *sc, int table_id, int param_id, u_int8_t *result);
+static int	twe_get_param_2(struct twe_softc *sc, int table_id, int param_id, u_int16_t *result);
+static int	twe_get_param_4(struct twe_softc *sc, int table_id, int param_id, u_int32_t *result);
+static void	*twe_get_param(struct twe_softc *sc, int table_id, int parameter_id, size_t size, 
 					       void (* func)(struct twe_request *tr));
-static int			twe_init_connection(struct twe_softc *sc);
-/*static int			twe_wait_request(struct twe_request *tr);*/
-static int			twe_immediate_request(struct twe_request *tr);
-static void			twe_startio(struct twe_softc *sc);
-static void			twe_completeio(struct twe_request *tr);
+static int	twe_set_param_1(struct twe_softc *sc, int table_id, int param_id, u_int8_t value);
+static int	twe_set_param_2(struct twe_softc *sc, int table_id, int param_id, u_int16_t value);
+static int	twe_set_param_4(struct twe_softc *sc, int table_id, int param_id, u_int32_t value);
+static int	twe_set_param(struct twe_softc *sc, int table_id, int param_id, int param_size, 
+					      void *data);
+static int	twe_init_connection(struct twe_softc *sc, int mode);
+static int	twe_wait_request(struct twe_request *tr);
+static int	twe_immediate_request(struct twe_request *tr);
+static void	twe_startio(struct twe_softc *sc);
+static void	twe_completeio(struct twe_request *tr);
+static void	twe_reset(struct twe_softc *sc);
 
 /*
  * Command I/O to controller.
  */
-static int			twe_start(struct twe_request *tr);
-static void			twe_done(struct twe_softc *sc);
-static void			twe_complete(struct twe_softc *sc);
-static int			twe_wait_status(struct twe_softc *sc, u_int32_t status, int timeout);
-static int			twe_drain_response_queue(struct twe_softc *sc);
-static int			twe_check_bits(struct twe_softc *sc, u_int32_t status_reg);
+static int	twe_start(struct twe_request *tr);
+static void	twe_done(struct twe_softc *sc);
+static void	twe_complete(struct twe_softc *sc);
+static int	twe_wait_status(struct twe_softc *sc, u_int32_t status, int timeout);
+static int	twe_drain_response_queue(struct twe_softc *sc);
+static int	twe_check_bits(struct twe_softc *sc, u_int32_t status_reg);
+static int	twe_soft_reset(struct twe_softc *sc);
 
 /*
  * Interrupt handling.
  */
-static void			twe_host_intr(struct twe_softc *sc);
-static void			twe_attention_intr(struct twe_softc *sc);
-static void			twe_command_intr(struct twe_softc *sc);
-static void			twe_enable_interrupts(struct twe_softc *sc);
-static void			twe_disable_interrupts(struct twe_softc *sc);
+static void	twe_host_intr(struct twe_softc *sc);
+static void	twe_attention_intr(struct twe_softc *sc);
+static void	twe_command_intr(struct twe_softc *sc);
 
 /*
  * Asynchronous event handling.
  */
-static int			twe_fetch_aen(struct twe_softc *sc);
-static void			twe_handle_aen(struct twe_request *tr);
-static void			twe_enqueue_aen(struct twe_softc *sc, u_int16_t aen);
-/*static int			twe_dequeue_aen(struct twe_softc *sc);*/
-static int			twe_drain_aen_queue(struct twe_softc *sc);
-static int			twe_find_aen(struct twe_softc *sc, u_int16_t aen);
+static int	twe_fetch_aen(struct twe_softc *sc);
+static void	twe_handle_aen(struct twe_request *tr);
+static void	twe_enqueue_aen(struct twe_softc *sc, u_int16_t aen);
+static int	twe_dequeue_aen(struct twe_softc *sc);
+static int	twe_drain_aen_queue(struct twe_softc *sc);
+static int	twe_find_aen(struct twe_softc *sc, u_int16_t aen);
 
 /*
  * Command buffer management.
  */
-static struct twe_request	*twe_get_request(struct twe_softc *sc);
-static void			twe_release_request(struct twe_request *tr);
-static void			twe_free_request(struct twe_request *tr);
-static int			twe_get_requestid(struct twe_request *tr);
-static void			twe_release_requestid(struct twe_request *tr);
-static void			twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
-static void			twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
-static void			twe_map_request(struct twe_request *tr);
-static void			twe_unmap_request(struct twe_request *tr);
+static int	twe_get_request(struct twe_softc *sc, struct twe_request **tr);
+static void	twe_release_request(struct twe_request *tr);
 
 /*
  * Debugging.
  */
-static char 			*twe_name_aen(u_int16_t aen);
-#if 0
-static void			twe_print_request(struct twe_request *tr);
-void				twe_report(void);
-#endif
+static char 	*twe_format_aen(struct twe_softc *sc, u_int16_t aen);
+static int	twe_request_qlen(struct twe_request *tr);
+static void	twe_panic(struct twe_softc *sc, char *reason);
 
 /********************************************************************************
  ********************************************************************************
@@ -146,228 +105,74 @@ void				twe_report(void);
  ********************************************************************************
  ********************************************************************************/
 
-devclass_t	twe_devclass;
-
-static device_method_t twe_methods[] = {
-    /* Device interface */
-    DEVMETHOD(device_probe,	twe_probe),
-    DEVMETHOD(device_attach,	twe_attach),
-    DEVMETHOD(device_detach,	twe_detach),
-    DEVMETHOD(device_shutdown,	twe_shutdown),
-    DEVMETHOD(device_suspend,	twe_suspend),
-    DEVMETHOD(device_resume,	twe_resume),
-
-    DEVMETHOD(bus_print_child,	bus_generic_print_child),
-    DEVMETHOD(bus_driver_added,	bus_generic_driver_added),
-    { 0, 0 }
-};
-
-static driver_t twe_pci_driver = {
-	"twe",
-	twe_methods,
-	sizeof(struct twe_softc)
-};
-
-DRIVER_MODULE(tw, pci, twe_pci_driver, twe_devclass, 0, 0);
-
-#define TWE_CDEV_MAJOR  146
-
-static struct cdevsw twe_cdevsw = {
-                /* open */      twe_open,
-                /* close */     twe_close,
-                /* read */      noread,
-                /* write */     nowrite,
-                /* ioctl */     twe_ioctl,
-                /* poll */      nopoll,
-                /* mmap */      nommap,
-                /* strategy */  nostrategy,
-                /* name */      "twe",
-                /* maj */       TWE_CDEV_MAJOR,
-                /* dump */      nodump,
-                /* psize */     nopsize,
-                /* flags */     0,
-                /* bmaj */      -1
-};
-
 /********************************************************************************
- * Match a 3ware Escalade ATA RAID controller.
+ * Initialise the controller, set up driver data structures.
  */
-static int
-twe_probe(device_t dev)
-{
-
-    debug_called(4);
-
-    if ((pci_get_vendor(dev) == TWE_VENDOR_ID) &&
-	(pci_get_device(dev) == TWE_DEVICE_ID)) {
-	device_set_desc(dev, TWE_DEVICE_NAME);
-	return(0);
-    }
-    return(ENXIO);
-}
-
-/********************************************************************************
- * Free all of the resources associated with (sc).
- *
- * Should not be called if the controller is active.
- */
-static void
-twe_free(struct twe_softc *sc)
+int
+twe_setup(struct twe_softc *sc)
 {
     struct twe_request	*tr;
+    int			i;
 
     debug_called(4);
 
-    /* throw away any command buffers */
-    while ((tr = TAILQ_FIRST(&sc->twe_freecmds)) != NULL) {
-	TAILQ_REMOVE(&sc->twe_freecmds, tr, tr_link);
-	twe_free_request(tr);
-    }
-
-    /* destroy the data-transfer DMA tag */
-    if (sc->twe_buffer_dmat)
-	bus_dma_tag_destroy(sc->twe_buffer_dmat);
-
-    /* disconnect the interrupt handler */
-    if (sc->twe_intr)
-	bus_teardown_intr(sc->twe_dev, sc->twe_irq, sc->twe_intr);
-    if (sc->twe_irq != NULL)
-	bus_release_resource(sc->twe_dev, SYS_RES_IRQ, 0, sc->twe_irq);
-
-    /* destroy the parent DMA tag */
-    if (sc->twe_parent_dmat)
-	bus_dma_tag_destroy(sc->twe_parent_dmat);
-
-    /* release the register window mapping */
-    if (sc->twe_io != NULL)
-	bus_release_resource(sc->twe_dev, SYS_RES_IOPORT, TWE_IO_CONFIG_REG, sc->twe_io);
-
-    /* destroy control device */
-    if (sc->twe_dev_t != (dev_t)NULL)
-	destroy_dev(sc->twe_dev_t);
-}
-
-/********************************************************************************
- * Allocate resources, initialise the controller.
- */
-static int
-twe_attach(device_t dev)
-{
-    struct twe_softc	*sc;
-    int			rid, error;
-    u_int32_t		command;
-
-    debug_called(4);
-
-    /*
-     * Make sure we are going to be able to talk to this board.
-     */
-    command = pci_read_config(dev, PCIR_COMMAND, 2);
-    if ((command & PCIM_CMD_PORTEN) == 0) {
-	device_printf(dev, "register window not available\n");
-	return(ENXIO);
-    }
-    /*
-     * Force the busmaster enable bit on, in case the BIOS forgot.
-     */
-    command |= PCIM_CMD_BUSMASTEREN;
-    pci_write_config(dev, PCIR_COMMAND, command, 2);
-
     /*
-     * Initialise the softc structure.
+     * Initialise request queues.
      */
-    sc = device_get_softc(dev);
-    bzero(sc, sizeof(*sc)); 
-    sc->twe_dev = dev;
-    TAILQ_INIT(&sc->twe_work);
-    TAILQ_INIT(&sc->twe_freecmds);
-    bioq_init(&sc->twe_bioq);
+    twe_initq_free(sc);
+    twe_initq_bio(sc);
+    twe_initq_ready(sc);
+    twe_initq_busy(sc);
+    twe_initq_complete(sc);
     sc->twe_wait_aen = -1;
 
     /*
-     * Allocate the PCI register window.
+     * Allocate request structures up front.
      */
-    rid = TWE_IO_CONFIG_REG;
-    sc->twe_io = bus_alloc_resource(dev, SYS_RES_IOPORT, &rid, 0, ~0, 1, RF_ACTIVE);
-    if (sc->twe_io == NULL) {
-	device_printf(sc->twe_dev, "can't allocate register window\n");
-	twe_free(sc);
-	return(ENXIO);
-    }
-    sc->twe_btag = rman_get_bustag(sc->twe_io);
-    sc->twe_bhandle = rman_get_bushandle(sc->twe_io);
+    for (i = 0; i < TWE_Q_LENGTH; i++) {
+	if ((tr = twe_allocate_request(sc)) == NULL)
+	    return(ENOMEM);
+	/*
+	 * Set global defaults that won't change.
+	 */
+	tr->tr_command.generic.host_id = sc->twe_host_id;	/* controller-assigned host ID */
+	tr->tr_command.generic.request_id = i;			/* our index number */
+	sc->twe_lookup[i] = tr;
 
-    /*
-     * Allocate the parent bus DMA tag appropriate for PCI.
-     */
-    error = bus_dma_tag_create(NULL, 				/* parent */
-			       1, 0, 				/* alignment, boundary */
-			       BUS_SPACE_MAXADDR_32BIT, 	/* lowaddr */
-			       BUS_SPACE_MAXADDR, 		/* highaddr */
-			       NULL, NULL, 			/* filter, filterarg */
-			       MAXBSIZE, TWE_MAX_SGL_LENGTH,	/* maxsize, nsegments */
-			       BUS_SPACE_MAXSIZE_32BIT,		/* maxsegsize */
-			       BUS_DMA_ALLOCNOW,		/* flags */
-			       &sc->twe_parent_dmat);
-    if (error != 0) {
-	device_printf(dev, "can't allocate parent DMA tag\n");
-	twe_free(sc);
-	return(ENOMEM);
+	/*
+	 * Put command onto the freelist.
+	 */
+	twe_release_request(tr);
     }
 
-    /* 
-     * Allocate and connect our interrupt.
+    /*
+     * Wait for the controller to come ready.
      */
-    rid = 0;
-    sc->twe_irq = bus_alloc_resource(sc->twe_dev, SYS_RES_IRQ, &rid, 0, ~0, 1, RF_SHAREABLE | RF_ACTIVE);
-    if (sc->twe_irq == NULL) {
-	device_printf(sc->twe_dev, "can't allocate interrupt\n");
-	twe_free(sc);
-	return(ENXIO);
-    }
-    error = bus_setup_intr(sc->twe_dev, sc->twe_irq, INTR_TYPE_BIO,  twe_intr, sc, &sc->twe_intr);
-    if (error) {
-	device_printf(sc->twe_dev, "can't set up interrupt\n");
-	twe_free(sc);
+    if (twe_wait_status(sc, TWE_STATUS_MICROCONTROLLER_READY, 60)) {
+	twe_printf(sc, "microcontroller not ready\n");
 	return(ENXIO);
     }
 
     /*
-     * Create DMA tag for mapping objects into controller-addressable space.
+     * Disable interrupts from the card.
      */
-    error = bus_dma_tag_create(sc->twe_parent_dmat, 		/* parent */
-			       1, 0, 				/* alignment, boundary */
-			       BUS_SPACE_MAXADDR,		/* lowaddr */
-			       BUS_SPACE_MAXADDR, 		/* highaddr */
-			       NULL, NULL, 			/* filter, filterarg */
-			       MAXBSIZE, TWE_MAX_SGL_LENGTH,	/* maxsize, nsegments */
-			       BUS_SPACE_MAXSIZE_32BIT,		/* maxsegsize */
-			       0,				/* flags */
-			       &sc->twe_buffer_dmat);
-    if (error != 0) {
-	device_printf(sc->twe_dev, "can't allocate data buffer DMA tag\n");
-	twe_free(sc);
-	return(ENOMEM);
-    }
+    twe_disable_interrupts(sc);
 
     /*
-     * Create the control device.
+     * Soft reset the controller, look for the AEN acknowledging the reset,
+     * check for errors, drain the response queue.
      */
-    sc->twe_dev_t = make_dev(&twe_cdevsw, device_get_unit(sc->twe_dev), UID_ROOT, GID_OPERATOR,
-			     S_IRUSR | S_IWUSR, "twe%d", device_get_unit(sc->twe_dev));
+    for (i = 0; i < TWE_MAX_RESET_TRIES; i++) {
 
-    /*
-     * Schedule ourselves to bring the controller up once interrupts are available.
-     * This isn't strictly necessary, since we disable interrupts while probing the
-     * controller, but it is more in keeping with common practice for other disk 
-     * devices.
-     */
-    bzero(&sc->twe_ich, sizeof(struct intr_config_hook));
-    sc->twe_ich.ich_func = twe_startup;
-    sc->twe_ich.ich_arg = sc;
-    if (config_intrhook_establish(&sc->twe_ich) != 0) {
-	device_printf(sc->twe_dev, "can't establish configuration hook\n");
-	twe_free(sc);
+	if (i > 0)
+	    twe_printf(sc, "reset %d failed, trying again\n", i);
+	
+	if (!twe_soft_reset(sc))
+	    break;			/* reset process complete */
+    }
+    /* did we give up? */
+    if (i >= TWE_MAX_RESET_TRIES) {
+	twe_printf(sc, "can't initialise controller, giving up\n");
 	return(ENXIO);
     }
 
@@ -375,98 +180,57 @@ twe_attach(device_t dev)
 }
 
 /********************************************************************************
- * Initialise the controller, locate disk devices and attach children to them.
+ * Locate disk devices and attach children to them.
  */
-static void
-twe_startup(void *arg)
+void
+twe_init(struct twe_softc *sc)
 {
-    struct twe_softc		*sc = (struct twe_softc *)arg;
     struct twe_drive		*dr;
-    int				i, error;
-    u_int32_t			status_reg;
-    TWE_Param			*drives, *capacity;
-
-    debug_called(4);
+    int				i, table;
+    u_int16_t			dsize;
+    TWE_Param			*drives, *param;
+    TWE_Unit_Descriptor		*ud;
 
-    /* pull ourselves off the intrhook chain */
-    config_intrhook_disestablish(&sc->twe_ich);
 
     /*
-     * Wait for the controller to come ready.
+     * The controller is in a safe state, so try to find drives attached to it.
      */
-    if (twe_wait_status(sc, TWE_STATUS_MICROCONTROLLER_READY, 60)) {
-	device_printf(sc->twe_dev, "microcontroller not ready\n");
+    if ((drives = twe_get_param(sc, TWE_PARAM_UNITSUMMARY, TWE_PARAM_UNITSUMMARY_Status,
+				TWE_MAX_UNITS, NULL)) == NULL) {
+	twe_printf(sc, "can't detect attached units\n");
 	return;
     }
-
-    /*
-     * Disable interrupts from the card while we're getting it into a safe state.
-     */
-    twe_disable_interrupts(sc);
-
+    
     /*
-     * Soft reset the controller, look for the AEN acknowledging the reset,
-     * check for errors, drain the response queue.
+     * For each detected unit, create a child device.
      */
-    for (i = 0; i < TWE_MAX_RESET_TRIES; i++) {
+    for (i = 0, dr = &sc->twe_drive[0]; i < TWE_MAX_UNITS; i++, dr++) {
 
-	if (i > 0)
-	    device_printf(sc->twe_dev, "reset %d failed, trying again\n", i);
+	/* check that the drive is online */
+	if (!(drives->data[i] & TWE_PARAM_UNITSTATUS_Online))
+	    continue;
 
-	TWE_SOFT_RESET(sc);
+	table = TWE_PARAM_UNITINFO + i;
 
-	if (twe_wait_status(sc, TWE_STATUS_ATTENTION_INTERRUPT, 15)) {
-	    device_printf(sc->twe_dev, "no attention interrupt");
-	    continue;
-	}
-	if (twe_drain_aen_queue(sc)) {
-	    device_printf(sc->twe_dev, "can't drain AEN queue\n");
-	    continue;
-	}
-	if (twe_find_aen(sc, TWE_AEN_SOFT_RESET)) {
-	    device_printf(sc->twe_dev, "reset not reported\n");
+	if (twe_get_param_4(sc, table, TWE_PARAM_UNITINFO_Capacity, &dr->td_size)) {
+	    twe_printf(sc, "error fetching capacity for unit %d\n", i);
 	    continue;
 	}
-	status_reg = TWE_STATUS(sc);
-	if (TWE_STATUS_ERRORS(status_reg) || twe_check_bits(sc, status_reg)) {
-	    device_printf(sc->twe_dev, "controller errors detected\n");
+	if (twe_get_param_1(sc, table, TWE_PARAM_UNITINFO_Status, &dr->td_state)) {
+	    twe_printf(sc, "error fetching state for unit %d\n", i);
 	    continue;
 	}
-	if (twe_drain_response_queue(sc)) {
-	    device_printf(sc->twe_dev, "can't drain response queue\n");
+	if (twe_get_param_2(sc, table, TWE_PARAM_UNITINFO_DescriptorSize, &dsize)) {
+	    twe_printf(sc, "error fetching descriptor size for unit %d\n", i);
 	    continue;
 	}
-	break;			/* reset process complete */
-    }
-    /* did we give up? */
-    if (i >= TWE_MAX_RESET_TRIES) {
-	device_printf(sc->twe_dev, "can't initialise controller, giving up\n");
-	return;
-    }
-
-    /*
-     * The controller is in a safe state, so try to find drives attached to it.
-     * XXX ick, magic numbers
-     */
-    if ((drives = twe_get_param(sc, 3, 3, TWE_MAX_UNITS, NULL)) == NULL) {
-	device_printf(sc->twe_dev, "can't detect attached units\n");
-	return;
-    }
-    
-    /*
-     * For each detected unit, create a child device.
-     */
-    for (i = 0, dr = &sc->twe_drive[0]; i < TWE_MAX_UNITS; i++, dr++) {
-	
-	if (drives->data[i] == 0)	/* unit not present */
-	    continue;
-
-	if ((capacity = twe_get_param(sc, TWE_UNIT_INFORMATION_TABLE_BASE + i, 4, 4, NULL)) == NULL) {
-	    device_printf(sc->twe_dev, "error fetching capacity for unit %d\n", i);
+	if ((param = twe_get_param(sc, table, TWE_PARAM_UNITINFO_Descriptor, dsize - 3, NULL)) == NULL) {
+	    twe_printf(sc, "error fetching descriptor for unit %d\n", i);
 	    continue;
 	}
-	dr->td_size = *(u_int32_t *)capacity->data;
-	free(capacity, M_DEVBUF);
+	ud = (TWE_Unit_Descriptor *)param->data;
+	dr->td_type = ud->configuration;
+	free(param, M_DEVBUF);
 
 	/* build synthetic geometry as per controller internal rules */
 	if (dr->td_size > 0x200000) {
@@ -477,25 +241,23 @@ twe_startup(void *arg)
 	    dr->td_sectors = 32;
 	}
 	dr->td_cylinders = dr->td_size / (dr->td_heads * dr->td_sectors);
-
 	dr->td_unit = i;
-	dr->td_state = TWE_DRIVE_UNKNOWN;		/* XXX how do we find out what the real state is? */
-	dr->td_raidlevel = TWE_DRIVE_UNKNOWN;		/* XXX how do we find out what the real raidlevel is? */
 
-	dr->td_disk =  device_add_child(sc->twe_dev, NULL, -1);
-	if (dr->td_disk == 0)
-	    device_printf(sc->twe_dev, "device_add_child failed\n");
-	device_set_ivars(dr->td_disk, dr);
+	twe_attach_drive(sc, dr);
     }
     free(drives, M_DEVBUF);
 
-    if ((error = bus_generic_attach(sc->twe_dev)) != 0)
-	device_printf(sc->twe_dev, "bus_generic_attach returned %d\n", error);
-
     /*
      * Initialise connection with controller.
      */
-    twe_init_connection(sc);
+    twe_init_connection(sc, TWE_INIT_MESSAGE_CREDITS);
+
+#ifdef TWE_SHUTDOWN_NOTIFICATION
+    /*
+     * Tell the controller we support shutdown notification.
+     */
+    twe_set_param_1(sc, TWE_PARAM_FEATURES, TWE_PARAM_FEATURES_DriverShutdown, 1);
+#endif
 
     /* 
      * Mark controller up and ready to run.
@@ -503,126 +265,38 @@ twe_startup(void *arg)
     sc->twe_state &= ~TWE_STATE_SHUTDOWN;
 
     /*
-     * Finally enable interrupts .
+     * Finally enable interrupts.
      */
     twe_enable_interrupts(sc);
 }
 
 /********************************************************************************
- * Disconnect from the controller completely, in preparation for unload.
- */
-static int
-twe_detach(device_t dev)
-{
-    struct twe_softc	*sc = device_get_softc(dev);
-    int			s, error;
-
-    debug_called(4);
-
-    error = EBUSY;
-    s = splbio();
-    if (sc->twe_state & TWE_STATE_OPEN)
-	goto out;
-
-    /*	
-     * Shut the controller down.
-     */
-    if ((error = twe_shutdown(dev)))
-	goto out;
-
-    twe_free(sc);
-
-    error = 0;
- out:
-    splx(s);
-    return(error);
-}
-
-/********************************************************************************
- * Bring the controller down to a dormant state and detach all child devices.
- *
- * Note that we can assume that the bioq on the controller is empty, as we won't
- * allow shutdown if any device is open.
+ * Stop the controller
  */
-static int
-twe_shutdown(device_t dev)
+void
+twe_deinit(struct twe_softc *sc)
 {
-    struct twe_softc	*sc = device_get_softc(dev);
-    int			i, s, error;
-
-    debug_called(4);
-
-    s = splbio();
-    error = 0;
-
     /*
      * Mark the controller as shutting down, and disable any further interrupts.
      */
     sc->twe_state |= TWE_STATE_SHUTDOWN;
     twe_disable_interrupts(sc);
 
-    /* 
-     * Delete all our child devices.
+#ifdef TWE_SHUTDOWN_NOTIFICATION
+    /*
+     * Disconnect from the controller
      */
-    for (i = 0; i < TWE_MAX_UNITS; i++) {
-	if (sc->twe_drive[i].td_disk != 0) {
-	    if ((error = device_delete_child(sc->twe_dev, sc->twe_drive[i].td_disk)) != 0)
-		goto out;
-	    sc->twe_drive[i].td_disk = 0;
-	}
-    }
-
- out:
-    splx(s);
-    return(error);
-}
-
-/********************************************************************************
- * Bring the controller to a quiescent state, ready for system suspend.
- * 
- * XXX this isn't really very well implemented.
- */
-static int
-twe_suspend(device_t dev)
-{
-    struct twe_softc	*sc = device_get_softc(dev);
-    int			s;
-
-    debug_called(4);
-
-    s = splbio();
-    sc->twe_state |= TWE_STATE_SUSPEND;
-    
-    twe_disable_interrupts(sc);
-    splx(s);
-
-    return(0);
-}
-
-/********************************************************************************
- * Bring the controller back to a state ready for operation.
- */
-static int
-twe_resume(device_t dev)
-{
-    struct twe_softc	*sc = device_get_softc(dev);
-
-    debug_called(4);
-
-    sc->twe_state &= ~TWE_STATE_SUSPEND;
-    twe_enable_interrupts(sc);
-
-    return(0);
+    twe_init_connection(sc, TWE_SHUTDOWN_MESSAGE_CREDITS);
+#endif
 }
 
 /*******************************************************************************
  * Take an interrupt, or be poked by other code to look for interrupt-worthy
  * status.
  */
-static void
-twe_intr(void *arg)
+void
+twe_intr(struct twe_softc *sc)
 {
-    struct twe_softc	*sc = (struct twe_softc *)arg;
     u_int32_t		status_reg;
 
     debug_called(4);
@@ -646,49 +320,178 @@ twe_intr(void *arg)
 	twe_done(sc);
 };
 
-/********************************************************************************
- ********************************************************************************
-                                                                   Control Device
- ********************************************************************************
- ********************************************************************************/
-
-/********************************************************************************
- * Accept an open operation on the control device.
+/*******************************************************************************
+ * Receive a bio structure from a child device and queue it on a particular
+ * controller, then poke the controller to start as much work as it can.
  */
-static int
-twe_open(dev_t dev, int flags, int fmt, struct proc *p)
+int
+twe_submit_bio(struct twe_softc *sc, twe_bio *bp)
 {
-    int			unit = minor(dev);
-    struct twe_softc	*sc = devclass_get_softc(twe_devclass, unit);
+    
+    debug_called(4);
+
+    twe_enqueue_bio(sc, bp);
 
-    sc->twe_state |= TWE_STATE_OPEN;
+    twe_startio(sc);
     return(0);
 }
 
 /********************************************************************************
- * Accept the last close on the control device.
+ * Handle controller-specific control operations.
  */
-static int
-twe_close(dev_t dev, int flags, int fmt, struct proc *p)
+int
+twe_ioctl(struct twe_softc *sc, int cmd, void *addr)
 {
-    int			unit = minor(dev);
-    struct twe_softc	*sc = devclass_get_softc(twe_devclass, unit);
+    struct twe_usercommand	*tu = (struct twe_usercommand *)addr;
+    struct twe_paramcommand	*tp = (struct twe_paramcommand *)addr;
+    union twe_statrequest	*ts = (union twe_statrequest *)addr;
+    TWE_Param			*param;
+    void			*data;
+    int				*arg = (int *)addr;
+    struct twe_request		*tr;
+    int				s, error;
+
+    error = 0;
+    switch(cmd) {
+	/* handle a command from userspace */
+    case TWEIO_COMMAND:
+	/* get a request */
+	if (twe_get_request(sc, &tr)) {
+	    error = EBUSY;
+	    goto cmd_done;
+	}
+
+	/* copy the user-supplied command */
+	bcopy(&tu->tu_command, &tr->tr_command, sizeof(TWE_Command));
+
+	/* if there's a data buffer, allocate and copy it in */
+	tr->tr_length = tu->tu_size;
+	if (tr->tr_length > 0) {
+	    if ((tr->tr_data = malloc(tr->tr_length, M_DEVBUF, M_WAITOK)) == NULL) {
+		error = ENOMEM;
+		goto cmd_done;
+	    }
+	    if ((error = copyin(tu->tu_data, tr->tr_data, tr->tr_length)) != 0)
+		goto cmd_done;
+	    tr->tr_flags |= TWE_CMD_DATAIN | TWE_CMD_DATAOUT;
+	}
+
+	/* run the command */
+	twe_wait_request(tr);
+
+	/* copy the command out again */
+	bcopy(&tr->tr_command, &tu->tu_command, sizeof(TWE_Command));
+	
+	/* if there was a data buffer, copy it out */
+	if (tr->tr_length > 0)
+	    error = copyout(tr->tr_data, tu->tu_data, tr->tr_length);
+
+    cmd_done:
+	/* free resources */
+	if (tr->tr_data != NULL)
+	    free(tr->tr_data, M_DEVBUF);
+	if (tr != NULL)
+	    twe_release_request(tr);
+
+	break;
+
+	/* fetch statistics counter */
+    case TWEIO_STATS:
+	switch (ts->ts_item) {
+#ifdef TWE_PERFORMANCE_MONITOR
+	case TWEQ_FREE:
+	case TWEQ_BIO:
+	case TWEQ_READY:
+	case TWEQ_BUSY:
+	case TWEQ_COMPLETE:
+	    bcopy(&sc->twe_qstat[ts->ts_item], &ts->ts_qstat, sizeof(struct twe_qstat));
+	    break;
+#endif
+	default:
+	    error = ENOENT;
+	    break;
+	}
+	break;
+
+	/* poll for an AEN */
+    case TWEIO_AEN_POLL:
+	*arg = twe_dequeue_aen(sc);
+	if (*arg == -1)
+	    error = ENOENT;
+	break;
+
+	/* wait for another AEN to show up */
+    case TWEIO_AEN_WAIT:
+	s = splbio();
+	while ((*arg = twe_dequeue_aen(sc)) == -1) {
+	    error = tsleep(&sc->twe_aen_queue, PRIBIO | PCATCH, "tweaen", 0);
+	    if (error == EINTR)
+		break;
+	}
+	splx(s);
+	break;
+
+    case TWEIO_GET_PARAM:
+	if ((param = twe_get_param(sc, tp->tp_table_id, tp->tp_param_id, tp->tp_size, NULL)) == NULL) {
+	    twe_printf(sc, "TWEIO_GET_PARAM failed for 0x%x/0x%x/%d\n", 
+		       tp->tp_table_id, tp->tp_param_id, tp->tp_size);
+	    error = EINVAL;
+	} else {
+	    if (param->parameter_size_bytes > tp->tp_size) {
+		twe_printf(sc, "TWEIO_GET_PARAM parameter too large (%d > %d)\n",	
+			   param->parameter_size_bytes, tp->tp_size);
+		error = EFAULT;
+	    } else {
+		error = copyout(param->data, tp->tp_data, param->parameter_size_bytes);
+	    }
+	    free(param, M_DEVBUF);
+	}
+	break;
+
+    case TWEIO_SET_PARAM:
+	if ((data = malloc(tp->tp_size, M_DEVBUF, M_WAITOK)) == NULL) {
+	    error = ENOMEM;
+	} else {
+	    error = copyin(tp->tp_data, data, tp->tp_size);
+	    if (error == 0)
+		error = twe_set_param(sc, tp->tp_table_id, tp->tp_param_id, tp->tp_size, data);
+	    free(data, M_DEVBUF);
+	}
+	break;
+
+    case TWEIO_RESET:
+	twe_reset(sc);
+	break;
+
+	/* nothing we understand */
+    default:	
+	error = ENOTTY;
+    }
+
+    return(error);
+}
 
-    sc->twe_state &= ~TWE_STATE_OPEN;
-    return (0);
+/********************************************************************************
+ * Enable the useful interrupts from the controller.
+ */
+void
+twe_enable_interrupts(struct twe_softc *sc)
+{
+    sc->twe_state |= TWE_STATE_INTEN;
+    TWE_CONTROL(sc, 
+	       TWE_CONTROL_CLEAR_ATTENTION_INTERRUPT |
+	       TWE_CONTROL_UNMASK_RESPONSE_INTERRUPT |
+	       TWE_CONTROL_ENABLE_INTERRUPTS);
 }
 
 /********************************************************************************
- * Handle controller-specific control operations.
+ * Disable interrupts from the controller.
  */
-static int
-twe_ioctl(dev_t dev, u_long cmd, caddr_t addr, int32_t flag, struct proc *p)
+void
+twe_disable_interrupts(struct twe_softc *sc)
 {
-    
-    switch(cmd) {
-    default:	
-	return(ENOTTY);
-    }
+    TWE_CONTROL(sc, TWE_CONTROL_DISABLE_INTERRUPTS);
+    sc->twe_state &= ~TWE_STATE_INTEN;
 }
 
 /********************************************************************************
@@ -697,22 +500,42 @@ twe_ioctl(dev_t dev, u_long cmd, caddr_t addr, int32_t flag, struct proc *p)
  ********************************************************************************
  ********************************************************************************/
 
-/*******************************************************************************
- * Receive a bio structure from a child device and queue it on a particular
- * controller, then poke the controller to start as much work as it can.
+/********************************************************************************
+ * Read integer parameter table entries.
  */
-int
-twe_submit_buf(struct twe_softc *sc, struct bio *bp)
+static int
+twe_get_param_1(struct twe_softc *sc, int table_id, int param_id, u_int8_t *result)
 {
-    int		s;
-    
-    debug_called(4);
+    TWE_Param	*param;
 
-    s = splbio();
-    bioq_insert_tail(&sc->twe_bioq, bp);
-    splx(s);
+    if ((param = twe_get_param(sc, table_id, param_id, 1, NULL)) == NULL)
+	return(ENOENT);
+    *result = *(u_int8_t *)param->data;
+    free(param, M_DEVBUF);
+    return(0);
+}
 
-    twe_startio(sc);
+static int
+twe_get_param_2(struct twe_softc *sc, int table_id, int param_id, u_int16_t *result)
+{
+    TWE_Param	*param;
+
+    if ((param = twe_get_param(sc, table_id, param_id, 2, NULL)) == NULL)
+	return(ENOENT);
+    *result = *(u_int16_t *)param->data;
+    free(param, M_DEVBUF);
+    return(0);
+}
+
+static int
+twe_get_param_4(struct twe_softc *sc, int table_id, int param_id, u_int32_t *result)
+{
+    TWE_Param	*param;
+
+    if ((param = twe_get_param(sc, table_id, param_id, 4, NULL)) == NULL)
+	return(ENOENT);
+    *result = *(u_int32_t *)param->data;
+    free(param, M_DEVBUF);
     return(0);
 }
 
@@ -723,7 +546,8 @@ twe_submit_buf(struct twe_softc *sc, struct bio *bp)
  * The caller is responsible for freeing the data when done with it.
  */
 static void *
-twe_get_param(struct twe_softc *sc, int table_id, int parameter_id, size_t size, void (* func)(struct twe_request *tr))
+twe_get_param(struct twe_softc *sc, int table_id, int param_id, size_t param_size, 
+	      void (* func)(struct twe_request *tr))
 {
     struct twe_request	*tr;
     TWE_Command		*cmd;
@@ -736,7 +560,7 @@ twe_get_param(struct twe_softc *sc, int table_id, int parameter_id, size_t size,
     param = NULL;
 
     /* get a command */
-    if ((tr = twe_get_request(sc)) == NULL)
+    if (twe_get_request(sc, &tr))
 	goto err;
 
     /* get a buffer */
@@ -745,37 +569,45 @@ twe_get_param(struct twe_softc *sc, int table_id, int parameter_id, size_t size,
     tr->tr_data = param;
     tr->tr_length = TWE_SECTOR_SIZE;
     tr->tr_flags = TWE_CMD_DATAIN | TWE_CMD_DATAOUT;
-    tr->tr_complete = NULL;
-    tr->tr_private = NULL;
 
     /* build the command for the controller */
     cmd = &tr->tr_command;
-    cmd->opcode = TWE_OP_GET_PARAM;
-    cmd->sgl_offset = 2;
-    cmd->size = 2;
-    cmd->unit = 0;
-    cmd->count = 1;
+    cmd->param.opcode = TWE_OP_GET_PARAM;
+    cmd->param.size = 2;
+    cmd->param.unit = 0;
+    cmd->param.param_count = 1;
 
     /* map the command/data into controller-visible space */
     twe_map_request(tr);
 
     /* fill in the outbound parameter data */
     param->table_id = table_id;
-    param->parameter_id = parameter_id;
-    param->parameter_size_bytes = size;
+    param->parameter_id = param_id;
+    param->parameter_size_bytes = param_size;
 
     /* submit the command and either wait or let the callback handle it */
     if (func == NULL) {
 	/* XXX could use twe_wait_request here if interrupts were enabled? */
 	error = twe_immediate_request(tr);
 	if (error == 0) {
-	    if (tr->tr_command.status != 0) {
-		debug(2, "command failed - 0x%x", tr->tr_command.status);
+	    switch (cmd->generic.flags) {
+	    case TWE_FLAGS_SUCCESS:
+		break;
+	    case TWE_FLAGS_INFORMATIONAL:
+	    case TWE_FLAGS_WARNING:
+		twe_printf(sc, "command completed - %s\n", 
+			   twe_describe_code(twe_table_status, cmd->generic.status));
+		break;
+
+	    case TWE_FLAGS_FATAL:
+	    default:
+		twe_printf(sc, "command failed - %s\n", 
+			   twe_describe_code(twe_table_status, cmd->generic.status));
 		goto err;
 	    }
-	    twe_release_request(tr);
-	    return(param);
 	}
+	twe_release_request(tr);
+	return(param);
     } else {
 	tr->tr_complete = func;
 	error = twe_start(tr);
@@ -794,12 +626,106 @@ err:
 }
 
 /********************************************************************************
+ * Set integer parameter table entries.
+ */
+static int
+twe_set_param_1(struct twe_softc *sc, int table_id, int param_id, u_int8_t value)
+{
+    return(twe_set_param(sc, table_id, param_id, sizeof(value), &value));
+}
+
+static int
+twe_set_param_2(struct twe_softc *sc, int table_id, int param_id, u_int16_t value)
+{
+    return(twe_set_param(sc, table_id, param_id, sizeof(value), &value));
+}
+
+static int
+twe_set_param_4(struct twe_softc *sc, int table_id, int param_id, u_int32_t value)
+{
+    return(twe_set_param(sc, table_id, param_id, sizeof(value), &value));
+}
+
+/********************************************************************************
+ * Perform a TWE_OP_SET_PARAM command, returns nonzero on error.
+ */
+static int
+twe_set_param(struct twe_softc *sc, int table_id, int param_id, int param_size, void *data)
+{
+    struct twe_request	*tr;
+    TWE_Command		*cmd;
+    TWE_Param		*param;
+    int			error;
+
+    debug_called(4);
+
+    tr = NULL;
+    param = NULL;
+    error = ENOMEM;
+
+    /* get a command */
+    if (twe_get_request(sc, &tr))
+	goto out;
+
+    /* get a buffer */
+    if ((param = (TWE_Param *)malloc(TWE_SECTOR_SIZE, M_DEVBUF, M_NOWAIT)) == NULL)
+	goto out;
+    tr->tr_data = param;
+    tr->tr_length = TWE_SECTOR_SIZE;
+    tr->tr_flags = TWE_CMD_DATAIN | TWE_CMD_DATAOUT;
+
+    /* build the command for the controller */
+    cmd = &tr->tr_command;
+    cmd->param.opcode = TWE_OP_SET_PARAM;
+    cmd->param.size = 2;
+    cmd->param.unit = 0;
+    cmd->param.param_count = 1;
+
+    /* map the command/data into controller-visible space */
+    twe_map_request(tr);
+
+    /* fill in the outbound parameter data */
+    param->table_id = table_id;
+    param->parameter_id = param_id;
+    param->parameter_size_bytes = param_size;
+    bcopy(data, param->data, param_size);
+
+    /* XXX could use twe_wait_request here if interrupts were enabled? */
+    error = twe_immediate_request(tr);
+    if (error == 0) {
+	switch (cmd->generic.flags) {
+	case TWE_FLAGS_SUCCESS:
+	    break;
+	case TWE_FLAGS_INFORMATIONAL:
+	case TWE_FLAGS_WARNING:
+	    twe_printf(sc, "command completed - %s\n", 
+		       twe_describe_code(twe_table_status, cmd->generic.status));
+	    break;
+
+	case TWE_FLAGS_FATAL:
+	default:
+	    twe_printf(sc, "command failed - %s\n", 
+		       twe_describe_code(twe_table_status, cmd->generic.status));
+	    error = EIO;
+	    break;
+	}
+    }
+
+out:
+    if (tr != NULL)
+	twe_release_request(tr);
+    if (param != NULL)
+	free(param, M_DEVBUF);
+    return(error);
+}
+
+/********************************************************************************
  * Perform a TWE_OP_INIT_CONNECTION command, returns nonzero on error.
  *
  * Typically called with interrupts disabled.
  */
 static int
-twe_init_connection(struct twe_softc *sc)
+twe_init_connection(struct twe_softc *sc, int mode)
 {
     struct twe_request	*tr;
     TWE_Command		*cmd;
@@ -808,17 +734,16 @@ twe_init_connection(struct twe_softc *sc)
     debug_called(4);
 
     /* get a command */
-    if ((tr = twe_get_request(sc)) == NULL)
+    if (twe_get_request(sc, &tr))
 	return(NULL);
 
     /* build the command */
     cmd = &tr->tr_command;
-    cmd->opcode = TWE_OP_INIT_CONNECTION;
-    cmd->sgl_offset = 0;
-    cmd->size = 3;
-    cmd->unit = 0;
-    cmd->count = TWE_INIT_MESSAGE_CREDITS;
-    cmd->args.init_connection.response_queue_pointer = 0;
+    cmd->initconnection.opcode = TWE_OP_INIT_CONNECTION;
+    cmd->initconnection.size = 3;
+    cmd->initconnection.host_id = 0;
+    cmd->initconnection.message_credits = mode;
+    cmd->initconnection.response_queue_pointer = 0;
 
     /* map the command into controller-visible space */
     twe_map_request(tr);
@@ -829,10 +754,11 @@ twe_init_connection(struct twe_softc *sc)
     twe_unmap_request(tr);
     twe_release_request(tr);
 
+    if (mode == TWE_INIT_MESSAGE_CREDITS)
+	sc->twe_host_id = cmd->initconnection.host_id;
     return(error);
 }
 
-#if 0
 /********************************************************************************
  * Start the command (tr) and sleep waiting for it to complete.
  *
@@ -841,22 +767,21 @@ twe_init_connection(struct twe_softc *sc)
 static int
 twe_wait_request(struct twe_request *tr)
 {
-    int			error, s;
+    int		s;
 
     debug_called(4);
 
-    error = 0;
-    tr->tr_private = tr;				/* our wait channel */
+    tr->tr_flags |= TWE_CMD_SLEEPER;
+    tr->tr_status = TWE_CMD_BUSY;
+    twe_enqueue_ready(tr);
+    twe_startio(tr->tr_sc);
     s = splbio();
-    if ((error = twe_start(tr)) != 0)
-	goto out;
-    tsleep(tr->tr_private, PUSER, "twwcmd", hz);	/* XXX more sensible timeout than 1s? */
+    while (tr->tr_status == TWE_CMD_BUSY)
+	tsleep(tr, PRIBIO, "twewait", 0);
     splx(s);
-
-out:
-    return(error);
+    
+    return(0);
 }
-#endif
 
 /********************************************************************************
  * Start the command (tr) and busy-wait for it to complete.
@@ -889,77 +814,67 @@ twe_startio(struct twe_softc *sc)
 {
     struct twe_request	*tr;
     TWE_Command		*cmd;
-    struct bio		*bp;
-    int			s, error;
+    twe_bio		*bp;
+    int			error;
 
     debug_called(4);
 
     /* spin until something prevents us from doing any work */
-    s = splbio();
     for (;;) {
 
-	if (sc->twe_deferred == NULL) {
+	/* try to get a command that's already ready to go */
+	tr = twe_dequeue_ready(sc);
+
+	/* build a command from an outstanding bio */
+	if (tr == NULL) {
 	    /* see if there's work to be done */
-	    if ((bp = bioq_first(&sc->twe_bioq)) == NULL)
+	    if ((bp = twe_dequeue_bio(sc)) == NULL)
 		break;
 	    /* get a command */
-	    if ((tr = twe_get_request(sc)) == NULL)
+	    if (twe_get_request(sc, &tr))
 		break;
 
-	    /* get the bio containing our work */
-	    bioq_remove(&sc->twe_bioq, bp);
-	    splx(s);
-	
 	    /* connect the bio to the command */
 	    tr->tr_complete = twe_completeio;
 	    tr->tr_private = bp;
-	    tr->tr_data = bp->bio_data;
-	    tr->tr_length = bp->bio_bcount;
+	    tr->tr_data = TWE_BIO_DATA(bp);
+	    tr->tr_length = TWE_BIO_LENGTH(bp);
 	    cmd = &tr->tr_command;
-#ifdef FREEBSD_4
-	    if (bp->bio_flags & B_READ)
-#else
-	    if (bp->bio_cmd == BIO_READ)
-#endif
-	    {
+	    if (TWE_BIO_IS_READ(bp)) {
 		tr->tr_flags |= TWE_CMD_DATAIN;
-		cmd->opcode = TWE_OP_READ;
+		cmd->io.opcode = TWE_OP_READ;
 	    } else {
 		tr->tr_flags |= TWE_CMD_DATAOUT;
-		cmd->opcode = TWE_OP_WRITE;
+		cmd->io.opcode = TWE_OP_WRITE;
 	    }
 	
 	    /* build a suitable I/O command (assumes 512-byte rounded transfers) */
-	    cmd->sgl_offset = 3;
-	    cmd->size = 3;
-	    cmd->unit = ((struct twed_softc *)bp->bio_dev->si_drv1)->twed_drive->td_unit;
-	    cmd->args.io.lba = bp->bio_pblkno;
-	    cmd->count = (bp->bio_bcount + TWE_BLOCK_SIZE - 1) / TWE_BLOCK_SIZE;
+	    cmd->io.size = 3;
+	    cmd->io.unit = TWE_BIO_UNIT(bp);
+	    cmd->io.block_count = (tr->tr_length + TWE_BLOCK_SIZE - 1) / TWE_BLOCK_SIZE;
+	    cmd->io.lba = TWE_BIO_LBA(bp);
 
 	    /* map the command so the controller can work with it */
 	    twe_map_request(tr);
-	} else {
-
-	    /* we previously deferred a command, try to submit it again */
-	    tr = sc->twe_deferred;
-	    sc->twe_deferred = NULL;
 	}
 	
+	/* did we find something to do? */
+	if (tr == NULL)
+	    break;
+	
 	/* try to give command to controller */
 	error = twe_start(tr);
 
 	if (error != 0) {
 	    if (error == EBUSY) {
-		sc->twe_deferred = tr;		/* try it again later */
+		twe_requeue_ready(tr);		/* try it again later */
 		break;				/* don't try anything more for now */
 	    }
 	    /* otherwise, fail the command */
 	    tr->tr_status = TWE_CMD_FAILED;
 	    twe_completeio(tr);
 	}
-	s = splbio();
     }
-    splx(s);
 }
 
 /********************************************************************************
@@ -968,30 +883,104 @@ twe_startio(struct twe_softc *sc)
 static void
 twe_completeio(struct twe_request *tr)
 {
-    TWE_Command		*cmd;
+    TWE_Command		*cmd = &tr->tr_command;
     struct twe_softc	*sc = tr->tr_sc;
-    struct bio		*bp = (struct bio *)tr->tr_private;
-    struct twed_softc	*twed = (struct twed_softc *)bp->bio_dev->si_drv1;
+    twe_bio		*bp = (twe_bio *)tr->tr_private;
 
     debug_called(4);
 
     if (tr->tr_status == TWE_CMD_COMPLETE) {
-	cmd = &tr->tr_command;
-	if (cmd->status != 0) {
-	    bp->bio_error = EIO;
-	    bp->bio_flags |= BIO_ERROR;
-	    device_printf(twed->twed_dev, "command failed - 0x%x\n", cmd->status);
+
+	switch (cmd->generic.flags) {
+	case TWE_FLAGS_SUCCESS:
+	    break;
+	case TWE_FLAGS_INFORMATIONAL:
+	case TWE_FLAGS_WARNING:
+	    twe_printf(sc, "command completed - %s\n", 
+		       twe_describe_code(twe_table_status, cmd->generic.status));
+	    break;
+
+	case TWE_FLAGS_FATAL:
+	default:
+	    twe_printf(sc, "command failed - %s\n", 
+		       twe_describe_code(twe_table_status, cmd->generic.status));
+	    TWE_BIO_SET_ERROR(bp, EIO);
+
+	    /* 
+	     * XXX some status values suggest that the controller should be reset and all outstanding
+	     * commands retried.  This might be a good place for that.
+	     */
+	    break;
 	}
     } else if (tr->tr_status == TWE_CMD_FAILED) {	/* could be more verbose here? */
-	bp->bio_error = EIO;
-	bp->bio_flags |= BIO_ERROR;
-	device_printf(sc->twe_dev, "command failed submission - controller wedged\n");
+	TWE_BIO_SET_ERROR(bp, EIO);
+	twe_printf(sc, "command failed submission - controller wedged\n");
+	/*
+	 * XXX reset controller and retry?
+	 */
     }
     twe_release_request(tr);
     twed_intr(bp);
 }
 
 /********************************************************************************
+ * Reset the controller and pull all the active commands back onto the ready
+ * queue.  Used to restart a controller that's exhibiting bad behaviour.
+ */
+static void
+twe_reset(struct twe_softc *sc)
+{
+    struct twe_request	*tr;
+    int			i, s;
+
+    twe_printf(sc, "Controller reset in progress...\n");
+
+    /*
+     * Disable interrupts from the controller, and mask any accidental entry
+     * into our interrupt handler.
+     */
+    twe_disable_interrupts(sc);
+    s = splbio();
+
+    /*
+     * Try to soft-reset the controller.
+     */
+    for (i = 0; i < TWE_MAX_RESET_TRIES; i++) {
+
+	if (i > 0)
+	    twe_printf(sc, "reset %d failed, trying again\n", i);
+	
+	if (!twe_soft_reset(sc))
+	    break;			/* reset process complete */
+    }
+    /* did we give up? */
+    if (i >= TWE_MAX_RESET_TRIES) {
+	twe_printf(sc, "can't reset controller, giving up\n");
+	goto out;
+    }
+
+    /*
+     * Move all of the commands that were busy back to the ready queue.
+     */
+    i = 0;
+    while ((tr = twe_dequeue_busy(sc)) != NULL) {
+	twe_enqueue_ready(tr);
+	i++;
+    }
+
+    /*
+     * Kick the controller to start things going again, then re-enable interrupts.
+     */
+    twe_startio(sc);
+    twe_enable_interrupts(sc);
+    twe_printf(sc, "controller reset done, %d commands restarted\n", i);
+
+out:
+    splx(s);
+    twe_enable_interrupts(sc);
+}
+
+/********************************************************************************
  ********************************************************************************
                                                         Command I/O to Controller
  ********************************************************************************
@@ -1011,14 +1000,15 @@ twe_start(struct twe_request *tr)
 
     debug_called(4);
 
-    /* give the command a request ID */
-    if (twe_get_requestid(tr))
-	return(EBUSY);				/* can't handle it now, try later */
-
     /* mark the command as currently being processed */
     tr->tr_status = TWE_CMD_BUSY;
 
-    /* spin briefly waiting for the controller to come ready */
+    /* 
+     * Spin briefly waiting for the controller to come ready 
+     *
+     * XXX it might be more efficient to return EBUSY immediately
+     *     and let the command be rescheduled.
+     */
     for (i = 100000, done = 0; (i > 0) && !done; i--) {
 	s = splbio();
 	
@@ -1030,14 +1020,16 @@ twe_start(struct twe_request *tr)
 	    TWE_COMMAND_QUEUE(sc, tr->tr_cmdphys);
 	    done = 1;
 	    /* move command to work queue */
-	    TAILQ_INSERT_TAIL(&sc->twe_work, tr, tr_link);
+	    twe_enqueue_busy(tr);
+#ifdef TWE_DEBUG
 	    if (tr->tr_complete != NULL) {
-		debug(3, "queued request %d with callback %p", tr->tr_command.request_id, tr->tr_complete);
-	    } else if (tr->tr_private != NULL) {
-		debug(3, "queued request %d with wait channel %p", tr->tr_command.request_id, tr->tr_private);
+		debug(3, "queued request %d with callback %p", tr->tr_command.generic.request_id, tr->tr_complete);
+	    } else if (tr->tr_flags & TWE_CMD_SLEEPER) {
+		debug(3, "queued request %d with wait channel %p", tr->tr_command.generic.request_id, tr);
 	    } else {
-		debug(3, "queued request %d for polling caller", tr->tr_command.request_id);
+		debug(3, "queued request %d for polling caller", tr->tr_command.generic.request_id);
 	    }
+#endif
 	}
 	splx(s);	/* drop spl to allow completion interrupts */
     }
@@ -1052,7 +1044,6 @@ twe_start(struct twe_request *tr)
      * overestimated the number of commands it can accept.  (Should we actually reject
      * the command at this point?)
      */
-    twe_release_requestid(tr);
     return(EBUSY);
 }
 
@@ -1081,11 +1072,13 @@ twe_done(struct twe_softc *sc)
 	if (!(status_reg & TWE_STATUS_RESPONSE_QUEUE_EMPTY)) {
 	    found = 1;
 	    rq = TWE_RESPONSE_QUEUE(sc);
-	    tr = sc->twe_cmdlookup[rq.u.response_id];	/* find command */
+	    tr = sc->twe_lookup[rq.u.response_id];	/* find command */
 	    if (tr != NULL) {				/* paranoia */
-		tr->tr_status = TWE_CMD_COMPLETE;
-		debug(3, "completed request id %d with status %d", tr->tr_command.request_id, tr->tr_command.status);
-		twe_release_requestid(tr);
+		debug(3, "completed request id %d with status %d", 
+		      tr->tr_command.generic.request_id, tr->tr_command.generic.status);
+		/* move to completed queue */
+		twe_remove_busy(tr);
+		twe_enqueue_complete(tr);
 	    } else {
 		debug(2, "done event for nonbusy id %d\n", rq.u.response_id);
 	    }
@@ -1100,7 +1093,7 @@ twe_done(struct twe_softc *sc)
 	twe_startio(sc);
 
     /* handle completion and timeouts */
-    twe_complete(sc);
+    twe_complete(sc);		/* XXX use deferred completion? */
 }
 
 /********************************************************************************
@@ -1112,45 +1105,34 @@ twe_done(struct twe_softc *sc)
 static void
 twe_complete(struct twe_softc *sc) 
 {
-    struct twe_request	*tr, *nr;
-    int			s;
+    struct twe_request	*tr;
     
     debug_called(5);
 
-    s = splbio();
-
     /*
-     * Scan the list of busy/done commands, dispatch them appropriately.
+     * Pull commands off the completed list, dispatch them appropriately
      */
-    tr = TAILQ_FIRST(&sc->twe_work);
-    while (tr != NULL) {
-	nr = TAILQ_NEXT(tr, tr_link);
+    while ((tr = twe_dequeue_complete(sc)) != NULL) {
 
-	/* command has been completed in some fashion */
-	if (tr->tr_status > TWE_CMD_BUSY) {
-	
-	    /* unmap the command's data buffer */
-	    twe_unmap_request(tr);
+	/* unmap the command's data buffer */
+	twe_unmap_request(tr);
 
-	    /* remove from work list */
-	    TAILQ_REMOVE(&sc->twe_work, tr, tr_link);
+	/* mark command as complete */
+	tr->tr_status = TWE_CMD_COMPLETE;
 
-	    /* dispatch to suit command originator */
-	    if (tr->tr_complete != NULL) {		/* completion callback */
-		debug(2, "call completion handler %p", tr->tr_complete);
-		tr->tr_complete(tr);
+	/* dispatch to suit command originator */
+	if (tr->tr_complete != NULL) {		/* completion callback */
+	    debug(2, "call completion handler %p", tr->tr_complete);
+	    tr->tr_complete(tr);
 
-	    } else if (tr->tr_private != NULL) {	/* caller is asleep waiting */
-		debug(2, "wake up command owner on %p", tr->tr_private);
-		wakeup_one(tr->tr_private);
+	} else if (tr->tr_flags & TWE_CMD_SLEEPER) {	/* caller is asleep waiting */
+	    debug(2, "wake up command owner on %p", tr);
+	    wakeup_one(tr);
 
-	    } else {					/* caller is polling command */
-		debug(2, "command left for owner");
-	    }
+	} else {					/* caller is polling command */
+	    debug(2, "command left for owner");
 	}
-	tr = nr;
-    }
-    splx(s);
+    }   
 }
 
 /********************************************************************************
@@ -1203,6 +1185,42 @@ twe_drain_response_queue(struct twe_softc *sc)
 }
 
 /********************************************************************************
+ * Soft-reset the controller
+ */
+static int
+twe_soft_reset(struct twe_softc *sc)
+{
+    u_int32_t		status_reg;
+
+    debug_called(2);
+
+    TWE_SOFT_RESET(sc);
+
+    if (twe_wait_status(sc, TWE_STATUS_ATTENTION_INTERRUPT, 15)) {
+	twe_printf(sc, "no attention interrupt");
+	return(1);
+    }
+    if (twe_drain_aen_queue(sc)) {
+	twe_printf(sc, "can't drain AEN queue\n");
+	return(1);
+    }
+    if (twe_find_aen(sc, TWE_AEN_SOFT_RESET)) {
+	twe_printf(sc, "reset not reported\n");
+	return(1);
+    }
+    status_reg = TWE_STATUS(sc);
+    if (TWE_STATUS_ERRORS(status_reg) || twe_check_bits(sc, status_reg)) {
+	twe_printf(sc, "controller errors detected\n");
+	return(1);
+    }
+    if (twe_drain_response_queue(sc)) {
+	twe_printf(sc, "can't drain response queue\n");
+	return(1);
+    }
+    return(0);
+}
+
+/********************************************************************************
  ********************************************************************************
                                                                Interrupt Handling
  ********************************************************************************
@@ -1218,7 +1236,7 @@ twe_host_intr(struct twe_softc *sc)
 {
     debug_called(4);
 
-    device_printf(sc->twe_dev, "host interrupt\n");
+    twe_printf(sc, "host interrupt\n");
     TWE_CONTROL(sc, TWE_CONTROL_CLEAR_HOST_INTERRUPT);
 }
 
@@ -1234,7 +1252,7 @@ twe_attention_intr(struct twe_softc *sc)
 
     /* instigate a poll for AENs */
     if (twe_fetch_aen(sc)) {
-	device_printf(sc->twe_dev, "error polling for signalled AEN\n");
+	twe_printf(sc, "error polling for signalled AEN\n");
     } else {
 	TWE_CONTROL(sc, TWE_CONTROL_CLEAR_ATTENTION_INTERRUPT);
     }
@@ -1255,34 +1273,11 @@ twe_command_intr(struct twe_softc *sc)
      * them, and when other commands have completed.  Mask it so we don't get
      * another one.
      */
-    device_printf(sc->twe_dev, "command interrupt\n");
+    twe_printf(sc, "command interrupt\n");
     TWE_CONTROL(sc, TWE_CONTROL_MASK_COMMAND_INTERRUPT);
 }
 
 /********************************************************************************
- * Enable the useful interrupts from the controller.
- */
-static void
-twe_enable_interrupts(struct twe_softc *sc)
-{
-    sc->twe_state |= TWE_STATE_INTEN;
-    TWE_CONTROL(sc, 
-	       TWE_CONTROL_CLEAR_ATTENTION_INTERRUPT |
-	       TWE_CONTROL_UNMASK_RESPONSE_INTERRUPT |
-	       TWE_CONTROL_ENABLE_INTERRUPTS);
-}
-
-/********************************************************************************
- * Disable interrupts from the controller.
- */
-static void
-twe_disable_interrupts(struct twe_softc *sc)
-{
-    TWE_CONTROL(sc, TWE_CONTROL_DISABLE_INTERRUPTS);
-    sc->twe_state &= ~TWE_STATE_INTEN;
-}
-
-/********************************************************************************
  ********************************************************************************
                                                       Asynchronous Event Handling
  ********************************************************************************
@@ -1297,8 +1292,7 @@ twe_fetch_aen(struct twe_softc *sc)
 
     debug_called(4);
 
-    /* XXX ick, magic numbers */
-    if ((twe_get_param(sc, 0x401, 2, 2, twe_handle_aen)) == NULL)
+    if ((twe_get_param(sc, TWE_PARAM_AEN, TWE_PARAM_AEN_UnitCode, 2, twe_handle_aen)) == NULL)
 	return(EIO);
     return(0);
 }
@@ -1335,15 +1329,11 @@ twe_handle_aen(struct twe_request *tr)
 static int
 twe_drain_aen_queue(struct twe_softc *sc)
 {
-    TWE_Param	*param;
     u_int16_t	aen;
 
     for (;;) {
-	/* XXX ick, magic numbers */
-	param = twe_get_param(sc, 0x401, 2, 2, NULL);
-	if (param == NULL)
+	if (twe_get_param_2(sc, TWE_PARAM_AEN, TWE_PARAM_AEN_UnitCode, &aen))
 	    return(1);
-	aen = *(u_int16_t *)(param->data);
 	if (aen == TWE_AEN_QUEUE_EMPTY)
 	    return(0);
 	twe_enqueue_aen(sc, aen);
@@ -1361,22 +1351,32 @@ twe_drain_aen_queue(struct twe_softc *sc)
 static void
 twe_enqueue_aen(struct twe_softc *sc, u_int16_t aen)
 {
-    int		s, next;
+    char	*msg;
+    int		s, next, nextnext;
 
     debug_called(4);
 
-    debug(1, "queueing AEN <%s>", twe_name_aen(aen));
+    if ((msg = twe_format_aen(sc, aen)) != NULL)
+	twe_printf(sc, "AEN: <%s>\n", msg);
 
     s = splbio();
     /* enqueue the AEN */
     next = ((sc->twe_aen_head + 1) % TWE_Q_LENGTH);
+    nextnext = ((sc->twe_aen_head + 2) % TWE_Q_LENGTH);
+    
+    /* check to see if this is the last free slot, and subvert the AEN if it is */
+    if (nextnext == sc->twe_aen_tail)
+	aen = TWE_AEN_QUEUE_FULL;
+
+    /* look to see if there's room for this AEN */
     if (next != sc->twe_aen_tail) {
 	sc->twe_aen_queue[sc->twe_aen_head] = aen;
 	sc->twe_aen_head = next;
-    } else {
-	device_printf(sc->twe_dev, "AEN queue overflow, lost AEN <%s>\n", twe_name_aen(aen));
     }
 
+    /* wake up anyone asleep on the queue */
+    wakeup(&sc->twe_aen_queue);
+
     /* anyone looking for this AEN? */
     if (sc->twe_wait_aen == aen) {
 	sc->twe_wait_aen = -1;
@@ -1385,7 +1385,6 @@ twe_enqueue_aen(struct twe_softc *sc, u_int16_t aen)
     splx(s);
 }
 
-#if 0
 /********************************************************************************
  * Pop an AEN off the queue, or return -1 if there are none left.
  *
@@ -1406,7 +1405,6 @@ twe_dequeue_aen(struct twe_softc *sc)
     }
     return(result);
 }
-#endif
 
 /********************************************************************************
  * Check to see if the requested AEN is in the queue.
@@ -1424,6 +1422,7 @@ twe_find_aen(struct twe_softc *sc, u_int16_t aen)
 	if (sc->twe_aen_queue[i] == aen)
 	    missing = 0;
     }
+    splx(s);
     return(missing);
 }
 
@@ -1469,429 +1468,271 @@ twe_wait_aen(struct twe_softc *sc, int aen, int timeout)
 /********************************************************************************
  * Get a new command buffer.
  *
- * This may return NULL in low-memory cases.
- *
- * Note that using malloc() is expensive (the command buffer is << 1 page) but
- * necessary if we are to be a loadable module before the zone allocator is fixed.
- * On the other hand, using malloc ensures that the command structure at the top
- * of the request is aligned within the controller's constraints (64 bytes).
- *
- * If possible, we recycle a command buffer that's been used before.
- *
- * XXX currently, commands are mapped into controller space just before being
- * handed to the controller.  It may be more efficient to do that here.
+ * This will return NULL if all command buffers are in use.
  */
-static struct twe_request *
-twe_get_request(struct twe_softc *sc)
+static int
+twe_get_request(struct twe_softc *sc, struct twe_request **tr)
 {
-    struct twe_request	*tr;
-    int			s, error;
-
     debug_called(4);
 
     /* try to reuse an old buffer */
-    s = splbio();
-    if ((tr = TAILQ_FIRST(&sc->twe_freecmds)) != NULL)
-	TAILQ_REMOVE(&sc->twe_freecmds, tr, tr_link);
-    splx(s);
-
-    /* allocate a new command buffer? */
-    if (tr == NULL) {
-	tr = (struct twe_request *)malloc(sizeof(*tr), M_DEVBUF, M_NOWAIT);
-	if (tr != NULL) {
-	    bzero(tr, sizeof(*tr));
-	    tr->tr_sc = sc;
-	    error = bus_dmamap_create(sc->twe_buffer_dmat, 0, &tr->tr_cmdmap);
-	    if (error) {
-		free(tr, M_DEVBUF);
-		return(NULL);
-	    }
-	    error = bus_dmamap_create(sc->twe_buffer_dmat, 0, &tr->tr_dmamap);
-	    if (error) {
-		bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_cmdmap);
-		free(tr, M_DEVBUF);
-		return(NULL);
-	    }
-	}
-    }
+    *tr = twe_dequeue_free(sc);
 
     /* initialise some fields to their defaults */
-    tr->tr_status = TWE_CMD_SETUP;		/* command is in setup phase */
-    tr->tr_flags = 0;
-    tr->tr_command.host_id = 0;			/* not used */
-    tr->tr_command.status = 0;			/* before submission to controller */
-    tr->tr_command.flags = 0;			/* not used */
-    return(tr);
+    if (*tr != NULL) {
+	(*tr)->tr_data = NULL;
+	(*tr)->tr_status = TWE_CMD_SETUP;		/* command is in setup phase */
+	(*tr)->tr_flags = 0;
+	(*tr)->tr_complete = NULL;
+	(*tr)->tr_command.generic.status = 0;		/* before submission to controller */
+	(*tr)->tr_command.generic.flags = 0;		/* not used */
+    }
+    return(*tr == NULL);
 }
 
 /********************************************************************************
- * Release a command buffer for recycling.
+ * Release a command buffer for reuse.
  *
- * XXX It might be a good idea to limit the number of commands we save for reuse
- *     if it's shown that this list bloats out massively.
  */
 static void
 twe_release_request(struct twe_request *tr)
 {
-    int		s;
-    
     debug_called(4);
 
-    s = splbio();
-    TAILQ_INSERT_HEAD(&tr->tr_sc->twe_freecmds, tr, tr_link);
-    splx(s);
+    twe_enqueue_free(tr);
 }
 
 /********************************************************************************
- * Permanently discard a command buffer.
- */
-static void
-twe_free_request(struct twe_request *tr) 
-{
-    struct twe_softc	*sc = tr->tr_sc;
-    
-    debug_called(4);
-
-    bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_cmdmap);
-    bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_dmamap);
-    free(tr, M_DEVBUF);
-}
+ ********************************************************************************
+                                                                        Debugging
+ ********************************************************************************
+ ********************************************************************************/
 
 /********************************************************************************
- * Allocate a request ID for a command about to be submitted.
+ * Print some information about the controller
  */
-static int
-twe_get_requestid(struct twe_request *tr)
+void
+twe_describe_controller(struct twe_softc *sc)
 {
-    struct twe_softc	*sc = tr->tr_sc;
-    int			i, s, result;
-
-    debug_called(4);
-
-    s = splbio();
-    result = 1;
+    TWE_Param		*p[6];
+    u_int8_t		ports;
+    u_int32_t		size;
+    int			i;
 
-    /* XXX linear search is slow */
-    for (i = 0; i < TWE_Q_LENGTH; i++) {
-	if (sc->twe_cmdlookup[i] == NULL) {
-	    tr->tr_command.request_id = i;
-	    sc->twe_cmdlookup[i] = tr;
-	    result = 0;
-	    break;
+    debug_called(2);
+
+    /* get the port count */
+    twe_get_param_1(sc, TWE_PARAM_CONTROLLER, TWE_PARAM_CONTROLLER_PortCount, &ports);
+
+    /* get version strings */
+    p[0] = twe_get_param(sc, TWE_PARAM_VERSION, TWE_PARAM_VERSION_Mon,  16, NULL);
+    p[1] = twe_get_param(sc, TWE_PARAM_VERSION, TWE_PARAM_VERSION_FW,   16, NULL);
+    p[2] = twe_get_param(sc, TWE_PARAM_VERSION, TWE_PARAM_VERSION_BIOS, 16, NULL);
+    p[3] = twe_get_param(sc, TWE_PARAM_VERSION, TWE_PARAM_VERSION_PCB,  8, NULL);
+    p[4] = twe_get_param(sc, TWE_PARAM_VERSION, TWE_PARAM_VERSION_ATA,  8, NULL);
+    p[5] = twe_get_param(sc, TWE_PARAM_VERSION, TWE_PARAM_VERSION_PCI,  8, NULL);
+
+    twe_printf(sc, "%d ports, Firmware %.16s, BIOS %.16s\n", ports, p[1]->data, p[2]->data);
+    if (bootverbose)
+	twe_printf(sc, "Monitor %.16s, PCB %.8s, Achip %.8s, Pchip %.8s\n", p[0]->data, p[3]->data,
+		   p[4]->data, p[5]->data);
+    free(p[0], M_DEVBUF);
+    free(p[1], M_DEVBUF);
+    free(p[2], M_DEVBUF);
+    free(p[3], M_DEVBUF);
+    free(p[4], M_DEVBUF);
+    free(p[5], M_DEVBUF);
+
+    /* print attached drives */
+    if (bootverbose) {
+	p[0] = twe_get_param(sc, TWE_PARAM_DRIVESUMMARY, TWE_PARAM_DRIVESUMMARY_Status, 16, NULL);
+	for (i = 0; i < ports; i++) {
+	    if (p[0]->data[i] != TWE_PARAM_DRIVESTATUS_Present)
+		continue;
+	    twe_get_param_4(sc, TWE_PARAM_DRIVEINFO + i, TWE_PARAM_DRIVEINFO_Size, &size);
+	    p[1] = twe_get_param(sc, TWE_PARAM_DRIVEINFO + i, TWE_PARAM_DRIVEINFO_Model, 40, NULL);
+	    if (p[1] != NULL) {
+		twe_printf(sc, "port %d: %.40s %dMB\n", i, p[1]->data, size / 2048);
+		free(p[1], M_DEVBUF);
+	    } else {
+		twe_printf(sc, "port %d, drive status unavailable\n", i);
+	    }
 	}
+	free(p[0], M_DEVBUF);
     }
-    splx(s);
-
-    return(result);
-}
-
-/********************************************************************************
- * Free a command's request ID for reuse.
- */
-static void
-twe_release_requestid(struct twe_request *tr)
-{
-    struct twe_softc	*sc = tr->tr_sc;
-
-    debug_called(4);
-
-    sc->twe_cmdlookup[tr->tr_command.request_id] = 0;	/* XXX atomic? */
 }
 
 /********************************************************************************
- * Map/unmap (tr)'s command and data in the controller's addressable space.
+ * Complain if the status bits aren't what we're expecting.
  *
- * These routines ensure that the data which the controller is going to try to
- * access is actually visible to the controller, in a machine-independant 
- * fasion.  Due to a hardware limitation, I/O buffers must be 512-byte aligned
- * and we take care of that here as well.
+ * Rate-limit the complaints to at most one of each every five seconds, but
+ * always return the correct status.
  */
-static void
-twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
-{
-    struct twe_request	*tr = (struct twe_request *)arg;
-    TWE_Command		*cmd = &tr->tr_command;
-    int			i;
-
-    debug_called(4);
-
-    /* save base of first segment in command (applicable if there only one segment) */
-    tr->tr_dataphys = segs[0].ds_addr;
-
-    /* correct command size for s/g list size */
-    tr->tr_command.size += 2 * nsegments;
-
-    /*
-     * Due to the fact that parameter and I/O commands have the scatter/gather list in
-     * different places, we need to determine which sort of command this actually is
-     * before we can populate it correctly.
-     */
-    switch(cmd->sgl_offset) {
-    case 2:
-	for (i = 0; i < nsegments; i++) {
-	    cmd->args.param.sgl[i].address = segs[i].ds_addr;
-	    cmd->args.param.sgl[i].length = segs[i].ds_len;
-	}
-	for (; i < TWE_MAX_SGL_LENGTH; i++) {		/* XXX necessary? */
-	    cmd->args.param.sgl[i].address = 0;
-	    cmd->args.param.sgl[i].length = 0;
-	}
-	break;
-    case 3:
-	for (i = 0; i < nsegments; i++) {
-	    cmd->args.io.sgl[i].address = segs[i].ds_addr;
-	    cmd->args.io.sgl[i].length = segs[i].ds_len;
-	}
-	for (; i < TWE_MAX_SGL_LENGTH; i++) {		/* XXX necessary? */
-	    cmd->args.io.sgl[i].address = 0;
-	    cmd->args.io.sgl[i].length = 0;
-	}
-	break;
-    default:
-	/* no s/g list, nothing to do */
-    }
-}
-
-static void
-twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
-{
-    struct twe_request	*tr = (struct twe_request *)arg;
-
-    debug_called(4);
-
-    /* command can't cross a page boundary */
-    tr->tr_cmdphys = segs[0].ds_addr;
-}
-
-static void
-twe_map_request(struct twe_request *tr)
-{
-    struct twe_softc	*sc = tr->tr_sc;
-
-    debug_called(4);
-
-
-    /*
-     * Map the command into bus space.
-     */
-    bus_dmamap_load(sc->twe_buffer_dmat, tr->tr_cmdmap, &tr->tr_command, sizeof(tr->tr_command), 
-		    twe_setup_request_dmamap, tr, 0);
-    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_cmdmap, BUS_DMASYNC_PREWRITE);
-
-    /*
-     * If the command involves data, map that too.
-     */
-    if (tr->tr_data != NULL) {
-
-	/* 
-	 * Data must be 64-byte aligned; allocate a fixup buffer if it's not.
-	 */
-	if (((vm_offset_t)tr->tr_data % TWE_ALIGNMENT) != 0) {
-	    tr->tr_realdata = tr->tr_data;				/* save pointer to 'real' data */
-	    tr->tr_flags |= TWE_CMD_ALIGNBUF;
-	    tr->tr_data = malloc(tr->tr_length, M_DEVBUF, M_NOWAIT);	/* XXX check result here */
-	}
-	
-	/*
-	 * Map the data buffer into bus space and build the s/g list.
-	 */
-	bus_dmamap_load(sc->twe_buffer_dmat, tr->tr_dmamap, tr->tr_data, tr->tr_length, 
-			twe_setup_data_dmamap, tr, 0);
-	if (tr->tr_flags & TWE_CMD_DATAIN)
-	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_PREREAD);
-	if (tr->tr_flags & TWE_CMD_DATAOUT) {
-	    /* if we're using an alignment buffer, and we're writing data, copy the real data out */
-	    if (tr->tr_flags & TWE_CMD_ALIGNBUF)
-		bcopy(tr->tr_realdata, tr->tr_data, tr->tr_length);
-	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_PREWRITE);
-	}
-    }
-}
-
-static void
-twe_unmap_request(struct twe_request *tr)
+static int
+twe_check_bits(struct twe_softc *sc, u_int32_t status_reg)
 {
-    struct twe_softc	*sc = tr->tr_sc;
-
-    debug_called(4);
+    int			result;
+    static time_t	lastwarn[2] = {0, 0};
 
     /*
-     * Unmap the command from bus space.
+     * This can be a little problematic, as twe_panic may call twe_reset if 
+     * TWE_DEBUG is not set, which will call us again as part of the soft reset.
      */
-    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_cmdmap, BUS_DMASYNC_POSTWRITE);
-    bus_dmamap_unload(sc->twe_buffer_dmat, tr->tr_cmdmap); 
-
-    /*
-     * If the command involved data, unmap that too.
-     */
-    if (tr->tr_data != NULL) {
-	
-	if (tr->tr_flags & TWE_CMD_DATAIN) {
-	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_POSTREAD);
-	    /* if we're using an alignment buffer, and we're reading data, copy the real data in */
-	    if (tr->tr_flags & TWE_CMD_ALIGNBUF)
-		bcopy(tr->tr_data, tr->tr_realdata, tr->tr_length);
-	}
-	if (tr->tr_flags & TWE_CMD_DATAOUT)
-	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_POSTWRITE);
-
-	bus_dmamap_unload(sc->twe_buffer_dmat, tr->tr_dmamap); 
+    if ((status_reg & TWE_STATUS_PANIC_BITS) != 0) {
+	twe_printf(sc, "FATAL STATUS BIT(S) %b\n", status_reg & TWE_STATUS_PANIC_BITS,
+		   TWE_STATUS_BITS_DESCRIPTION);
+	twe_panic(sc, "fatal status bits");
     }
 
-    /* free alignment buffer if it was used */
-    if (tr->tr_flags & TWE_CMD_ALIGNBUF) {
-	free(tr->tr_data, M_DEVBUF);
-	tr->tr_data = tr->tr_realdata;		/* restore 'real' data pointer */
-    }
-}
-
-/********************************************************************************
- ********************************************************************************
-                                                                        Debugging
- ********************************************************************************
- ********************************************************************************/
-
-/********************************************************************************
- * Complain if the status bits aren't what we're expecting.
- */
-static int
-twe_check_bits(struct twe_softc *sc, u_int32_t status_reg)
-{
-    int		result;
-
     result = 0;
     if ((status_reg & TWE_STATUS_EXPECTED_BITS) != TWE_STATUS_EXPECTED_BITS) {
-	device_printf(sc->twe_dev, "missing expected status bit(s) %b\n", ~status_reg & TWE_STATUS_EXPECTED_BITS, 
-		      TWE_STATUS_BITS_DESCRIPTION);
+	if (time_second > (lastwarn[0] + 5)) {
+	    twe_printf(sc, "missing expected status bit(s) %b\n", ~status_reg & TWE_STATUS_EXPECTED_BITS, 
+		       TWE_STATUS_BITS_DESCRIPTION);
+	    lastwarn[0] = time_second;
+	}
 	result = 1;
     }
 
     if ((status_reg & TWE_STATUS_UNEXPECTED_BITS) != 0) {
-	device_printf(sc->twe_dev, "unexpected status bit(s) %b\n", status_reg & TWE_STATUS_UNEXPECTED_BITS, 
-		      TWE_STATUS_BITS_DESCRIPTION);
+	if (time_second > (lastwarn[1] + 5)) {
+	    twe_printf(sc, "unexpected status bit(s) %b\n", status_reg & TWE_STATUS_UNEXPECTED_BITS, 
+		       TWE_STATUS_BITS_DESCRIPTION);
+	    lastwarn[1] = time_second;
+	}
 	result = 1;
     }
+
     return(result);
 }	
 
 /********************************************************************************
- * Return a string naming (aen).
+ * Return a string describing (aen).
+ *
+ * The low 8 bits of the aen are the code, the high 8 bits give the unit number
+ * where an AEN is specific to a unit.
+ *
+ * Note that we could expand this routine to handle eg. up/downgrading the status
+ * of a drive if we had some idea of what the drive's initial status was.
  */
-static struct {
-    u_int16_t	aen;
-    char	*desc;
-} twe_aen_names[] = {
-    {0x0000,	"queue empty"},
-    {0x0001,	"soft reset"},
-    {0x0002,	"degraded mirror"},
-    {0x0003,	"controller error"},
-    {0x0004,	"rebuild fail"},
-    {0x0005,	"rebuild done"},
-    {0x00ff,	"aen queue full"},
-    {0, NULL}
-};
 
 static char *
-twe_name_aen(u_int16_t aen)
+twe_format_aen(struct twe_softc *sc, u_int16_t aen)
 {
-    int		i;
-    static char	buf[16];
-
-    for (i = 0; twe_aen_names[i].desc != NULL; i++)
-	if (twe_aen_names[i].aen == aen)
-	    return(twe_aen_names[i].desc);
-
-    sprintf(buf, "0x%x", aen);
+    static char	buf[80];
+    device_t	child;
+    char	*code, *msg;
+
+    code = twe_describe_code(twe_table_aen, TWE_AEN_CODE(aen));
+    msg = code + 2;
+
+    switch (*code) {
+    case 'q':
+	if (!bootverbose)
+	    return(NULL);
+	/* FALLTHROUGH */
+    case 'p':
+	return(msg);
+
+    case 'c':
+	if ((child = sc->twe_drive[TWE_AEN_UNIT(aen)].td_disk) != NULL) {
+	    sprintf(buf, "twed%d: %s", device_get_unit(child), msg);
+	} else {
+	    sprintf(buf, "twe%d: %s for unknown unit %d", device_get_unit(sc->twe_dev),
+		    msg, TWE_AEN_UNIT(aen));
+	}
+	return(buf);
+	
+    case 'x':
+    default:
+	break;
+    }
+    sprintf(buf, "unknown AEN 0x%x", aen);
     return(buf);
 }
 
-#if 0
-/********************************************************************************
- * Return a string naming (opcode).
- */
-static struct {
-    u_int8_t	opcode;
-    char	*desc;
-} twe_opcode_names[] = {
-    {0x00,	"nop"},
-    {0x01,	"init connection"},
-    {0x02,	"read"},
-    {0x03,	"write"},
-    {0x04,	"verify"},
-    {0x12,	"get param"},
-    {0x13,	"set param"},
-    {0x1a,	"sector info"},
-    {0x1c,	"listen"},
-    {0, NULL}
-};
+static int
+twe_request_qlen(struct twe_request *tr)
+{
+    int len = 0;
 
-static char *
-twe_name_opcode(u_int8_t opcode)
+    while (tr != NULL) {
+	tr = TAILQ_NEXT(tr, tr_link);
+	len++;
+    }
+    return(len);
+}
+
+void
+twe_print_controller(struct twe_softc *sc)
 {
-    int		i;
-    static char	buf[16];
+    u_int32_t		status_reg;
 
-    for (i = 0; twe_opcode_names[i].desc != NULL; i++)
-	if (twe_opcode_names[i].opcode == opcode)
-	    return(twe_opcode_names[i].desc);
+    status_reg = TWE_STATUS(sc);
+    twe_printf(sc, "status   %b\n", status_reg, TWE_STATUS_BITS_DESCRIPTION);
+    twe_printf(sc, "free     %d\n", twe_request_qlen(TAILQ_FIRST(&sc->twe_free)));
+    twe_printf(sc, "ready    %d\n", twe_request_qlen(TAILQ_FIRST(&sc->twe_ready)));
+    twe_printf(sc, "busy     %d\n", twe_request_qlen(TAILQ_FIRST(&sc->twe_busy)));
+    twe_printf(sc, "complete %d\n", twe_request_qlen(TAILQ_FIRST(&sc->twe_complete)));
+    twe_printf(sc, "AEN queue head %d  tail %d\n", sc->twe_aen_head, sc->twe_aen_tail);
+}	
 
-    sprintf(buf, "0x%x", opcode);
-    return(buf);
+static void
+twe_panic(struct twe_softc *sc, char *reason)
+{
+    twe_print_controller(sc);
+#ifdef TWE_DEBUG
+    panic(reason);
+#else
+    twe_reset(sc);
+#endif
 }
 
+#if 0
 /********************************************************************************
  * Print a request/command in human-readable format.
  */
 static void
 twe_print_request(struct twe_request *tr)
 {
-    device_t	dev = tr->tr_sc->twe_dev;
+    struct twe_softc	*sc = tr->tr_sc;
     TWE_Command	*cmd = &tr->tr_command;
     int		i;
 
-    device_printf(dev, "CMD: request_id %d  opcode <%s>  size %d  unit %d  host_id %d\n", 
-		  cmd->request_id, twe_name_opcode(cmd->opcode), cmd->size, cmd->unit, cmd->host_id);
-    device_printf(dev, " status %d  flags 0x%x  count %d  sgl_offset %d\n", 
-		  cmd->status, cmd->flags, cmd->count, cmd->sgl_offset);
-    switch(cmd->sgl_offset) {
-    case 3:
-	device_printf(dev, " lba %d\n", cmd->args.io.lba);
-	for (i = 0; (i < TWE_MAX_SGL_LENGTH) && (cmd->args.io.sgl[i].length != 0); i++)
-	    device_printf(dev, "  %d: 0x%x/%d\n", 
-			  i, cmd->args.io.sgl[i].address, cmd->args.io.sgl[i].length);
+    twe_printf(sc, "CMD: request_id %d  opcode <%s>  size %d  unit %d  host_id %d\n", 
+	       cmd->generic.request_id, twe_describe_code(twe_table_opcode, cmd->generic.opcode), cmd->generic.size, 
+	       cmd->generic.unit, cmd->generic.host_id);
+    twe_printf(sc, " status %d  flags 0x%x  count %d  sgl_offset %d\n", 
+	       cmd->generic.status, cmd->generic.flags, cmd->generic.count, cmd->generic.sgl_offset);
+
+    switch(cmd->generic.opcode) {	/* XXX add more opcodes? */
+    case TWE_OP_READ:
+    case TWE_OP_WRITE:
+	twe_printf(sc, " lba %d\n", cmd->io.lba);
+	for (i = 0; (i < TWE_MAX_SGL_LENGTH) && (cmd->io.sgl[i].length != 0); i++)
+	    twe_printf(sc, "  %d: 0x%x/%d\n", 
+		       i, cmd->io.sgl[i].address, cmd->io.sgl[i].length);
+	break;
+
+    case TWE_OP_GET_PARAM:
+    case TWE_OP_SET_PARAM:
+	for (i = 0; (i < TWE_MAX_SGL_LENGTH) && (cmd->param.sgl[i].length != 0); i++)
+	    twe_printf(sc, "  %d: 0x%x/%d\n", 
+		       i, cmd->param.sgl[i].address, cmd->param.sgl[i].length);
 	break;
 
-    case 2:
-	for (i = 0; (i < TWE_MAX_SGL_LENGTH) && (cmd->args.param.sgl[i].length != 0); i++)
-	    device_printf(dev, "  %d: 0x%x/%d\n", 
-			  i, cmd->args.param.sgl[i].address, cmd->args.param.sgl[i].length);
+    case TWE_OP_INIT_CONNECTION:
+	twe_printf(sc, " response queue pointer 0x%x\n", 
+		   cmd->initconnection.response_queue_pointer);
 	break;
 
     default:
-	device_printf(dev, " response queue pointer 0x%x\n", 
-		      cmd->args.init_connection.response_queue_pointer);
+	break;
     }
-    device_printf(dev, " tr_command %p/0x%x  tr_data %p/0x%x,%d\n", 
-		  tr, tr->tr_cmdphys, tr->tr_data, tr->tr_dataphys, tr->tr_length);
-    device_printf(dev, " tr_status %d  tr_flags 0x%x  tr_complete %p  tr_private %p\n", 
-		  tr->tr_status, tr->tr_flags, tr->tr_complete, tr->tr_private);
+    twe_printf(sc, " tr_command %p/0x%x  tr_data %p/0x%x,%d\n", 
+	       tr, tr->tr_cmdphys, tr->tr_data, tr->tr_dataphys, tr->tr_length);
+    twe_printf(sc, " tr_status %d  tr_flags 0x%x  tr_complete %p  tr_private %p\n", 
+	       tr->tr_status, tr->tr_flags, tr->tr_complete, tr->tr_private);
 }
 
-/********************************************************************************
- * Print current controller status, call from DDB.
- */
-void
-twe_report(void)
-{
-    u_int32_t		status_reg;
-    struct twe_softc	*sc;
-    int			i, s;
-
-    s = splbio();
-    for (i = 0; (sc = devclass_get_softc(twe_devclass, i)) != NULL; i++) {
-	status_reg = TWE_STATUS(sc);
-	device_printf(sc->twe_dev, "status %b\n", status_reg, TWE_STATUS_BITS_DESCRIPTION);
-    }
-    splx(s);
-}
 #endif
diff --git a/sys/dev/twe/twe_compat.h b/sys/dev/twe/twe_compat.h
new file mode 100644
index 0000000..c2f75e1
--- /dev/null
+++ b/sys/dev/twe/twe_compat.h
@@ -0,0 +1,179 @@
+/*-
+ * Copyright (c) 2000 Michael Smith
+ * Copyright (c) 2000 BSDi
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+/*
+ * Portability and compatibility interfaces.
+ */
+
+#ifdef __FreeBSD__
+/******************************************************************************
+ * FreeBSD
+ */
+#define TWE_SUPPORTED_PLATFORM
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/malloc.h>
+#include <sys/kernel.h>
+
+#include <sys/bus.h>
+#include <sys/conf.h>
+#include <sys/disk.h>
+#include <sys/stat.h>
+
+#include <machine/bus_pio.h>
+#include <machine/bus.h>
+#include <machine/resource.h>
+#include <sys/rman.h>
+
+#include <pci/pcireg.h>
+#include <pci/pcivar.h>
+
+/*
+ * These macros allows us to build a version of the driver which can 
+ * safely be loaded into a kernel which already contains a 'twe' driver,
+ * and which will override it in all things.
+ *
+ * All public symbols must be listed here.
+ */
+#ifdef TWE_OVERRIDE
+#define twe_setup		Xtwe_setup
+#define twe_init		Xtwe_init
+#define twe_deinit		Xtwe_deinit
+#define twe_intr		Xtwe_intr
+#define twe_submit_bio		Xtwe_submit_bio
+#define twe_ioctl		Xtwe_ioctl
+#define twe_describe_controller	Xtwe_describe_controller
+#define twe_print_controller	Xtwe_print_controller
+#define twe_enable_interrupts	Xtwe_enable_interrupts
+#define twe_disable_interrupts	Xtwe_disable_interrupts
+#define twe_attach_drive	Xtwe_attach_drive
+#define twed_intr		Xtwed_intr
+#define twe_allocate_request	Xtwe_allocate_request
+#define twe_free_request	Xtwe_free_request
+#define twe_map_request		Xtwe_map_request
+#define twe_unmap_request	Xtwe_unmap_request
+#define twe_describe_code	Xtwe_describe_code
+#define twe_table_status	Xtwe_table_status
+#define twe_table_unitstate	Xtwe_table_unitstate
+#define twe_table_unittype	Xtwe_table_unittype
+#define twe_table_aen		Xtwe_table_aen
+#define TWE_DRIVER_NAME		Xtwe
+#define TWED_DRIVER_NAME	Xtwed
+#define TWE_MALLOC_CLASS	M_XTWE
+#else
+#define TWE_DRIVER_NAME		twe
+#define TWED_DRIVER_NAME	twed
+#define TWE_MALLOC_CLASS	M_TWE
+#endif
+
+/* 
+ * Wrappers for bus-space actions
+ */
+#define TWE_CONTROL(sc, val)		bus_space_write_4(sc->twe_btag, sc->twe_bhandle, 0x0, (u_int32_t)val)
+#define TWE_STATUS(sc)			(u_int32_t)bus_space_read_4(sc->twe_btag, sc->twe_bhandle, 0x4)
+#define TWE_COMMAND_QUEUE(sc, val)	bus_space_write_4(sc->twe_btag, sc->twe_bhandle, 0x8, (u_int32_t)val)
+#define TWE_RESPONSE_QUEUE(sc)		(TWE_Response_Queue)bus_space_read_4(sc->twe_btag, sc->twe_bhandle, 0xc)
+
+/*
+ * FreeBSD-specific softc elements
+ */
+#define TWE_PLATFORM_SOFTC								\
+    device_t			twe_dev;		/* bus device */		\
+    dev_t			twe_dev_t;		/* control device */		\
+    struct resource		*twe_io;		/* register interface window */	\
+    bus_space_handle_t		twe_bhandle;		/* bus space handle */		\
+    bus_space_tag_t		twe_btag;		/* bus space tag */		\
+    bus_dma_tag_t		twe_parent_dmat;	/* parent DMA tag */		\
+    bus_dma_tag_t		twe_buffer_dmat;	/* data buffer DMA tag */	\
+    struct resource		*twe_irq;		/* interrupt */			\
+    void			*twe_intr;		/* interrupt handle */		\
+    struct intr_config_hook	twe_ich;		/* delayed-startup hook */
+
+/*
+ * FreeBSD-specific request elements
+ */
+#define TWE_PLATFORM_REQUEST										\
+    bus_dmamap_t		tr_cmdmap;	/* DMA map for command */				\
+    u_int32_t			tr_cmdphys;	/* address of command in controller space */		\
+    bus_dmamap_t		tr_dmamap;	/* DMA map for data */					\
+    u_int32_t			tr_dataphys;	/* data buffer base address in controller space */
+
+/*
+ * Output identifying the controller/disk
+ */
+#define twe_printf(sc, fmt, args...)	device_printf(sc->twe_dev, fmt , ##args)
+#define twed_printf(twed, fmt, args...)	device_printf(twed->twed_dev, fmt , ##args)
+
+#if __FreeBSD_version < 500003		/* old buf style */
+# include <sys/buf.h>
+typedef struct buf			twe_bio;
+typedef struct buf_queue_head		twe_bioq;
+# define TWE_BIO_QINIT(bq)		bufq_init(&bq);
+# define TWE_BIO_QINSERT(bq, bp)	bufq_insert_tail(&bq, bp)
+# define TWE_BIO_QFIRST(bq)		bufq_first(&bq)
+# define TWE_BIO_QREMOVE(bq, bp)	bufq_remove(&bq, bp)
+# define TWE_BIO_IS_READ(bp)		((bp)->b_flags & B_READ)
+# define TWE_BIO_DATA(bp)		(bp)->b_data
+# define TWE_BIO_LENGTH(bp)		(bp)->b_bcount
+# define TWE_BIO_LBA(bp)		(bp)->b_pblkno
+# define TWE_BIO_SOFTC(bp)		(bp)->b_dev->si_drv1
+# define TWE_BIO_UNIT(bp)		*(int *)((bp)->b_dev->si_drv2)
+# define TWE_BIO_SET_ERROR(bp, err)	do { (bp)->b_error = err; (bp)->b_flags |= B_ERROR;} while(0)
+# define TWE_BIO_HAS_ERROR(bp)		((bp)->b_flags & B_ERROR)
+# define TWE_BIO_RESID(bp)		(bp)->b_resid
+# define TWE_BIO_DONE(bp)		biodone(bp)
+# define TWE_BIO_STATS_START(bp)	devstat_start_transaction(&((struct twed_softc *)TWE_BIO_SOFTC(bp))->twed_stats)
+# define TWE_BIO_STATS_END(bp)		devstat_end_transaction_buf(&((struct twed_softc *)TWE_BIO_SOFTC(bp))->twed_stats, bp)
+#else
+# include <sys/bio.h>
+typedef struct bio			twe_bio;
+typedef struct bio_queue_head		twe_bioq;
+# define TWE_BIO_QINIT(bq)		bioq_init(&bq);
+# define TWE_BIO_QINSERT(bq, bp)	bioq_insert_tail(&bq, bp)
+# define TWE_BIO_QFIRST(bq)		bioq_first(&bq)
+# define TWE_BIO_QREMOVE(bq, bp)	bioq_remove(&bq, bp)
+# define TWE_BIO_IS_READ(bp)		((bp)->bio_cmd == BIO_READ)
+# define TWE_BIO_DATA(bp)		(bp)->bio_data
+# define TWE_BIO_LENGTH(bp)		(bp)->bio_bcount
+# define TWE_BIO_LBA(bp)		(bp)->bio_pblkno
+# define TWE_BIO_SOFTC(bp)		(bp)->bio_dev->si_drv1
+# define TWE_BIO_UNIT(bp)		*(int *)((bp)->bio_dev->si_drv2)
+# define TWE_BIO_SET_ERROR(bp, err)	do { (bp)->bio_error = err; (bp)->bio_flags |= BIO_ERROR;} while(0)
+# define TWE_BIO_HAS_ERROR(bp)		((bp)->bio_flags & BIO_ERROR)
+# define TWE_BIO_RESID(bp)		(bp)->bio_resid
+# define TWE_BIO_DONE(bp)		biodone(bp)
+# define TWE_BIO_STATS_START(bp)	devstat_start_transaction(&((struct twed_softc *)TWE_BIO_SOFTC(bp))->twed_stats)
+# define TWE_BIO_STATS_END(bp)		devstat_end_transaction_bio(&((struct twed_softc *)TWE_BIO_SOFTC(bp))->twed_stats, bp)
+#endif
+
+#endif /* FreeBSD */
+
+#ifndef TWE_SUPPORTED_PLATFORM
+#error platform not supported
+#endif
diff --git a/sys/dev/twe/twe_disk.c b/sys/dev/twe/twe_disk.c
deleted file mode 100644
index bb8fba1..0000000
--- a/sys/dev/twe/twe_disk.c
+++ /dev/null
@@ -1,289 +0,0 @@
-/*-
- * Copyright (c) 2000 Michael Smith
- * Copyright (c) 2000 BSDi
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/kernel.h>
-
-#if __FreeBSD_version < 500000
-# include <dev/twe/twe_compat.h>
-#else
-# include <sys/bio.h>
-#endif
-#include <sys/bus.h>
-#include <sys/conf.h>
-#include <sys/devicestat.h>
-#include <sys/disk.h>
-
-#include <machine/bus.h>
-#include <sys/rman.h>
-
-#include <dev/twe/twereg.h>
-#include <dev/twe/twevar.h>
-
-/*
- * Interface to controller.
- */
-static int twed_probe(device_t dev);
-static int twed_attach(device_t dev);
-static int twed_detach(device_t dev);
-
-/*
- * Interface to the device switch
- */
-static	d_open_t	twed_open;
-static	d_close_t	twed_close;
-static	d_strategy_t	twed_strategy;
-
-#define TWED_CDEV_MAJOR	147
-
-static struct cdevsw twed_cdevsw = {
-    /* open */		twed_open,
-    /* close */		twed_close,
-    /* read */		physread,
-    /* write */		physwrite,
-    /* ioctl */		noioctl,
-    /* poll */		nopoll,
-    /* mmap */		nommap,
-    /* strategy */	twed_strategy,
-    /* name */ 		"twed",
-    /* maj */		TWED_CDEV_MAJOR,
-    /* dump */		nodump,
-    /* psize */ 	nopsize,
-    /* flags */		D_DISK,
-    /* bmaj */		-1
-};
-
-devclass_t		twed_devclass;
-static struct cdevsw	tweddisk_cdevsw;
-#ifdef FREEBSD_4
-static int		disks_registered = 0;
-#endif
-
-static device_method_t twed_methods[] = {
-    DEVMETHOD(device_probe,	twed_probe),
-    DEVMETHOD(device_attach,	twed_attach),
-    DEVMETHOD(device_detach,	twed_detach),
-    { 0, 0 }
-};
-
-static driver_t twed_driver = {
-    "twed",
-    twed_methods,
-    sizeof(struct twed_softc)
-};
-
-DRIVER_MODULE(twed, twe, twed_driver, twed_devclass, 0, 0);
-
-/********************************************************************************
- * Handle open from generic layer.
- *
- * Note that this is typically only called by the diskslice code, and not
- * for opens on subdevices (eg. slices, partitions).
- */
-static int
-twed_open(dev_t dev, int flags, int fmt, struct proc *p)
-{
-    struct twed_softc	*sc = (struct twed_softc *)dev->si_drv1;
-    struct disklabel	*label;
-
-    debug_called(4);
-	
-    if (sc == NULL)
-	return (ENXIO);
-
-    /* check that the controller is up and running */
-    if (sc->twed_controller->twe_state & TWE_STATE_SHUTDOWN)
-	return(ENXIO);
-
-    /* build synthetic label */
-    label = &sc->twed_disk.d_label;
-    bzero(label, sizeof(*label));
-    label->d_type = DTYPE_ESDI;
-    label->d_secsize    = TWE_BLOCK_SIZE;
-    label->d_nsectors   = sc->twed_drive->td_sectors;
-    label->d_ntracks    = sc->twed_drive->td_heads;
-    label->d_ncylinders = sc->twed_drive->td_cylinders;
-    label->d_secpercyl  = sc->twed_drive->td_sectors * sc->twed_drive->td_heads;
-    label->d_secperunit = sc->twed_drive->td_size;
-
-    sc->twed_flags |= TWED_OPEN;
-    return (0);
-}
-
-/********************************************************************************
- * Handle last close of the disk device.
- */
-static int
-twed_close(dev_t dev, int flags, int fmt, struct proc *p)
-{
-    struct twed_softc	*sc = (struct twed_softc *)dev->si_drv1;
-
-    debug_called(4);
-	
-    if (sc == NULL)
-	return (ENXIO);
-
-    sc->twed_flags &= ~TWED_OPEN;
-    return (0);
-}
-
-/********************************************************************************
- * Handle an I/O request.
- */
-static void
-twed_strategy(struct bio *bp)
-{
-    struct twed_softc	*sc = (struct twed_softc *)bp->bio_dev->si_drv1;
-
-    debug_called(4);
-
-    /* bogus disk? */
-    if (sc == NULL) {
-	bp->bio_flags |= BIO_ERROR;
-	bp->bio_error = EINVAL;
-	return;
-    }
-
-    /* do-nothing operation? */
-    if (bp->bio_bcount == 0) {
-	bp->bio_resid = bp->bio_bcount;
-	biodone(bp);
-	return;
-    }
-
-    /* perform accounting */
-    devstat_start_transaction(&sc->twed_stats);
-
-    /* pass the bio to the controller - it can work out who we are */
-    twe_submit_buf(sc->twed_controller, bp);
-    return;
-}
-
-/********************************************************************************
- * Handle completion of an I/O request.
- */
-void
-twed_intr(void *data)
-{
-    struct bio *bp = (struct bio *)data;
-    struct twed_softc	*sc = (struct twed_softc *)bp->bio_dev->si_drv1;
-
-    debug_called(4);
-
-    /* check for error from controller code */
-    if (bp->bio_flags & BIO_ERROR) {
-	bp->bio_error = EIO;
-    } else {
-	bp->bio_resid = 0;
-    }
-
-    devstat_end_transaction_bio(&sc->twed_stats, bp);
-    biodone(bp);
-}
-
-/********************************************************************************
- * Stub to provide device identification when probed.
- */
-static int
-twed_probe(device_t dev)
-{
-
-    debug_called(4);
-	
-    device_set_desc(dev, "3ware RAID unit");
-    return (0);
-}
-
-/********************************************************************************
- * Attach a unit to the controller.
- */
-static int
-twed_attach(device_t dev)
-{
-    struct twed_softc	*sc = (struct twed_softc *)device_get_softc(dev);
-    device_t		parent;
-    dev_t		dsk;
-    
-    debug_called(4);
-
-    /* initialise our softc */
-    parent = device_get_parent(dev);
-    sc->twed_controller = (struct twe_softc *)device_get_softc(parent);
-    sc->twed_drive = device_get_ivars(dev);
-    sc->twed_dev = dev;
-
-    /* report the drive */
-    device_printf(dev, "%uMB (%u sectors)\n",
-		  sc->twed_drive->td_size / ((1024 * 1024) / TWE_BLOCK_SIZE),
-		  sc->twed_drive->td_size);
-
-    devstat_add_entry(&sc->twed_stats, "twed", device_get_unit(dev), TWE_BLOCK_SIZE,
-		      DEVSTAT_NO_ORDERED_TAGS,
-		      DEVSTAT_TYPE_STORARRAY | DEVSTAT_TYPE_IF_OTHER, 
-		      DEVSTAT_PRIORITY_ARRAY);
-
-    /* attach a generic disk device to ourselves */
-    dsk = disk_create(device_get_unit(dev), &sc->twed_disk, 0, &twed_cdevsw, &tweddisk_cdevsw);
-    dsk->si_drv1 = sc;
-    sc->twed_dev_t = dsk;
-#ifdef FREEBSD_4
-    disks_registered++;
-#endif
-
-    /* set the maximum I/O size to the theoretical maximum allowed by the S/G list size */
-    dsk->si_iosize_max = (TWE_MAX_SGL_LENGTH - 1) * PAGE_SIZE;
-
-    return (0);
-}
-
-/********************************************************************************
- * Disconnect ourselves from the system.
- */
-static int
-twed_detach(device_t dev)
-{
-    struct twed_softc *sc = (struct twed_softc *)device_get_softc(dev);
-
-    debug_called(4);
-
-    if (sc->twed_flags & TWED_OPEN)
-	return(EBUSY);
-
-    devstat_remove_entry(&sc->twed_stats);
-#ifdef FREEBSD_4
-    if (--disks_registered == 0)
-	cdevsw_remove(&tweddisk_cdevsw);
-#else
-    disk_destroy(sc->twed_dev_t);
-#endif
-
-    return(0);
-}
-
diff --git a/sys/dev/twe/twe_freebsd.c b/sys/dev/twe/twe_freebsd.c
new file mode 100644
index 0000000..761117d
--- /dev/null
+++ b/sys/dev/twe/twe_freebsd.c
@@ -0,0 +1,971 @@
+/*-
+ * Copyright (c) 2000 Michael Smith
+ * Copyright (c) 2000 BSDi
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+/*
+ * FreeBSD-specific code.
+ */
+
+#include <dev/twe/twe_compat.h>
+#include <dev/twe/twereg.h>
+#include <dev/twe/tweio.h>
+#include <dev/twe/twevar.h>
+#include <dev/twe/twe_tables.h>
+
+#include <sys/devicestat.h>
+
+static devclass_t	twe_devclass;
+
+/********************************************************************************
+ ********************************************************************************
+                                                         Control device interface
+ ********************************************************************************
+ ********************************************************************************/
+
+static	d_open_t		twe_open;
+static	d_close_t		twe_close;
+static	d_ioctl_t		twe_ioctl_wrapper;
+
+#define TWE_CDEV_MAJOR  146
+
+static struct cdevsw twe_cdevsw = {
+    twe_open,
+    twe_close,
+    noread,
+    nowrite,
+    twe_ioctl_wrapper,
+    nopoll,
+    nommap,
+    nostrategy,
+    "twe",
+    TWE_CDEV_MAJOR,
+    nodump,
+    nopsize,
+    0,
+    -1
+};
+
+/********************************************************************************
+ * Accept an open operation on the control device.
+ */
+static int
+twe_open(dev_t dev, int flags, int fmt, struct proc *p)
+{
+    int			unit = minor(dev);
+    struct twe_softc	*sc = devclass_get_softc(twe_devclass, unit);
+
+    sc->twe_state |= TWE_STATE_OPEN;
+    return(0);
+}
+
+/********************************************************************************
+ * Accept the last close on the control device.
+ */
+static int
+twe_close(dev_t dev, int flags, int fmt, struct proc *p)
+{
+    int			unit = minor(dev);
+    struct twe_softc	*sc = devclass_get_softc(twe_devclass, unit);
+
+    sc->twe_state &= ~TWE_STATE_OPEN;
+    return (0);
+}
+
+/********************************************************************************
+ * Handle controller-specific control operations.
+ */
+static int
+twe_ioctl_wrapper(dev_t dev, u_long cmd, caddr_t addr, int32_t flag, struct proc *p)
+{
+    struct twe_softc		*sc = (struct twe_softc *)dev->si_drv1;
+    
+    return(twe_ioctl(sc, cmd, addr));
+}
+
+/********************************************************************************
+ ********************************************************************************
+                                                             PCI device interface
+ ********************************************************************************
+ ********************************************************************************/
+
+static int	twe_probe(device_t dev);
+static int	twe_attach(device_t dev);
+static void	twe_free(struct twe_softc *sc);
+static int	twe_detach(device_t dev);
+static int	twe_shutdown(device_t dev);
+static int	twe_suspend(device_t dev);
+static int	twe_resume(device_t dev);
+static void	twe_pci_intr(void *arg);
+static void	twe_intrhook(void *arg);
+
+static device_method_t twe_methods[] = {
+    /* Device interface */
+    DEVMETHOD(device_probe,	twe_probe),
+    DEVMETHOD(device_attach,	twe_attach),
+    DEVMETHOD(device_detach,	twe_detach),
+    DEVMETHOD(device_shutdown,	twe_shutdown),
+    DEVMETHOD(device_suspend,	twe_suspend),
+    DEVMETHOD(device_resume,	twe_resume),
+
+    DEVMETHOD(bus_print_child,	bus_generic_print_child),
+    DEVMETHOD(bus_driver_added,	bus_generic_driver_added),
+    { 0, 0 }
+};
+
+static driver_t twe_pci_driver = {
+	"twe",
+	twe_methods,
+	sizeof(struct twe_softc)
+};
+
+#ifdef TWE_OVERRIDE
+DRIVER_MODULE(Xtwe, pci, twe_pci_driver, twe_devclass, 0, 0);
+#else
+DRIVER_MODULE(twe, pci, twe_pci_driver, twe_devclass, 0, 0);
+#endif
+
+/********************************************************************************
+ * Match a 3ware Escalade ATA RAID controller.
+ */
+static int
+twe_probe(device_t dev)
+{
+
+    debug_called(4);
+
+    if ((pci_get_vendor(dev) == TWE_VENDOR_ID) &&
+	(pci_get_device(dev) == TWE_DEVICE_ID)) {
+	device_set_desc(dev, TWE_DEVICE_NAME);
+#ifdef TWE_OVERRIDE
+	return(0);
+#else
+	return(-10);
+#endif
+    }
+    return(ENXIO);
+}
+
+/********************************************************************************
+ * Allocate resources, initialise the controller.
+ */
+static int
+twe_attach(device_t dev)
+{
+    struct twe_softc	*sc;
+    int			rid, error;
+    u_int32_t		command;
+
+    debug_called(4);
+
+    /*
+     * Initialise the softc structure.
+     */
+    sc = device_get_softc(dev);
+    sc->twe_dev = dev;
+
+    /*
+     * Make sure we are going to be able to talk to this board.
+     */
+    command = pci_read_config(dev, PCIR_COMMAND, 2);
+    if ((command & PCIM_CMD_PORTEN) == 0) {
+	twe_printf(sc, "register window not available\n");
+	return(ENXIO);
+    }
+    /*
+     * Force the busmaster enable bit on, in case the BIOS forgot.
+     */
+    command |= PCIM_CMD_BUSMASTEREN;
+    pci_write_config(dev, PCIR_COMMAND, command, 2);
+
+    /*
+     * Allocate the PCI register window.
+     */
+    rid = TWE_IO_CONFIG_REG;
+    if ((sc->twe_io = bus_alloc_resource(dev, SYS_RES_IOPORT, &rid, 0, ~0, 1, RF_ACTIVE)) == NULL) {
+	twe_printf(sc, "can't allocate register window\n");
+	twe_free(sc);
+	return(ENXIO);
+    }
+    sc->twe_btag = rman_get_bustag(sc->twe_io);
+    sc->twe_bhandle = rman_get_bushandle(sc->twe_io);
+
+    /*
+     * Allocate the parent bus DMA tag appropriate for PCI.
+     */
+    if (bus_dma_tag_create(NULL, 				/* parent */
+			   1, 0, 				/* alignment, boundary */
+			   BUS_SPACE_MAXADDR_32BIT, 		/* lowaddr */
+			   BUS_SPACE_MAXADDR, 			/* highaddr */
+			   NULL, NULL, 				/* filter, filterarg */
+			   MAXBSIZE, TWE_MAX_SGL_LENGTH,	/* maxsize, nsegments */
+			   BUS_SPACE_MAXSIZE_32BIT,		/* maxsegsize */
+			   BUS_DMA_ALLOCNOW,			/* flags */
+			   &sc->twe_parent_dmat)) {
+	twe_printf(sc, "can't allocate parent DMA tag\n");
+	twe_free(sc);
+	return(ENOMEM);
+    }
+
+    /* 
+     * Allocate and connect our interrupt.
+     */
+    rid = 0;
+    if ((sc->twe_irq = bus_alloc_resource(sc->twe_dev, SYS_RES_IRQ, &rid, 0, ~0, 1, RF_SHAREABLE | RF_ACTIVE)) == NULL) {
+	twe_printf(sc, "can't allocate interrupt\n");
+	twe_free(sc);
+	return(ENXIO);
+    }
+    if (bus_setup_intr(sc->twe_dev, sc->twe_irq, INTR_TYPE_BIO,  twe_pci_intr, sc, &sc->twe_intr)) {
+	twe_printf(sc, "can't set up interrupt\n");
+	twe_free(sc);
+	return(ENXIO);
+    }
+
+    /*
+     * Create DMA tag for mapping objects into controller-addressable space.
+     */
+    if (bus_dma_tag_create(sc->twe_parent_dmat, 		/* parent */
+			   1, 0, 				/* alignment, boundary */
+			   BUS_SPACE_MAXADDR,		/* lowaddr */
+			   BUS_SPACE_MAXADDR, 		/* highaddr */
+			   NULL, NULL, 			/* filter, filterarg */
+			   MAXBSIZE, TWE_MAX_SGL_LENGTH,	/* maxsize, nsegments */
+			   BUS_SPACE_MAXSIZE_32BIT,		/* maxsegsize */
+			   0,				/* flags */
+			   &sc->twe_buffer_dmat)) {
+	twe_printf(sc, "can't allocate data buffer DMA tag\n");
+	twe_free(sc);
+	return(ENOMEM);
+    }
+
+    /*
+     * Initialise the controller and driver core.
+     */
+    if ((error = twe_setup(sc)))
+	return(error);
+
+    /*
+     * Print some information about the controller and configuration.
+     */
+    twe_describe_controller(sc);
+
+    /*
+     * Create the control device.
+     */
+    sc->twe_dev_t = make_dev(&twe_cdevsw, device_get_unit(sc->twe_dev), UID_ROOT, GID_OPERATOR,
+			     S_IRUSR | S_IWUSR, "twe%d", device_get_unit(sc->twe_dev));
+    sc->twe_dev_t->si_drv1 = sc;
+    /*
+     * Schedule ourselves to bring the controller up once interrupts are available.
+     * This isn't strictly necessary, since we disable interrupts while probing the
+     * controller, but it is more in keeping with common practice for other disk 
+     * devices.
+     */
+    sc->twe_ich.ich_func = twe_intrhook;
+    sc->twe_ich.ich_arg = sc;
+    if (config_intrhook_establish(&sc->twe_ich) != 0) {
+	twe_printf(sc, "can't establish configuration hook\n");
+	twe_free(sc);
+	return(ENXIO);
+    }
+
+    return(0);
+}
+
+/********************************************************************************
+ * Free all of the resources associated with (sc).
+ *
+ * Should not be called if the controller is active.
+ */
+static void
+twe_free(struct twe_softc *sc)
+{
+    struct twe_request	*tr;
+
+    debug_called(4);
+
+    /* throw away any command buffers */
+    while ((tr = twe_dequeue_free(sc)) != NULL)
+	twe_free_request(tr);
+
+    /* destroy the data-transfer DMA tag */
+    if (sc->twe_buffer_dmat)
+	bus_dma_tag_destroy(sc->twe_buffer_dmat);
+
+    /* disconnect the interrupt handler */
+    if (sc->twe_intr)
+	bus_teardown_intr(sc->twe_dev, sc->twe_irq, sc->twe_intr);
+    if (sc->twe_irq != NULL)
+	bus_release_resource(sc->twe_dev, SYS_RES_IRQ, 0, sc->twe_irq);
+
+    /* destroy the parent DMA tag */
+    if (sc->twe_parent_dmat)
+	bus_dma_tag_destroy(sc->twe_parent_dmat);
+
+    /* release the register window mapping */
+    if (sc->twe_io != NULL)
+	bus_release_resource(sc->twe_dev, SYS_RES_IOPORT, TWE_IO_CONFIG_REG, sc->twe_io);
+
+    /* destroy control device */
+    if (sc->twe_dev_t != (dev_t)NULL)
+	destroy_dev(sc->twe_dev_t);
+}
+
+/********************************************************************************
+ * Disconnect from the controller completely, in preparation for unload.
+ */
+static int
+twe_detach(device_t dev)
+{
+    struct twe_softc	*sc = device_get_softc(dev);
+    int			s, error;
+
+    debug_called(4);
+
+    error = EBUSY;
+    s = splbio();
+    if (sc->twe_state & TWE_STATE_OPEN)
+	goto out;
+
+    /*	
+     * Shut the controller down.
+     */
+    if ((error = twe_shutdown(dev)))
+	goto out;
+
+    twe_free(sc);
+
+    error = 0;
+ out:
+    splx(s);
+    return(error);
+}
+
+/********************************************************************************
+ * Bring the controller down to a dormant state and detach all child devices.
+ *
+ * Note that we can assume that the bioq on the controller is empty, as we won't
+ * allow shutdown if any device is open.
+ */
+static int
+twe_shutdown(device_t dev)
+{
+    struct twe_softc	*sc = device_get_softc(dev);
+    int			i, s, error;
+
+    debug_called(4);
+
+    s = splbio();
+    error = 0;
+
+    /* 
+     * Delete all our child devices.
+     */
+    for (i = 0; i < TWE_MAX_UNITS; i++) {
+	if (sc->twe_drive[i].td_disk != 0) {
+	    if ((error = device_delete_child(sc->twe_dev, sc->twe_drive[i].td_disk)) != 0)
+		goto out;
+	    sc->twe_drive[i].td_disk = 0;
+	}
+    }
+
+    /*
+     * Bring the controller down.
+     */
+    twe_deinit(sc);
+
+ out:
+    splx(s);
+    return(error);
+}
+
+/********************************************************************************
+ * Bring the controller to a quiescent state, ready for system suspend.
+ */
+static int
+twe_suspend(device_t dev)
+{
+    struct twe_softc	*sc = device_get_softc(dev);
+    int			s;
+
+    debug_called(4);
+
+    s = splbio();
+    sc->twe_state |= TWE_STATE_SUSPEND;
+    
+    twe_disable_interrupts(sc);
+    splx(s);
+
+    return(0);
+}
+
+/********************************************************************************
+ * Bring the controller back to a state ready for operation.
+ */
+static int
+twe_resume(device_t dev)
+{
+    struct twe_softc	*sc = device_get_softc(dev);
+
+    debug_called(4);
+
+    sc->twe_state &= ~TWE_STATE_SUSPEND;
+    twe_enable_interrupts(sc);
+
+    return(0);
+}
+
+/*******************************************************************************
+ * Take an interrupt, or be poked by other code to look for interrupt-worthy
+ * status.
+ */
+static void
+twe_pci_intr(void *arg)
+{
+    twe_intr((struct twe_softc *)arg);
+}
+
+/********************************************************************************
+ * Delayed-startup hook
+ */
+static void
+twe_intrhook(void *arg)
+{
+    struct twe_softc		*sc = (struct twe_softc *)arg;
+
+    /* pull ourselves off the intrhook chain */
+    config_intrhook_disestablish(&sc->twe_ich);
+
+    /* call core startup routine */
+    twe_init(sc);
+}
+
+/********************************************************************************
+ * Given a detected drive, attach it to the bio interface.
+ *
+ * This is called from twe_init.
+ */
+void
+twe_attach_drive(struct twe_softc *sc, struct twe_drive *dr)
+{
+    char	buf[80];
+    int		error;
+
+    dr->td_disk =  device_add_child(sc->twe_dev, NULL, -1);
+    if (dr->td_disk == NULL) {
+	twe_printf(sc, "device_add_child failed\n");
+	return;
+    }
+    device_set_ivars(dr->td_disk, dr);
+
+    /* 
+     * XXX It would make sense to test the online/initialising bits, but they seem to be
+     * always set...
+     */
+    sprintf(buf, "%s, %s", twe_describe_code(twe_table_unittype, dr->td_type),
+	    twe_describe_code(twe_table_unitstate, dr->td_state & TWE_PARAM_UNITSTATUS_MASK));
+    device_set_desc_copy(dr->td_disk, buf);
+
+    if ((error = bus_generic_attach(sc->twe_dev)) != 0)
+	twe_printf(sc, "bus_generic_attach returned %d\n", error);
+}
+
+/********************************************************************************
+ ********************************************************************************
+                                                                      Disk device
+ ********************************************************************************
+ ********************************************************************************/
+
+/*
+ * Disk device softc
+ */
+struct twed_softc 
+{
+    device_t		twed_dev;
+    dev_t		twed_dev_t;
+    struct twe_softc	*twed_controller;	/* parent device softc */
+    struct twe_drive	*twed_drive;		/* drive data in parent softc */
+    struct disk		twed_disk;		/* generic disk handle */
+    struct devstat	twed_stats;		/* accounting */
+    struct disklabel	twed_label;		/* synthetic label */
+    int			twed_flags;
+#define TWED_OPEN	(1<<0)			/* drive is open (can't shut down) */
+};
+
+/*
+ * Disk device bus interface
+ */
+static int twed_probe(device_t dev);
+static int twed_attach(device_t dev);
+static int twed_detach(device_t dev);
+
+static device_method_t twed_methods[] = {
+    DEVMETHOD(device_probe,	twed_probe),
+    DEVMETHOD(device_attach,	twed_attach),
+    DEVMETHOD(device_detach,	twed_detach),
+    { 0, 0 }
+};
+
+static driver_t twed_driver = {
+    "twed",
+    twed_methods,
+    sizeof(struct twed_softc)
+};
+
+static devclass_t	twed_devclass;
+#ifdef TWE_OVERRIDE
+DRIVER_MODULE(Xtwed, Xtwe, twed_driver, twed_devclass, 0, 0);
+#else
+DRIVER_MODULE(twed, twe, twed_driver, twed_devclass, 0, 0);
+#endif
+
+/*
+ * Disk device control interface.
+ */
+static	d_open_t	twed_open;
+static	d_close_t	twed_close;
+static	d_strategy_t	twed_strategy;
+
+#define TWED_CDEV_MAJOR	147
+
+static struct cdevsw twed_cdevsw = {
+    twed_open,
+    twed_close,
+    physread,
+    physwrite,
+    noioctl,
+    nopoll,
+    nommap,
+    twed_strategy,
+    "twed",
+    TWED_CDEV_MAJOR,
+    nodump,
+    nopsize,
+    D_DISK,
+    -1
+};
+
+static struct cdevsw	tweddisk_cdevsw;
+#ifdef FREEBSD_4
+static int		disks_registered = 0;
+#endif
+
+/********************************************************************************
+ * Handle open from generic layer.
+ *
+ * Note that this is typically only called by the diskslice code, and not
+ * for opens on subdevices (eg. slices, partitions).
+ */
+static int
+twed_open(dev_t dev, int flags, int fmt, struct proc *p)
+{
+    struct twed_softc	*sc = (struct twed_softc *)dev->si_drv1;
+    struct disklabel	*label;
+
+    debug_called(4);
+	
+    if (sc == NULL)
+	return (ENXIO);
+
+    /* check that the controller is up and running */
+    if (sc->twed_controller->twe_state & TWE_STATE_SHUTDOWN)
+	return(ENXIO);
+
+    /* build synthetic label */
+    label = &sc->twed_disk.d_label;
+    bzero(label, sizeof(*label));
+    label->d_type = DTYPE_ESDI;
+    label->d_secsize    = TWE_BLOCK_SIZE;
+    label->d_nsectors   = sc->twed_drive->td_sectors;
+    label->d_ntracks    = sc->twed_drive->td_heads;
+    label->d_ncylinders = sc->twed_drive->td_cylinders;
+    label->d_secpercyl  = sc->twed_drive->td_sectors * sc->twed_drive->td_heads;
+    label->d_secperunit = sc->twed_drive->td_size;
+
+    sc->twed_flags |= TWED_OPEN;
+    return (0);
+}
+
+/********************************************************************************
+ * Handle last close of the disk device.
+ */
+static int
+twed_close(dev_t dev, int flags, int fmt, struct proc *p)
+{
+    struct twed_softc	*sc = (struct twed_softc *)dev->si_drv1;
+
+    debug_called(4);
+	
+    if (sc == NULL)
+	return (ENXIO);
+
+    sc->twed_flags &= ~TWED_OPEN;
+    return (0);
+}
+
+/********************************************************************************
+ * Handle an I/O request.
+ */
+static void
+twed_strategy(twe_bio *bp)
+{
+    struct twed_softc	*sc = (struct twed_softc *)TWE_BIO_SOFTC(bp);
+
+    debug_called(4);
+
+    /* bogus disk? */
+    if (sc == NULL) {
+	TWE_BIO_SET_ERROR(bp, EINVAL);
+	return;
+    }
+
+    /* do-nothing operation? */
+    if (TWE_BIO_LENGTH(bp) == 0) {
+	TWE_BIO_RESID(bp) = 0;
+	TWE_BIO_DONE(bp);
+	return;
+    }
+
+    /* perform accounting */
+    TWE_BIO_STATS_START(bp);
+
+    /* pass the bio to the controller - it can work out who we are */
+    twe_submit_bio(sc->twed_controller, bp);
+    return;
+}
+
+/********************************************************************************
+ * Handle completion of an I/O request.
+ */
+void
+twed_intr(twe_bio *bp)
+{
+    debug_called(4);
+
+    /* if no error, transfer completed */
+    if (!TWE_BIO_HAS_ERROR(bp))
+	TWE_BIO_RESID(bp) = 0;
+
+    TWE_BIO_STATS_END(bp);
+    TWE_BIO_DONE(bp);
+}
+
+/********************************************************************************
+ * Default probe stub.
+ */
+static int
+twed_probe(device_t dev)
+{
+    return (0);
+}
+
+/********************************************************************************
+ * Attach a unit to the controller.
+ */
+static int
+twed_attach(device_t dev)
+{
+    struct twed_softc	*sc;
+    device_t		parent;
+    dev_t		dsk;
+    
+    debug_called(4);
+
+    /* initialise our softc */
+    sc = device_get_softc(dev);
+    parent = device_get_parent(dev);
+    sc->twed_controller = (struct twe_softc *)device_get_softc(parent);
+    sc->twed_drive = device_get_ivars(dev);
+    sc->twed_dev = dev;
+
+    /* report the drive */
+    twed_printf(sc, "%uMB (%u sectors)\n",
+		sc->twed_drive->td_size / ((1024 * 1024) / TWE_BLOCK_SIZE),
+		sc->twed_drive->td_size);
+    
+    devstat_add_entry(&sc->twed_stats, "twed", device_get_unit(dev), TWE_BLOCK_SIZE,
+		      DEVSTAT_NO_ORDERED_TAGS,
+		      DEVSTAT_TYPE_STORARRAY | DEVSTAT_TYPE_IF_OTHER, 
+		      DEVSTAT_PRIORITY_ARRAY);
+
+    /* attach a generic disk device to ourselves */
+    dsk = disk_create(device_get_unit(dev), &sc->twed_disk, 0, &twed_cdevsw, &tweddisk_cdevsw);
+    dsk->si_drv1 = sc;
+    dsk->si_drv2 = &sc->twed_drive->td_unit;
+    sc->twed_dev_t = dsk;
+#ifdef FREEBSD_4
+    disks_registered++;
+#endif
+
+    /* set the maximum I/O size to the theoretical maximum allowed by the S/G list size */
+    dsk->si_iosize_max = (TWE_MAX_SGL_LENGTH - 1) * PAGE_SIZE;
+
+    return (0);
+}
+
+/********************************************************************************
+ * Disconnect ourselves from the system.
+ */
+static int
+twed_detach(device_t dev)
+{
+    struct twed_softc *sc = (struct twed_softc *)device_get_softc(dev);
+
+    debug_called(4);
+
+    if (sc->twed_flags & TWED_OPEN)
+	return(EBUSY);
+
+    devstat_remove_entry(&sc->twed_stats);
+#ifdef FREEBSD_4
+    if (--disks_registered == 0)
+	cdevsw_remove(&tweddisk_cdevsw);
+#else
+    disk_destroy(sc->twed_dev_t);
+#endif
+
+    return(0);
+}
+
+/********************************************************************************
+ ********************************************************************************
+                                                                             Misc
+ ********************************************************************************
+ ********************************************************************************/
+
+static void	twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
+static void	twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error);
+
+/********************************************************************************
+ * Allocate a command buffer
+ */
+MALLOC_DEFINE(TWE_MALLOC_CLASS, "twe commands", "twe commands");
+
+struct twe_request *
+twe_allocate_request(struct twe_softc *sc)
+{
+    struct twe_request	*tr;
+
+    if ((tr = malloc(sizeof(struct twe_request), TWE_MALLOC_CLASS, M_NOWAIT)) == NULL)
+	return(NULL);
+    bzero(tr, sizeof(*tr));
+    tr->tr_sc = sc;
+    if (bus_dmamap_create(sc->twe_buffer_dmat, 0, &tr->tr_cmdmap)) {
+	twe_free_request(tr);
+	return(NULL);
+    }
+    if (bus_dmamap_create(sc->twe_buffer_dmat, 0, &tr->tr_dmamap)) {
+	bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_cmdmap);
+	twe_free_request(tr);
+	return(NULL);
+    }    
+    return(tr);
+}
+
+/********************************************************************************
+ * Permanently discard a command buffer.
+ */
+void
+twe_free_request(struct twe_request *tr) 
+{
+    struct twe_softc	*sc = tr->tr_sc;
+    
+    debug_called(4);
+
+    bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_cmdmap);
+    bus_dmamap_destroy(sc->twe_buffer_dmat, tr->tr_dmamap);
+    free(tr, TWE_MALLOC_CLASS);
+}
+
+/********************************************************************************
+ * Map/unmap (tr)'s command and data in the controller's addressable space.
+ *
+ * These routines ensure that the data which the controller is going to try to
+ * access is actually visible to the controller, in a machine-independant 
+ * fasion.  Due to a hardware limitation, I/O buffers must be 512-byte aligned
+ * and we take care of that here as well.
+ */
+static void
+twe_setup_data_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
+{
+    struct twe_request	*tr = (struct twe_request *)arg;
+    TWE_Command		*cmd = &tr->tr_command;
+    int			i;
+
+    debug_called(4);
+
+    /* save base of first segment in command (applicable if there only one segment) */
+    tr->tr_dataphys = segs[0].ds_addr;
+
+    /* correct command size for s/g list size */
+    tr->tr_command.generic.size += 2 * nsegments;
+
+    /*
+     * Due to the fact that parameter and I/O commands have the scatter/gather list in
+     * different places, we need to determine which sort of command this actually is
+     * before we can populate it correctly.
+     */
+    switch(cmd->generic.opcode) {
+    case TWE_OP_GET_PARAM:
+    case TWE_OP_SET_PARAM:
+	cmd->generic.sgl_offset = 2;
+	for (i = 0; i < nsegments; i++) {
+	    cmd->param.sgl[i].address = segs[i].ds_addr;
+	    cmd->param.sgl[i].length = segs[i].ds_len;
+	}
+	for (; i < TWE_MAX_SGL_LENGTH; i++) {		/* XXX necessary? */
+	    cmd->param.sgl[i].address = 0;
+	    cmd->param.sgl[i].length = 0;
+	}
+	break;
+    case TWE_OP_READ:
+    case TWE_OP_WRITE:
+	cmd->generic.sgl_offset = 3;
+	for (i = 0; i < nsegments; i++) {
+	    cmd->io.sgl[i].address = segs[i].ds_addr;
+	    cmd->io.sgl[i].length = segs[i].ds_len;
+	}
+	for (; i < TWE_MAX_SGL_LENGTH; i++) {		/* XXX necessary? */
+	    cmd->io.sgl[i].address = 0;
+	    cmd->io.sgl[i].length = 0;
+	}
+	break;
+    default:
+	/* no s/g list, nothing to do */
+    }
+}
+
+static void
+twe_setup_request_dmamap(void *arg, bus_dma_segment_t *segs, int nsegments, int error)
+{
+    struct twe_request	*tr = (struct twe_request *)arg;
+
+    debug_called(4);
+
+    /* command can't cross a page boundary */
+    tr->tr_cmdphys = segs[0].ds_addr;
+}
+
+void
+twe_map_request(struct twe_request *tr)
+{
+    struct twe_softc	*sc = tr->tr_sc;
+
+    debug_called(4);
+
+
+    /*
+     * Map the command into bus space.
+     */
+    bus_dmamap_load(sc->twe_buffer_dmat, tr->tr_cmdmap, &tr->tr_command, sizeof(tr->tr_command), 
+		    twe_setup_request_dmamap, tr, 0);
+    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_cmdmap, BUS_DMASYNC_PREWRITE);
+
+    /*
+     * If the command involves data, map that too.
+     */
+    if (tr->tr_data != NULL) {
+
+	/* 
+	 * Data must be 64-byte aligned; allocate a fixup buffer if it's not.
+	 */
+	if (((vm_offset_t)tr->tr_data % TWE_ALIGNMENT) != 0) {
+	    tr->tr_realdata = tr->tr_data;				/* save pointer to 'real' data */
+	    tr->tr_flags |= TWE_CMD_ALIGNBUF;
+	    tr->tr_data = malloc(tr->tr_length, TWE_MALLOC_CLASS, M_NOWAIT);	/* XXX check result here */
+	}
+	
+	/*
+	 * Map the data buffer into bus space and build the s/g list.
+	 */
+	bus_dmamap_load(sc->twe_buffer_dmat, tr->tr_dmamap, tr->tr_data, tr->tr_length, 
+			twe_setup_data_dmamap, tr, 0);
+	if (tr->tr_flags & TWE_CMD_DATAIN)
+	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_PREREAD);
+	if (tr->tr_flags & TWE_CMD_DATAOUT) {
+	    /* if we're using an alignment buffer, and we're writing data, copy the real data out */
+	    if (tr->tr_flags & TWE_CMD_ALIGNBUF)
+		bcopy(tr->tr_realdata, tr->tr_data, tr->tr_length);
+	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_PREWRITE);
+	}
+    }
+}
+
+void
+twe_unmap_request(struct twe_request *tr)
+{
+    struct twe_softc	*sc = tr->tr_sc;
+
+    debug_called(4);
+
+    /*
+     * Unmap the command from bus space.
+     */
+    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_cmdmap, BUS_DMASYNC_POSTWRITE);
+    bus_dmamap_unload(sc->twe_buffer_dmat, tr->tr_cmdmap); 
+
+    /*
+     * If the command involved data, unmap that too.
+     */
+    if (tr->tr_data != NULL) {
+	
+	if (tr->tr_flags & TWE_CMD_DATAIN) {
+	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_POSTREAD);
+	    /* if we're using an alignment buffer, and we're reading data, copy the real data in */
+	    if (tr->tr_flags & TWE_CMD_ALIGNBUF)
+		bcopy(tr->tr_data, tr->tr_realdata, tr->tr_length);
+	}
+	if (tr->tr_flags & TWE_CMD_DATAOUT)
+	    bus_dmamap_sync(sc->twe_buffer_dmat, tr->tr_dmamap, BUS_DMASYNC_POSTWRITE);
+
+	bus_dmamap_unload(sc->twe_buffer_dmat, tr->tr_dmamap); 
+    }
+
+    /* free alignment buffer if it was used */
+    if (tr->tr_flags & TWE_CMD_ALIGNBUF) {
+	free(tr->tr_data, TWE_MALLOC_CLASS);
+	tr->tr_data = tr->tr_realdata;		/* restore 'real' data pointer */
+    }
+}
+
+#ifdef TWE_DEBUG
+/********************************************************************************
+ * Print current controller status, call from DDB.
+ */
+void
+twe_report(void)
+{
+    struct twe_softc	*sc;
+    int			i, s;
+
+    s = splbio();
+    for (i = 0; (sc = devclass_get_softc(twe_devclass, i)) != NULL; i++)
+	twe_print_controller(sc);
+    splx(s);
+}
+#endif
diff --git a/sys/dev/twe/twe_tables.h b/sys/dev/twe/twe_tables.h
new file mode 100644
index 0000000..aa1ab02
--- /dev/null
+++ b/sys/dev/twe/twe_tables.h
@@ -0,0 +1,161 @@
+/*-
+ * Copyright (c) 2000 Michael Smith
+ * Copyright (c) 2000 BSDi
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	$FreeBSD$
+ */
+
+/*
+ * Lookup table for code-to-text translations.
+ */
+struct twe_code_lookup {
+    char	*string;
+    u_int32_t	code;
+};
+
+extern char	*twe_describe_code(struct twe_code_lookup *table, u_int32_t code);
+
+#ifndef TWE_DEFINE_TABLES
+extern struct twe_code_lookup twe_table_status[];
+extern struct twe_code_lookup twe_table_unitstate[];
+extern struct twe_code_lookup twe_table_unittype[];
+extern struct twe_code_lookup twe_table_aen[];
+extern struct twe_code_lookup twe_table_opcode[];
+#else /* TWE_DEFINE_TABLES */
+
+/********************************************************************************
+ * Look up a text description of a numeric code and return a pointer to same.
+ */
+char *
+twe_describe_code(struct twe_code_lookup *table, u_int32_t code)
+{
+    int		i;
+
+    for (i = 0; table[i].string != NULL; i++)
+	if (table[i].code == code)
+	    return(table[i].string);
+    return(table[i+1].string);
+}
+
+
+struct twe_code_lookup twe_table_status[] = {
+    /* success */
+    {"successful completion",					0x00},
+    /* info */
+    {"command in progress",					0x42},
+    {"retrying interface CRC error from UDMA command",		0x6c},
+    /* warning */
+    {"redundant/inconsequential request ignored",		0x81},
+    {"failed to write zeroes to LBA 0",				0x8e},
+    {"failed to profile TwinStor zones",			0x8f},
+    /* fatal */
+    {"aborted due to system command or reconfiguration",	0xc1},
+    {"aborted",							0xc4},
+    {"access error",						0xc5},
+    {"access violation",					0xc6},
+    {"device failure",						0xc7},	/* high byte may be port number */
+    {"controller error",					0xc8},
+    {"timed out",						0xc9},
+    {"invalid unit number",					0xcb},
+    {"unit not available",					0xcf},
+    {"undefined opcode",					0xd2},
+    {"request incompatible with unit",				0xdb},
+    {"invalid request",						0xdc},
+    {"firmware error, reset requested",				0xff},
+    {NULL,	0},
+    {"unknown status",	0}
+};
+
+struct twe_code_lookup twe_table_unitstate[] = {
+    {"Normal",		TWE_PARAM_UNITSTATUS_Normal},
+    {"Initialising",	TWE_PARAM_UNITSTATUS_Initialising},
+    {"Degraded",	TWE_PARAM_UNITSTATUS_Degraded},
+    {"Rebuilding",	TWE_PARAM_UNITSTATUS_Rebuilding},
+    {"Verifying",	TWE_PARAM_UNITSTATUS_Verifying},
+    {"Corrupt",		TWE_PARAM_UNITSTATUS_Corrupt},
+    {"Missing",		TWE_PARAM_UNITSTATUS_Missing},
+    {NULL, 0},
+    {"unknown state",	0}
+};
+
+struct twe_code_lookup twe_table_unittype[] = {
+    {"RAID0",		TWE_UD_CONFIG_RAID0},
+    {"RAID1",		TWE_UD_CONFIG_RAID1},
+    {"TwinStor",	TWE_UD_CONFIG_TwinStor},
+    {"RAID5",		TWE_UD_CONFIG_RAID5},
+    {"RAID10",		TWE_UD_CONFIG_RAID10},
+    {"CBOD",		TWE_UD_CONFIG_CBOD},
+    {"SPARE",		TWE_UD_CONFIG_SPARE},
+    {"SUBUNIT",		TWE_UD_CONFIG_SUBUNIT},
+    {"JBOD",		TWE_UD_CONFIG_JBOD},
+    {NULL, 0},
+    {"unknown type",	0}
+};
+
+struct twe_code_lookup twe_table_aen[] = {
+    {"q queue empty",			0x00},
+    {"q soft reset",			0x01},
+    {"c degraded mirror",		0x02},
+    {"p controller error",		0x03},
+    {"c rebuild fail",			0x04},
+    {"c rebuild done",			0x05},
+    {"c incomplete unit",		0x06},
+    {"c initialisation done",		0x07},
+    {"c unclean shutdown detected",	0x08},
+    {"c drive timeout",			0x09},
+    {"c drive error",			0x0a},
+    {"c rebuild started",		0x0b},
+    {"p aen queue full",		0xff},
+    {NULL, 0},
+    {"x unknown AEN",		0}
+};
+
+struct twe_code_lookup twe_table_opcode[] = {
+    {"NOP",			0x00},
+    {"INIT_CONNECTION",		0x01},
+    {"READ",			0x02},
+    {"WRITE",			0x03},
+    {"READVERIFY",		0x04},
+    {"VERIFY",			0x05},
+    {"ZEROUNIT",		0x08},
+    {"REPLACEUNIT",		0x09},
+    {"HOTSWAP",			0x0a},
+    {"SETATAFEATURE",		0x0c},
+    {"FLUSH",			0x0e},
+    {"ABORT",			0x0f},
+    {"CHECKSTATUS",		0x10},
+    {"GET_PARAM",		0x12},
+    {"SET_PARAM",		0x13},
+    {"CREATEUNIT",		0x14},
+    {"DELETEUNIT",		0x15},
+    {"REBUILDUNIT",		0x17},
+    {"SECTOR_INFO",		0x1a},
+    {"AEN_LISTEN",		0x1c},
+    {"CMD_PACKET",		0x1d},
+    {NULL, 0},
+    {"unknown opcode",		0}
+};    
+    
+#endif
diff --git a/sys/dev/twe/tweio.h b/sys/dev/twe/tweio.h
new file mode 100644
index 0000000..eaf0d7b
--- /dev/null
+++ b/sys/dev/twe/tweio.h
@@ -0,0 +1,92 @@
+/*-
+ * Copyright (c) 2000 Michael Smith
+ * Copyright (c) 2000 BSDi
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ *	$FreeBSD$
+ */
+
+
+/*
+ * User-space command
+ *
+ * Note that the command's scatter/gather list will be computed by the
+ * driver, and cannot be filled in by the consumer.
+ */
+struct twe_usercommand {
+    TWE_Command	tu_command;	/* command ready for the controller */
+    void	*tu_data;	/* pointer to data in userspace */
+    size_t	tu_size;	/* userspace data length */
+};
+
+#define TWEIO_COMMAND		_IOWR('T', 100, struct twe_usercommand)
+
+/*
+ * Command queue statistics
+ */
+#define TWEQ_FREE	0
+#define TWEQ_BIO	1
+#define TWEQ_READY	2
+#define TWEQ_BUSY	3
+#define TWEQ_COMPLETE	4
+#define TWEQ_COUNT	5	/* total number of queues */
+
+struct twe_qstat {
+    u_int32_t	q_length;
+    u_int32_t	q_max;
+};
+
+/*
+ * Statistics request
+ */
+union twe_statrequest {
+    u_int32_t		ts_item;
+    struct twe_qstat	ts_qstat;
+};
+
+#define TWEIO_STATS		_IOWR('T', 101, union twe_statrequest)
+
+/*
+ * AEN listen
+ */
+#define TWEIO_AEN_POLL		_IOR('T', 102, int)
+#define TWEIO_AEN_WAIT		_IOR('T', 103, int)
+
+/*
+ * Controller parameter access
+ */
+struct twe_paramcommand {
+    u_int16_t	tp_table_id;
+    u_int8_t	tp_param_id;
+    void	*tp_data;
+    u_int8_t	tp_size;
+};
+
+#define TWEIO_SET_PARAM		_IOW ('T', 104, struct twe_paramcommand)
+#define TWEIO_GET_PARAM		_IOW ('T', 105, struct twe_paramcommand)
+
+/*
+ * Request a controller soft-reset
+ */
+#define TWEIO_RESET		_IO  ('T', 106)
diff --git a/sys/dev/twe/twereg.h b/sys/dev/twe/twereg.h
index db9a4f8..5330015 100644
--- a/sys/dev/twe/twereg.h
+++ b/sys/dev/twe/twereg.h
@@ -76,6 +76,9 @@
 #define TWE_STATUS_EXPECTED_BITS		0x00002000
 #define TWE_STATUS_UNEXPECTED_BITS		0x00F80000
 
+/* XXX this is a little harsh, but necessary to chase down firmware problems */
+#define TWE_STATUS_PANIC_BITS			(TWE_STATUS_MICROCONTROLLER_ERROR)
+
 /* for use with the %b printf format */
 #define TWE_STATUS_BITS_DESCRIPTION \
 	"\20\15CQEMPTY\16UCREADY\17RQEMPTY\20CQFULL\21RINTR\22CINTR\23AINTR\24HINTR\25PCIABRT\26MCERR\27QERR\30PCIPERR\n"
@@ -98,47 +101,44 @@
 #define TWE_DEVICE_ID			0x1000
 
 /* command packet opcodes */
-#define TWE_OP_NOP			0x0
-#define TWE_OP_INIT_CONNECTION		0x1
-#define TWE_OP_READ			0x2
-#define TWE_OP_WRITE			0x3
-#define TWE_OP_VERIFY			0x4
+#define TWE_OP_NOP			0x00
+#define TWE_OP_INIT_CONNECTION		0x01
+#define TWE_OP_READ			0x02
+#define TWE_OP_WRITE			0x03
+#define TWE_OP_READVERIFY		0x04
+#define TWE_OP_VERIFY			0x05
+#define TWE_OP_ZEROUNIT			0x08
+#define TWE_OP_REPLACEUNIT		0x09
+#define TWE_OP_HOTSWAP			0x0a
+#define TWE_OP_SETATAFEATURE		0x0c
+#define TWE_OP_FLUSH			0x0e
+#define TWE_OP_ABORT			0x0f
+#define TWE_OP_CHECKSTATUS		0x10
 #define TWE_OP_GET_PARAM		0x12
 #define TWE_OP_SET_PARAM		0x13
+#define TWE_OP_CREATEUNIT		0x14
+#define TWE_OP_DELETEUNIT		0x15
+#define TWE_OP_REBUILDUNIT		0x17
 #define TWE_OP_SECTOR_INFO		0x1a
 #define TWE_OP_AEN_LISTEN		0x1c
-
-/* asynchronous event notification (AEN) codes */
-#define TWE_AEN_QUEUE_EMPTY		0x0000
-#define TWE_AEN_SOFT_RESET		0x0001
-#define TWE_AEN_DEGRADED_MIRROR		0x0002
-#define TWE_AEN_CONTROLLER_ERROR	0x0003
-#define TWE_AEN_REBUILD_FAIL		0x0004
-#define TWE_AEN_REBUILD_DONE		0x0005
-#define TWE_AEN_QUEUE_FULL		0x00ff
-#define TWE_AEN_TABLE_UNDEFINED		0x15
+#define TWE_OP_CMD_PACKET		0x1d
 
 /* misc defines */
 #define TWE_ALIGNMENT			0x200
 #define TWE_MAX_UNITS			16
 #define TWE_COMMAND_ALIGNMENT_MASK	0x1ff
-#define TWE_INIT_MESSAGE_CREDITS	0x100
+#define TWE_INIT_MESSAGE_CREDITS	0xff	/* older firmware has issues with 256 commands */
+#define TWE_SHUTDOWN_MESSAGE_CREDITS	0x001
 #define TWE_INIT_COMMAND_PACKET_SIZE	0x3
 #define TWE_MAX_SGL_LENGTH		62
-#define TWE_Q_LENGTH			50
+#define TWE_Q_LENGTH			TWE_INIT_MESSAGE_CREDITS
 #define TWE_Q_START			0
 #define TWE_MAX_RESET_TRIES		3
-#define TWE_UNIT_INFORMATION_TABLE_BASE	0x300
 #define TWE_BLOCK_SIZE			0x200	/* 512-byte blocks */
 #define TWE_SECTOR_SIZE			0x200	/* generic I/O bufffer */
 #define TWE_IOCTL			0x80
 #define TWE_MAX_AEN_TRIES		100
-
-/* wrappers for bus-space actions */
-#define TWE_CONTROL(sc, val)		bus_space_write_4(sc->twe_btag, sc->twe_bhandle, 0x0, (u_int32_t)val)
-#define TWE_STATUS(sc)			(u_int32_t)bus_space_read_4(sc->twe_btag, sc->twe_bhandle, 0x4)
-#define TWE_COMMAND_QUEUE(sc, val)	bus_space_write_4(sc->twe_btag, sc->twe_bhandle, 0x8, (u_int32_t)val)
-#define TWE_RESPONSE_QUEUE(sc)		(TWE_Response_Queue)bus_space_read_4(sc->twe_btag, sc->twe_bhandle, 0xc)
+#define TWE_UNIT_ONLINE			1
 
 /* scatter/gather list entry */
 typedef struct
@@ -147,7 +147,115 @@ typedef struct
     u_int32_t	length;
 } TWE_SG_Entry __attribute__ ((packed));
 
-/* command packet - must be TWE_ALIGNMENT aligned */
+typedef struct {
+    u_int8_t	opcode:5;		/* TWE_OP_INITCONNECTION */
+    u_int8_t	res1:3;		
+    u_int8_t	size;
+    u_int8_t	request_id;
+    u_int8_t	res2:4;
+    u_int8_t	host_id:4;
+    u_int8_t	status;
+    u_int8_t	flags;
+    u_int16_t	message_credits;
+    u_int32_t	response_queue_pointer;
+} TWE_Command_INITCONNECTION __attribute__ ((packed));
+
+typedef struct
+{
+    u_int8_t	opcode:5;		/* TWE_OP_READ/TWE_OP_WRITE */
+    u_int8_t	res1:3;
+    u_int8_t	size;
+    u_int8_t	request_id;
+    u_int8_t	unit:4;
+    u_int8_t	host_id:4;
+    u_int8_t	status;
+    u_int8_t	flags;
+    u_int16_t	block_count;
+    u_int32_t	lba;
+    TWE_SG_Entry sgl[TWE_MAX_SGL_LENGTH];
+} TWE_Command_IO __attribute__ ((packed));
+
+typedef struct
+{
+    u_int8_t	opcode:5;		/* TWE_OP_HOTSWAP */
+    u_int8_t	res1:3;
+    u_int8_t	size;
+    u_int8_t	request_id;
+    u_int8_t	unit:4;
+    u_int8_t	host_id:4;
+    u_int8_t	status;
+    u_int8_t	flags;
+    u_int8_t	action;
+#define TWE_OP_HOTSWAP_REMOVE		0x00	/* remove assumed-degraded unit */
+#define TWE_OP_HOTSWAP_ADD_CBOD		0x01	/* add CBOD to empty port */
+#define TWE_OP_HOTSWAP_ADD_SPARE	0x02	/* add spare to empty port */
+    u_int8_t	aport;
+} TWE_Command_HOTSWAP __attribute__ ((packed));
+
+typedef struct
+{
+    u_int8_t	opcode:5;		/* TWE_OP_SETATAFEATURE */
+    u_int8_t	res1:3;
+    u_int8_t	size;
+    u_int8_t	request_id;
+    u_int8_t	unit:4;
+    u_int8_t	host_id:4;
+    u_int8_t	status;
+    u_int8_t	flags;
+    u_int8_t	feature;
+#define TWE_OP_SETATAFEATURE_WCE	0x02
+#define TWE_OP_SETATAFEATURE_DIS_WCE	0x82
+    u_int8_t	feature_mode;
+    u_int16_t	all_units;
+    u_int16_t	persistence;
+} TWE_Command_SETATAFEATURE __attribute__ ((packed));
+
+typedef struct
+{
+    u_int8_t	opcode:5;		/* TWE_OP_CHECKSTATUS */
+    u_int8_t	res1:3;
+    u_int8_t	size;
+    u_int8_t	request_id;
+    u_int8_t	unit:4;
+    u_int8_t	res2:4;
+    u_int8_t	status;
+    u_int8_t	flags;
+    u_int16_t	target_status;		/* set low byte to target request's ID */
+} TWE_Command_CHECKSTATUS __attribute__ ((packed));
+
+typedef struct
+{
+    u_int8_t	opcode:5;		/* TWE_OP_GETPARAM, TWE_OP_SETPARAM */
+    u_int8_t	res1:3;
+    u_int8_t	size;
+    u_int8_t	request_id;
+    u_int8_t	unit:4;
+    u_int8_t	host_id:4;
+    u_int8_t	status;
+    u_int8_t	flags;
+    u_int16_t	param_count;
+    TWE_SG_Entry sgl[TWE_MAX_SGL_LENGTH];
+} TWE_Command_PARAM __attribute__ ((packed));
+
+typedef struct
+{
+    u_int8_t	opcode:5;		/* TWE_OP_REBUILDUNIT */
+    u_int8_t	res1:3;
+    u_int8_t	size;
+    u_int8_t	request_id;
+    u_int8_t	src_unit:4;
+    u_int8_t	host_id:4;
+    u_int8_t	status;
+    u_int8_t	flags;
+    u_int8_t	action:7;
+#define TWE_OP_REBUILDUNIT_NOP		0
+#define TWE_OP_REBUILDUNIT_STOP		2	/* stop all rebuilds */
+#define TWE_OP_REBUILDUNIT_START	4	/* start rebuild with lowest unit */
+#define TWE_OP_REBUILDUNIT_STARTUNIT	5	/* rebuild src_unit (not supported) */
+    u_int8_t	cs:1;				/* request state change on src_unit */
+    u_int8_t	logical_subunit;		/* for RAID10 rebuild of logical subunit */
+} TWE_Command_REBUILDUNIT __attribute__ ((packed));
+
 typedef struct
 {
     u_int8_t	opcode:5;
@@ -158,29 +266,26 @@ typedef struct
     u_int8_t	host_id:4;
     u_int8_t	status;
     u_int8_t	flags;
+#define TWE_FLAGS_SUCCESS	0x00
+#define TWE_FLAGS_INFORMATIONAL	0x01
+#define TWE_FLAGS_WARNING	0x02
+#define TWE_FLAGS_FATAL		0x03
+#define TWE_FLAGS_PERCENTAGE	(1<<8)	/* bits 0-6 indicate completion percentage */
     u_int16_t	count;			/* block count, parameter count, message credits */
-    union {
-	struct {
-	    u_int32_t	lba;
-	    TWE_SG_Entry sgl[TWE_MAX_SGL_LENGTH];
-	} io __attribute__ ((packed));
-	struct {
-	    TWE_SG_Entry sgl[TWE_MAX_SGL_LENGTH];
-	} param;
-	struct {
-	    u_int32_t	response_queue_pointer;
-	} init_connection;
-    } args;
-} TWE_Command __attribute__ ((packed));
-
-/* argument to TWE_OP_GET/SET_PARAM */
-typedef struct 
+} TWE_Command_Generic __attribute__ ((packed));
+
+/* command packet - must be TWE_ALIGNMENT aligned */
+typedef union
 {
-    u_int16_t	table_id;
-    u_int8_t	parameter_id;
-    u_int8_t	parameter_size_bytes;
-    u_int8_t	data[1];
-} TWE_Param __attribute__ ((packed));
+    TWE_Command_INITCONNECTION	initconnection;
+    TWE_Command_IO		io;
+    TWE_Command_PARAM		param;
+    TWE_Command_CHECKSTATUS	checkstatus;
+    TWE_Command_REBUILDUNIT	rebuildunit;
+    TWE_Command_SETATAFEATURE	setatafeature;
+    TWE_Command_Generic		generic;
+    u_int8_t			pad[512];
+} TWE_Command;
 
 /* response queue entry */
 typedef union
@@ -194,33 +299,157 @@ typedef union
     u_int32_t	value;
 } TWE_Response_Queue;
 
-#if 0 /* no idea what these will be useful for yet */
+/*
+ * From 3ware's documentation:
+ *   All parameters maintained by the controller are grouped into related tables.
+ *   Tables are are accessed indirectly via get and set parameter commands.
+ *   To access a specific parameter in a table, the table ID and parameter index
+ *   are used to uniquely identify a parameter.  Table 0xffff is the directory
+ *   table and provides a list of the table IDs and sizes of all other tables.
+ *   Index zero in each table specifies the entire table, and index one specifies
+ *   the size of the table.  An entire table can be read or set by using index zero.
+ */
+
+#define TWE_PARAM_PARAM_ALL	0
+#define TWE_PARAM_PARAM_SIZE	1
+
+#define TWE_PARAM_DIRECTORY	0xffff			/* size is 4 * number of tables */
+#define TWE_PARAM_DIRECTORY_TABLES		2	/* 16 bits * number of tables */
+#define TWE_PARAM_DIRECTORY_SIZES		3	/* 16 bits * number of tables */
+
+#define TWE_PARAM_DRIVESUMMARY	0x0002
+#define TWE_PARAM_DRIVESUMMARY_Num		2	/* number of physical drives [2] */
+#define TWE_PARAM_DRIVESUMMARY_Status		3	/* array giving drive status per aport */
+#define TWE_PARAM_DRIVESTATUS_Missing	0x00
+#define TWE_PARAM_DRIVESTATUS_NotSupp	0xfe
+#define TWE_PARAM_DRIVESTATUS_Present	0xff
+
+#define TWE_PARAM_UNITSUMMARY	0x0003
+#define TWE_PARAM_UNITSUMMARY_Num		2	/* number of logical units [2] */
+#define TWE_PARAM_UNITSUMMARY_Status		3	/* array giving unit status [16] */
+#define TWE_PARAM_UNITSTATUS_Online		(1<<0)
+#define TWE_PARAM_UNITSTATUS_Complete		(1<<1)
+#define TWE_PARAM_UNITSTATUS_MASK		0xfc
+#define TWE_PARAM_UNITSTATUS_Normal		0xfc
+#define TWE_PARAM_UNITSTATUS_Initialising	0xf4	/* cannot be incomplete */
+#define TWE_PARAM_UNITSTATUS_Degraded		0xec
+#define TWE_PARAM_UNITSTATUS_Rebuilding		0xdc	/* cannot be incomplete */
+#define TWE_PARAM_UNITSTATUS_Verifying		0xcc	/* cannot be incomplete */
+#define TWE_PARAM_UNITSTATUS_Corrupt		0xbc	/* cannot be complete */
+#define TWE_PARAM_UNITSTATUS_Missing		0x00	/* cannot be complete or online */
+
+#define TWE_PARAM_DRIVEINFO	0x0200			/* add drive number 0x00-0x0f XXX docco confused 0x0100 vs 0x0200 */
+#define TWE_PARAM_DRIVEINFO_Size		2	/* size in blocks [4] */
+#define TWE_PARAM_DRIVEINFO_Model		3	/* drive model string [40] */
+#define TWE_PARAM_DRIVEINFO_Serial		4	/* drive serial number [20] */
+#define TWE_PARAM_DRIVEINFO_PhysCylNum		5	/* physical geometry [2] */
+#define TWE_PARAM_DRIVEINFO_PhysHeadNum		6	/* [2] */
+#define TWE_PARAM_DRIVEINFO_PhysSectorNym	7	/* [2] */
+#define TWE_PARAM_DRIVEINFO_LogCylNum		8	/* logical geometry [2] */
+#define TWE_PARAM_DRIVEINFO_LogHeadNum		9	/* [2] */
+#define TWE_PARAM_DRIVEINFO_LogSectorNum	10	/* [2] */
+#define TWE_PARAM_DRIVEINFO_UnitNum		11	/* unit number this drive is associated with or 0xff [1] */
+#define TWE_PARAM_DRIVEINFO_DriveFlags		12	/* N/A [1] */
+
+#define TWE_PARAM_APORTTIMEOUT	0x02c0			/* add (aport_number * 3) to parameter index */
+#define TWE_PARAM_APORTTIMEOUT_READ		2	/* read timeouts last 24hrs [2] */
+#define TWE_PARAM_APORTTIMEOUT_WRITE		3	/* write timeouts last 24hrs [2] */
+#define TWE_PARAM_APORTTIMEOUT_DEGRADE		4	/* degrade threshold [2] */
+
+#define TWE_PARAM_UNITINFO	0x0300			/* add unit number 0x00-0x0f */
+#define TWE_PARAM_UNITINFO_Number		2	/* unit number [1] */
+#define TWE_PARAM_UNITINFO_Status		3	/* unit status [1] */
+#define TWE_PARAM_UNITINFO_Capacity		4	/* unit capacity in blocks [4] */
+#define TWE_PARAM_UNITINFO_DescriptorSize	5	/* unit descriptor size + 3 bytes [2] */
+#define TWE_PARAM_UNITINFO_Descriptor		6	/* unit descriptor, TWE_UnitDescriptor or TWE_Array_Descriptor */
+#define TWE_PARAM_UNITINFO_Flags		7	/* unit flags [1] */
+#define TWE_PARAM_UNITFLAGS_WCE			(1<<0)
+
+#define TWE_PARAM_AEN		0x0401
+#define TWE_PARAM_AEN_UnitCode			2	/* (unit number << 8) | AEN code [2] */
+#define TWE_AEN_QUEUE_EMPTY		0x00
+#define TWE_AEN_SOFT_RESET		0x01
+#define TWE_AEN_DEGRADED_MIRROR		0x02	/* reports unit */
+#define TWE_AEN_CONTROLLER_ERROR	0x03
+#define TWE_AEN_REBUILD_FAIL		0x04	/* reports unit */
+#define TWE_AEN_REBUILD_DONE		0x05	/* reports unit */
+#define TWE_AEN_INCOMP_UNIT		0x06	/* reports unit */
+#define TWE_AEN_INIT_DONE		0x07	/* reports unit */
+#define TWE_AEN_UNCLEAN_SHUTDOWN	0x08	/* reports unit */
+#define TWE_AEN_APORT_TIMEOUT		0x09	/* reports unit, rate limited to 1 per 2^16 errors */
+#define TWE_AEN_DRIVE_ERROR		0x0a	/* reports unit */
+#define TWE_AEN_REBUILD_STARTED		0x0b	/* reports unit */
+#define TWE_AEN_QUEUE_FULL		0xff
+#define TWE_AEN_TABLE_UNDEFINED		0x15
+#define TWE_AEN_CODE(x)			((x) & 0xff)
+#define TWE_AEN_UNIT(x)			((x) >> 8)
+
+#define TWE_PARAM_VERSION	0x0402
+#define TWE_PARAM_VERSION_Mon			2	/* monitor version [16] */
+#define TWE_PARAM_VERSION_FW			3	/* firmware version [16] */
+#define TWE_PARAM_VERSION_BIOS			4	/* BIOSs version [16] */
+#define TWE_PARAM_VERSION_PCB			5	/* PCB version [8] */
+#define TWE_PARAM_VERSION_ATA			6	/* A-chip version [8] */
+#define TWE_PARAM_VERSION_PCI			7	/* P-chip version [8] */
+#define TWE_PARAM_VERSION_CtrlModel		8	/* N/A */
+#define TWE_PARAM_VERSION_CtrlSerial		9	/* N/A */
+#define TWE_PARAM_VERSION_SBufSize		10	/* N/A */
+#define TWE_PARAM_VERSION_CompCode		11	/* compatibility code [4] */
+
+#define TWE_PARAM_CONTROLLER	0x0403
+#define TWE_PARAM_CONTROLLER_DCBSectors		2	/* # sectors reserved for DCB per drive [2] */
+#define TWE_PARAM_CONTROLLER_PortCount		3	/* number of drive ports [1] */
+
+#define TWE_PARAM_FEATURES	0x404
+#define TWE_PARAM_FEATURES_DriverShutdown	2	/* set to 1 if driver supports shutdown notification [1] */
+
 typedef struct
 {
-    int32_t	buffer;
-    u_int8_t	opcode;
-    u_int16_t	table_id;
-    u_int8_t	parameter_id;
-    u_int8_t	parameter_size_bytes;
-    u_int8_t	data[1];
-} TWE_Ioctl __attribute__ ((packed));
+    u_int8_t		num_subunits;	/* must be zero */
+    u_int8_t		configuration;
+#define TWE_UD_CONFIG_CBOD	0x0c	/* JBOD with DCB, used for mirrors */
+#define TWE_UD_CONFIG_SPARE	0x0d	/* same as CBOD, but firmware will use as spare */
+#define TWE_UD_CONFIG_SUBUNIT	0x0e	/* drive is a subunit in an array */
+#define TWE_UD_CONFIG_JBOD	0x0f	/* plain drive */
+    u_int8_t		phys_drv_num;	/* may be 0xff if port can't be determined at runtime */
+    u_int8_t		log_drv_num;	/* must be zero for configuration == 0x0f */
+    u_int32_t		start_lba;
+    u_int32_t		block_count;	/* actual drive size if configuration == 0x0f, otherwise less DCB size */
+} TWE_Unit_Descriptor __attribute__ ((packed));
 
 typedef struct
 {
-    u_int32_t	base_addr;
-    u_int32_t	control_reg_addr;
-    u_int32_t	status_reg_addr;
-    u_int32_t	command_que_addr;
-    u_int32_t	response_que_addr;
-} TWE_Registers __attribute__ ((packed));
+    u_int8_t		flag;			/* must be 0xff */
+    u_int8_t		res1;
+    u_int8_t		mirunit_status[4];	/* bitmap of functional subunits in each mirror */
+    u_int8_t		res2[6];
+} TWE_Mirror_Descriptor __attribute__ ((packed));
 
 typedef struct
 {
-    char	*buffer;
-    int32_t	length;
-    int32_t	offset;
-    int32_t	position;
-} TWE_Info __attribute__ ((packed));
-#endif
-
+    u_int8_t		num_subunits;	/* number of subunits, or number of mirror units in RAID10 */
+    u_int8_t		configuration;
+#define TWE_UD_CONFIG_RAID0	0x00
+#define TWE_UD_CONFIG_RAID1	0x01
+#define TWE_UD_CONFIG_TwinStor	0x02
+#define TWE_UD_CONFIG_RAID5	0x05
+#define TWE_UD_CONFIG_RAID10	0x06
+    u_int8_t		stripe_size;
+#define TWE_UD_STRIPE_4k	0x03
+#define TWE_UD_STRIPE_8k	0x04
+#define TWE_UD_STRIPE_16k	0x05
+#define TWE_UD_STRIPE_32k	0x06
+#define TWE_UD_STRIPE_64k	0x07
+    u_int8_t		log_drv_status;	/* bitmap of functional subunits, or mirror units in RAID10 */
+    u_int32_t		start_lba;
+    u_int32_t		block_count;	/* actual drive size if configuration == 0x0f, otherwise less DCB size */
+    TWE_Unit_Descriptor	subunit[0];	/* subunit descriptors, in RAID10 mode is [mirunit][subunit] */
+} TWE_Array_Descriptor __attribute__ ((packed));
 
+typedef struct
+{
+    u_int16_t	table_id;
+    u_int8_t	parameter_id;
+    u_int8_t	parameter_size_bytes;
+    u_int8_t	data[0];
+} TWE_Param __attribute__ ((packed));
diff --git a/sys/dev/twe/twevar.h b/sys/dev/twe/twevar.h
index 673a278..ea12062 100644
--- a/sys/dev/twe/twevar.h
+++ b/sys/dev/twe/twevar.h
@@ -28,8 +28,14 @@
  */
 
 #ifdef TWE_DEBUG
-#define debug(level, fmt, args...)	do { if (level <= TWE_DEBUG) printf("%s: " fmt "\n", __FUNCTION__ , ##args); } while(0)
-#define debug_called(level)		do { if (level <= TWE_DEBUG) printf(__FUNCTION__ ": called\n"); } while(0)
+#define debug(level, fmt, args...)							\
+	do {										\
+	    if (level <= TWE_DEBUG) printf("%s: " fmt "\n", __FUNCTION__ , ##args);	\
+	} while(0)
+#define debug_called(level)						\
+	do {								\
+	    if (level <= TWE_DEBUG) printf(__FUNCTION__ ": called\n");	\
+	} while(0)
 #else
 #define debug(level, fmt, args...)
 #define debug_called(level)
@@ -47,16 +53,9 @@ struct twe_drive
     int			td_sectors;
     int			td_unit;
 
-    /* unit state */
-    int			td_state;
-#define TWE_DRIVE_READY		0
-#define TWE_DRIVE_DEGRADED	1
-#define TWE_DRIVE_OFFLINE	2
-    int			td_raidlevel;
-#define TWE_DRIVE_RAID0		0
-#define TWE_DRIVE_RAID1		1
-
-#define TWE_DRIVE_UNKNOWN	0xff
+    /* unit state and type */
+    u_int8_t		td_state;
+    u_int8_t		td_type;
 
     /* handle for attached driver */
     device_t		td_disk;
@@ -73,15 +72,11 @@ struct twe_request
 {
     /* controller command */
     TWE_Command			tr_command;	/* command as submitted to controller */
-    bus_dmamap_t		tr_cmdmap;	/* DMA map for command */
-    u_int32_t			tr_cmdphys;	/* address of command in controller space */
 
     /* command payload */
     void			*tr_data;	/* data buffer */
     void			*tr_realdata;	/* copy of real data buffer pointer for alignment fixup */
     size_t			tr_length;
-    bus_dmamap_t		tr_dmamap;	/* DMA map for data */
-    u_int32_t			tr_dataphys;	/* data buffer base address in controller space */
 
     TAILQ_ENTRY(twe_request)	tr_link;	/* list linkage */
     struct twe_softc		*tr_sc;		/* controller that owns us */
@@ -94,8 +89,11 @@ struct twe_request
 #define TWE_CMD_DATAIN		(1<<0)
 #define TWE_CMD_DATAOUT		(1<<1)
 #define TWE_CMD_ALIGNBUF	(1<<2)	/* data in bio is misaligned, have to copy to/from private buffer */
+#define TWE_CMD_SLEEPER		(1<<3)	/* owner is sleeping on this command */
     void			(* tr_complete)(struct twe_request *tr);	/* completion handler */
     void			*tr_private;	/* submitter-private data or wait channel */
+
+    TWE_PLATFORM_REQUEST	/* platform-specific request elements */
 };
 
 /*
@@ -103,28 +101,19 @@ struct twe_request
  */
 struct twe_softc 
 {
-    /* bus connections */
-    device_t		twe_dev;
-    dev_t		twe_dev_t;
-    struct resource	*twe_io;		/* register interface window */
-    bus_space_handle_t	twe_bhandle;		/* bus space handle */
-    bus_space_tag_t	twe_btag;		/* bus space tag */
-    bus_dma_tag_t	twe_parent_dmat;	/* parent DMA tag */
-    bus_dma_tag_t	twe_buffer_dmat;	/* data buffer DMA tag */
-    struct resource	*twe_irq;		/* interrupt */
-    void		*twe_intr;		/* interrupt handle */
-
     /* controller queues and arrays */
-    TAILQ_HEAD(, twe_request)	twe_freecmds;		/* command structures available for reuse */
-    TAILQ_HEAD(, twe_request)	twe_work;		/* active commands (busy or waiting for completion) */
-    struct twe_request	*twe_cmdlookup[TWE_Q_LENGTH];	/* busy commands indexed by request ID */
-    int			twe_busycmds;			/* count of busy commands */
+    TAILQ_HEAD(, twe_request)	twe_free;			/* command structures available for reuse */
+    twe_bioq 			twe_bioq;			/* outstanding I/O operations */
+    TAILQ_HEAD(, twe_request)	twe_ready;			/* requests ready for the controller */
+    TAILQ_HEAD(, twe_request)	twe_busy;			/* requests busy in the controller */
+    TAILQ_HEAD(, twe_request)	twe_complete;			/* active commands (busy or waiting for completion) */
+    struct twe_request		*twe_lookup[TWE_Q_LENGTH];	/* commands indexed by request ID */
     struct twe_drive	twe_drive[TWE_MAX_UNITS];		/* attached drives */
-    struct bio_queue_head twe_bioq;			/* outstanding I/O operations */
-    struct twe_request	*twe_deferred;			/* request that the controller wouldn't take */
 
+    /* asynchronous event handling */
     u_int16_t		twe_aen_queue[TWE_Q_LENGTH];	/* AENs queued for userland tool(s) */
     int			twe_aen_head, twe_aen_tail;	/* ringbuffer pointers for AEN queue */
+    int			twe_wait_aen;    		/* wait-for-aen notification */
 
     /* controller status */
     int			twe_state;
@@ -132,32 +121,154 @@ struct twe_softc
 #define TWE_STATE_SHUTDOWN	(1<<1)	/* controller is shut down */
 #define TWE_STATE_OPEN		(1<<2)	/* control device is open */
 #define TWE_STATE_SUSPEND	(1<<3)	/* controller is suspended */
+    int			twe_host_id;
+#ifdef TWE_PERFORMANCE_MONITOR
+    struct twe_qstat	twe_qstat[TWEQ_COUNT];	/* queue statistics */
+#endif
 
-    /* delayed configuration hook */
-    struct intr_config_hook	twe_ich;
-
-    /* wait-for-aen notification */
-    int			twe_wait_aen;
+    TWE_PLATFORM_SOFTC		/* platform-specific softc elements */
 };
 
 /*
- * Virtual disk driver.
+ * Interface betwen driver core and platform-dependant code.
  */
-struct twed_softc 
-{
-    device_t		twed_dev;
-    dev_t		twed_dev_t;
-    struct twe_softc	*twed_controller;	/* parent device softc */
-    struct twe_drive	*twed_drive;		/* drive data in parent softc */
-    struct disk		twed_disk;		/* generic disk handle */
-    struct devstat	twed_stats;		/* accounting */
-    struct disklabel	twed_label;		/* synthetic label */
-    int			twed_flags;
-#define TWED_OPEN	(1<<0)			/* drive is open (can't shut down) */
-};
+extern int	twe_setup(struct twe_softc *sc);		/* do early driver/controller setup */
+extern void	twe_init(struct twe_softc *sc);			/* init controller */
+extern void	twe_deinit(struct twe_softc *sc);		/* stop controller */
+extern void	twe_intr(struct twe_softc *sc);			/* hardware interrupt signalled */
+extern int	twe_submit_bio(struct twe_softc *sc, 
+				       twe_bio *bp);		/* pass bio to core */
+extern int	twe_ioctl(struct twe_softc *sc, int cmd,
+				  void *addr);			/* handle user request */
+extern void	twe_describe_controller(struct twe_softc *sc);	/* print controller info */
+extern void	twe_print_controller(struct twe_softc *sc);
+extern void	twe_enable_interrupts(struct twe_softc *sc);	/* enable controller interrupts */
+extern void	twe_disable_interrupts(struct twe_softc *sc);	/* disable controller interrupts */
+
+extern void	twe_attach_drive(struct twe_softc *sc,
+					 struct twe_drive *dr); /* attach drive when found in twe_init */
+extern void	twed_intr(twe_bio *bp);				/* return bio from core */
+extern struct twe_request *twe_allocate_request(struct twe_softc *sc);	/* allocate request structure */
+extern void	twe_free_request(struct twe_request *tr);	/* free request structure */
+extern void	twe_map_request(struct twe_request *tr);	/* make request visible to controller, do s/g */
+extern void	twe_unmap_request(struct twe_request *tr);	/* cleanup after transfer, unmap */
+
+/********************************************************************************
+ * Queue primitives
+ */
+#ifdef TWE_PERFORMANCE_MONITOR
+# define TWEQ_ADD(sc, qname)					\
+	do {							\
+	    struct twe_qstat *qs = &(sc)->twe_qstat[qname];	\
+								\
+	    qs->q_length++;					\
+	    if (qs->q_length > qs->q_max)			\
+		qs->q_max = qs->q_length;			\
+	} while(0)
+
+# define TWEQ_REMOVE(sc, qname)    (sc)->twe_qstat[qname].q_length--
+# define TWEQ_INIT(sc, qname)			\
+	do {					\
+	    sc->twe_qstat[qname].q_length = 0;	\
+	    sc->twe_qstat[qname].q_max = 0;	\
+	} while(0)
+#else
+# define TWEQ_ADD(sc, qname)
+# define TWEQ_REMOVE(sc, qname)
+# define TWEQ_INIT(sc, qname)
+#endif
+
+
+#define TWEQ_REQUEST_QUEUE(name, index)					\
+static __inline void							\
+twe_initq_ ## name (struct twe_softc *sc)				\
+{									\
+    TAILQ_INIT(&sc->twe_ ## name);					\
+    TWEQ_INIT(sc, index);						\
+}									\
+static __inline void							\
+twe_enqueue_ ## name (struct twe_request *tr)				\
+{									\
+    int		s;							\
+									\
+    s = splbio();							\
+    TAILQ_INSERT_TAIL(&tr->tr_sc->twe_ ## name, tr, tr_link);		\
+    TWEQ_ADD(tr->tr_sc, index);						\
+    splx(s);								\
+}									\
+static __inline void							\
+twe_requeue_ ## name (struct twe_request *tr)				\
+{									\
+    int		s;							\
+									\
+    s = splbio();							\
+    TAILQ_INSERT_HEAD(&tr->tr_sc->twe_ ## name, tr, tr_link);		\
+    TWEQ_ADD(tr->tr_sc, index);						\
+    splx(s);								\
+}									\
+static __inline struct twe_request *					\
+twe_dequeue_ ## name (struct twe_softc *sc)				\
+{									\
+    struct twe_request	*tr;						\
+    int			s;						\
+									\
+    s = splbio();							\
+    if ((tr = TAILQ_FIRST(&sc->twe_ ## name)) != NULL) {		\
+	TAILQ_REMOVE(&sc->twe_ ## name, tr, tr_link);			\
+	TWEQ_REMOVE(sc, index);						\
+    }									\
+    splx(s);								\
+    return(tr);								\
+}									\
+static __inline void							\
+twe_remove_ ## name (struct twe_request *tr)				\
+{									\
+    int			s;						\
+									\
+    s = splbio();							\
+    TAILQ_REMOVE(&tr->tr_sc->twe_ ## name, tr, tr_link);		\
+    TWEQ_REMOVE(tr->tr_sc, index);					\
+    splx(s);								\
+}
+
+TWEQ_REQUEST_QUEUE(free, TWEQ_FREE)
+TWEQ_REQUEST_QUEUE(ready, TWEQ_READY)
+TWEQ_REQUEST_QUEUE(busy, TWEQ_BUSY)
+TWEQ_REQUEST_QUEUE(complete, TWEQ_COMPLETE)
+
 
 /*
- * Interface betwen driver core and disk driver (should be using a bus?)
+ * outstanding bio queue
  */
-extern int	twe_submit_buf(struct twe_softc *sc, struct bio *bp);
-extern void	twed_intr(void *data);
+static __inline void
+twe_initq_bio(struct twe_softc *sc)
+{
+    TWE_BIO_QINIT(sc->twe_bioq);
+    TWEQ_INIT(sc, TWEQ_BIO);
+}
+
+static __inline void
+twe_enqueue_bio(struct twe_softc *sc, struct bio *bp)
+{
+    int		s;
+
+    s = splbio();
+    TWE_BIO_QINSERT(sc->twe_bioq, bp);
+    TWEQ_ADD(sc, TWEQ_BIO);
+    splx(s);
+}
+
+static __inline struct bio *
+twe_dequeue_bio(struct twe_softc *sc)
+{
+    int		s;
+    struct bio	*bp;
+
+    s = splbio();
+    if ((bp = TWE_BIO_QFIRST(sc->twe_bioq)) != NULL) {
+	TWE_BIO_QREMOVE(sc->twe_bioq, bp);
+	TWEQ_REMOVE(sc, TWEQ_BIO);
+    }
+    splx(s);
+    return(bp);
+}
diff --git a/sys/modules/twe/Makefile b/sys/modules/twe/Makefile
index f12f4e9..2d0639b 100644
--- a/sys/modules/twe/Makefile
+++ b/sys/modules/twe/Makefile
@@ -1,7 +1,32 @@
 # $FreeBSD$
 
+#
+# The 3ware controller offers a choice of options related to handling of
+# mirror consistency in shutdown situations.
+#
+# If TWE_SHUTDOWN_NOTIFICATION is defined, the driver will inform the 
+# controller on clean shutdown.  On reboot after an unclean shutdown, the 
+# controller will perform a complete mirror rebuild.  This is the most
+# "safe" configuration, but it imposes a severe performance penalty in the
+# event of an unexpected reboot.
+#
+# In the alternate configuration, the controller assumes that all writes to
+# a mirror complete successfully.  In the case of an unclean shutdown,
+# mirror corruption may occur if the controller is reset or power is lost
+# during a disk write.  This configuration is more suitable if the system
+# has reliable power, but may be unstable for other reasons.
+#
+CFLAGS+=-DTWE_SHUTDOWN_NOTIFICATION
+
+#
+# The module can be built to override a compiled-in driver
+#
+#CFLAGS+=-DTWE_OVERRIDE
+
 .PATH:	${.CURDIR}/../../dev/twe
 KMOD=	twe
-SRCS=	bus_if.h device_if.h pci_if.h twe.c twe_disk.c
+SRCS=	bus_if.h device_if.h pci_if.h twe.c twe_freebsd.c
+
+#CFLAGS+=-DTWE_DEBUG=0
 
 .include <bsd.kmod.mk>