- Add a command validator for use in debugging.

- Fix the locking protocol to eliminate races between normal I/O and AENs.
- Various small improvements and usability tweaks.

Sponsored by: IronPort
Portions Submitted by: Doug Ambrisko

5 files changed, 425 insertions, 53 deletions

diff --git a/sys/dev/mfi/mfi.c b/sys/dev/mfi/mfi.c
index f611370..823f2cc 100644
--- a/sys/dev/mfi/mfi.c
+++ b/sys/dev/mfi/mfi.c
@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/ioccom.h>
 #include <sys/uio.h>
 #include <sys/proc.h>
+#include <sys/signalvar.h>
 
 #include <machine/bus.h>
 #include <machine/resource.h>
@@ -342,8 +343,12 @@ mfi_attach(struct mfi_softc *sc)
 	if ((error = mfi_get_controller_info(sc)) != 0)
 		return (error);
 
-	if ((error = mfi_aen_setup(sc, 0), 0) != 0)
+	mtx_lock(&sc->mfi_io_lock);
+	if ((error = mfi_aen_setup(sc, 0), 0) != 0) {
+		mtx_unlock(&sc->mfi_io_lock);
 		return (error);
+	}
+	mtx_unlock(&sc->mfi_io_lock);
 
 	/*
 	 * Set up the interrupt handler.  XXX This should happen in
@@ -437,6 +442,7 @@ mfi_alloc_commands(struct mfi_softc *sc)
 static void
 mfi_release_command(struct mfi_command *cm)
 {
+	struct mfi_frame_header *hdr;
 	uint32_t *hdr_data;
 
 	/*
@@ -447,12 +453,18 @@ mfi_release_command(struct mfi_command *cm)
 	hdr_data[0] = 0;
 	hdr_data[1] = 0;
 
+	hdr = &cm->cm_frame->header;
+	if (hdr->sg_count) {
+		cm->cm_sg->sg32[0].len = 0;
+		cm->cm_sg->sg32[0].addr = 0;
+	}
 	cm->cm_extra_frames = 0;
 	cm->cm_flags = 0;
 	cm->cm_complete = NULL;
 	cm->cm_private = NULL;
 	cm->cm_sg = 0;
 	cm->cm_total_frame_size = 0;
+
 	mfi_enqueue_free(cm);
 }
 
@@ -511,6 +523,7 @@ mfi_comms_init(struct mfi_softc *sc)
 	struct mfi_init_qinfo *qinfo;
 	int error;
 
+	mtx_lock(&sc->mfi_io_lock);
 	if ((cm = mfi_dequeue_free(sc)) == NULL)
 		return (EBUSY);
 
@@ -536,9 +549,11 @@ mfi_comms_init(struct mfi_softc *sc)
 
 	if ((error = mfi_polled_command(sc, cm)) != 0) {
 		device_printf(sc->mfi_dev, "failed to send init command\n");
+		mtx_unlock(&sc->mfi_io_lock);
 		return (error);
 	}
 	mfi_release_command(cm);
+	mtx_unlock(&sc->mfi_io_lock);
 
 	return (0);
 }
@@ -562,6 +577,7 @@ mfi_get_controller_info(struct mfi_softc *sc)
 		device_printf(sc->mfi_dev, "Controller info buffer map failed\n");
 		free(ci, M_MFIBUF);
 		mfi_release_command(cm);
+		mtx_unlock(&sc->mfi_io_lock);
 		return (error);
 	}
 
@@ -597,7 +613,6 @@ mfi_get_log_state(struct mfi_softc *sc, struct mfi_evt_log_state **log_state)
 	struct mfi_command *cm = NULL;
 	int error;
 
-	mtx_lock(&sc->mfi_io_lock);
 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_EVENT_GETINFO,
 	    (void **)log_state, sizeof(**log_state));
 	if (error)
@@ -621,7 +636,6 @@ mfi_get_log_state(struct mfi_softc *sc, struct mfi_evt_log_state **log_state)
 out:
 	if (cm)
 		mfi_release_command(cm);
-	mtx_unlock(&sc->mfi_io_lock);
 
 	return (error);
 }
@@ -669,6 +683,8 @@ mfi_polled_command(struct mfi_softc *sc, struct mfi_command *cm)
 	struct mfi_frame_header *hdr;
 	int tm = MFI_POLL_TIMEOUT_SECS * 1000000;
 
+	mtx_assert(&sc->mfi_io_lock, MA_OWNED);
+
 	hdr = &cm->cm_frame->header;
 	hdr->cmd_status = 0xff;
 	hdr->flags |= MFI_FRAME_DONT_POST_IN_REPLY_QUEUE;
@@ -772,7 +788,9 @@ mfi_startup(void *arg)
 	config_intrhook_disestablish(&sc->mfi_ich);
 
 	mfi_enable_intr(sc);
+	mtx_lock(&sc->mfi_io_lock);
 	mfi_ldprobe(sc);
+	mtx_unlock(&sc->mfi_io_lock);
 }
 
 static void
@@ -787,6 +805,7 @@ mfi_intr(void *arg)
 	status = MFI_READ4(sc, MFI_OSTS);
 	if ((status & MFI_OSTS_INTR_VALID) == 0)
 		return;
+
 	MFI_WRITE4(sc, MFI_OSTS, status);
 
 	pi = sc->mfi_comms->hw_pi;
@@ -801,10 +820,15 @@ mfi_intr(void *arg)
 			ci = 0;
 		}
 	}
-	mtx_unlock(&sc->mfi_io_lock);
 
 	sc->mfi_comms->hw_ci = ci;
 
+	/* Give defered I/O a chance to run */
+	if (sc->mfi_flags & MFI_FLAGS_QFRZN)
+		sc->mfi_flags &= ~MFI_FLAGS_QFRZN;
+	mfi_startio(sc);
+	mtx_unlock(&sc->mfi_io_lock);
+
 	return;
 }
 
@@ -817,9 +841,10 @@ mfi_shutdown(struct mfi_softc *sc)
 
 	mtx_lock(&sc->mfi_io_lock);
 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_SHUTDOWN, NULL, 0);
-	mtx_unlock(&sc->mfi_io_lock);
-	if (error)
+	if (error) {
+		mtx_unlock(&sc->mfi_io_lock);
 		return (error);
+	}
 
 	if (sc->mfi_aen_cm != NULL)
 		mfi_abort(sc, sc->mfi_aen_cm);
@@ -832,6 +857,7 @@ mfi_shutdown(struct mfi_softc *sc)
 	}
 
 	mfi_release_command(cm);
+	mtx_unlock(&sc->mfi_io_lock);
 	return (error);
 }
 
@@ -850,7 +876,8 @@ mfi_ldprobe(struct mfi_softc *sc)
 	struct mfi_ld_list *list = NULL;
 	int error, i;
 
-	mtx_lock(&sc->mfi_io_lock);
+	mtx_assert(&sc->mfi_io_lock, MA_OWNED);
+
 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_LD_GET_LIST,
 	    (void **)&list, sizeof(*list));
 	if (error)
@@ -870,13 +897,13 @@ mfi_ldprobe(struct mfi_softc *sc)
 	}
 
 	for (i = 0; i < list->ld_count; i++)
-		mfi_add_ld(sc, list->ld_list[i].ld.target_id);
+		mfi_add_ld(sc, list->ld_list[i].ld.v.target_id);
 out:
 	if (list)
 		free(list, M_MFIBUF);
 	if (cm)
 		mfi_release_command(cm);
-	mtx_unlock(&sc->mfi_io_lock);
+
 	return;
 }
 
@@ -1175,7 +1202,7 @@ mfi_aen_register(struct mfi_softc *sc, int seq, int locale)
 	struct mfi_dcmd_frame *dcmd;
 	union mfi_evt current_aen, prior_aen;
 	struct mfi_evt_detail *ed = NULL;
-	int error;
+	int error = 0;
 
 	current_aen.word = locale;
 	if (sc->mfi_aen_cm != NULL) {
@@ -1195,12 +1222,11 @@ mfi_aen_register(struct mfi_softc *sc, int seq, int locale)
 		}
 	}
 
-	mtx_lock(&sc->mfi_io_lock);
 	error = mfi_dcmd_command(sc, &cm, MFI_DCMD_CTRL_EVENT_WAIT,
 	    (void **)&ed, sizeof(*ed));
-	mtx_unlock(&sc->mfi_io_lock);
-	if (error)
-		return (error);
+	if (error) {
+		goto out;
+	}
 
 	dcmd = &cm->cm_frame->dcmd;
 	((uint32_t *)&dcmd->mbox)[0] = seq;
@@ -1213,7 +1239,8 @@ mfi_aen_register(struct mfi_softc *sc, int seq, int locale)
 	mfi_enqueue_ready(cm);
 	mfi_startio(sc);
 
-	return (0);
+out:
+	return (error);
 }
 
 static void
@@ -1222,7 +1249,7 @@ mfi_aen_complete(struct mfi_command *cm)
 	struct mfi_frame_header *hdr;
 	struct mfi_softc *sc;
 	struct mfi_evt_detail *detail;
-	struct mfi_aen *mfi_aen_entry;
+	struct mfi_aen *mfi_aen_entry, *tmp;
 	int seq = 0, aborted = 0;
 
 	sc = cm->cm_sc;
@@ -1236,17 +1263,25 @@ mfi_aen_complete(struct mfi_command *cm)
 		aborted = 1;
 	} else {
 		sc->mfi_aen_triggered = 1;
-		if (sc->mfi_poll_waiting)
+		if (sc->mfi_poll_waiting) {
+			sc->mfi_poll_waiting = 0;
 			selwakeup(&sc->mfi_select);
+		}
 		detail = cm->cm_data;
+		/*
+		 * XXX If this function is too expensive or is recursive, then
+		 * events should be put onto a queue and processed later.
+		 */
 		mtx_unlock(&sc->mfi_io_lock);
 		mfi_decode_evt(sc, detail);
 		mtx_lock(&sc->mfi_io_lock);
 		seq = detail->seq + 1;
-		TAILQ_FOREACH(mfi_aen_entry, &sc->mfi_aen_pids, aen_link) {
+		TAILQ_FOREACH_SAFE(mfi_aen_entry, &sc->mfi_aen_pids, aen_link, tmp) {
 			TAILQ_REMOVE(&sc->mfi_aen_pids, mfi_aen_entry,
 			    aen_link);
+			PROC_LOCK(mfi_aen_entry->p);
 			psignal(mfi_aen_entry->p, SIGIO);
+			PROC_UNLOCK(mfi_aen_entry->p);
 			free(mfi_aen_entry, M_MFIBUF);
 		}
 	}
@@ -1258,9 +1293,7 @@ mfi_aen_complete(struct mfi_command *cm)
 
 	/* set it up again so the driver can catch more events */
 	if (!aborted) {
-		mtx_unlock(&sc->mfi_io_lock);
 		mfi_aen_setup(sc, seq);
-		mtx_lock(&sc->mfi_io_lock);
 	}
 }
 
@@ -1276,20 +1309,15 @@ mfi_get_entry(struct mfi_softc *sc, int seq)
 	int i;
 	int size;
 
-	mtx_lock(&sc->mfi_io_lock);
 	if ((cm = mfi_dequeue_free(sc)) == NULL) {
-		mtx_unlock(&sc->mfi_io_lock);
 		return (EBUSY);
 	}
-	mtx_unlock(&sc->mfi_io_lock);
 
 	size = sizeof(struct mfi_evt_list) + sizeof(struct mfi_evt_detail)
 		* (MAX_EVENTS - 1);
 	el = malloc(size, M_MFIBUF, M_NOWAIT | M_ZERO);
 	if (el == NULL) {
-		mtx_lock(&sc->mfi_io_lock);
 		mfi_release_command(cm);
-		mtx_unlock(&sc->mfi_io_lock);
 		return (ENOMEM);
 	}
 
@@ -1334,10 +1362,8 @@ mfi_get_entry(struct mfi_softc *sc, int seq)
 		}
 	}
 
-	mtx_lock(&sc->mfi_io_lock);
 	free(cm->cm_data, M_MFIBUF);
 	mfi_release_command(cm);
-	mtx_unlock(&sc->mfi_io_lock);
 	return (0);
 }
 
@@ -1408,7 +1434,7 @@ mfi_add_ld_complete(struct mfi_command *cm)
 		return;
 	}
 
-	ld->ld_id = ld_info->ld_config.properties.ld.target_id;
+	ld->ld_id = ld_info->ld_config.properties.ld.v.target_id;
 	ld->ld_disk = child;
 	ld->ld_info = ld_info;
 
@@ -1420,7 +1446,7 @@ mfi_add_ld_complete(struct mfi_command *cm)
 	mtx_unlock(&Giant);
 	mtx_lock(&sc->mfi_io_lock);
 }
-
+int mfi_io=0;
 static struct mfi_command *
 mfi_bio_command(struct mfi_softc *sc)
 {
@@ -1469,7 +1495,7 @@ mfi_bio_command(struct mfi_softc *sc)
 	cm->cm_sg = &io->sgl;
 	cm->cm_total_frame_size = MFI_IO_FRAME_SIZE;
 	cm->cm_flags = flags;
-
+	mfi_io++;
 	return (cm);
 }
 
@@ -1494,6 +1520,7 @@ mfi_bio_complete(struct mfi_command *cm)
 
 	mfi_release_command(cm);
 	mfi_disk_complete(bio);
+	mfi_io--;
 }
 
 void
@@ -1530,6 +1557,8 @@ mfi_mapcmd(struct mfi_softc *sc, struct mfi_command *cm)
 {
 	int error, polled;
 
+	mtx_assert(&sc->mfi_io_lock, MA_OWNED);
+
 	if (cm->cm_data != NULL) {
 		polled = (cm->cm_flags & MFI_CMD_POLLED) ? BUS_DMA_NOWAIT : 0;
 		error = bus_dmamap_load(sc->mfi_buffer_dmat, cm->cm_dmamap,
@@ -1654,9 +1683,6 @@ mfi_complete(struct mfi_softc *sc, struct mfi_command *cm)
 		cm->cm_complete(cm);
 	else
 		wakeup(cm);
-
-	sc->mfi_flags &= ~MFI_FLAGS_QFRZN;
-	mfi_startio(sc);
 }
 
 static int
@@ -1665,12 +1691,11 @@ mfi_abort(struct mfi_softc *sc, struct mfi_command *cm_abort)
 	struct mfi_command *cm;
 	struct mfi_abort_frame *abort;
 
-	mtx_lock(&sc->mfi_io_lock);
+	mtx_assert(&sc->mfi_io_lock, MA_OWNED);
+
 	if ((cm = mfi_dequeue_free(sc)) == NULL) {
-		mtx_unlock(&sc->mfi_io_lock);
 		return (EBUSY);
 	}
-	mtx_unlock(&sc->mfi_io_lock);
 
 	abort = &cm->cm_frame->abort;
 	abort->header.cmd = MFI_CMD_ABORT;
@@ -1683,12 +1708,10 @@ mfi_abort(struct mfi_softc *sc, struct mfi_command *cm_abort)
 	sc->mfi_aen_cm->cm_aen_abort = 1;
 	mfi_mapcmd(sc, cm);
 	mfi_polled_command(sc, cm);
-	mtx_lock(&sc->mfi_io_lock);
 	mfi_release_command(cm);
-	mtx_unlock(&sc->mfi_io_lock);
 
 	while (sc->mfi_aen_cm != NULL) {
-		tsleep(&sc->mfi_aen_cm, 0, "mfiabort", 5 * hz);
+		msleep(&sc->mfi_aen_cm, &sc->mfi_io_lock, 0, "mfiabort", 5 * hz);
 	}
 
 	return (0);
@@ -1741,7 +1764,10 @@ mfi_open(struct cdev *dev, int flags, int fmt, d_thread_t *td)
 	struct mfi_softc *sc;
 
 	sc = dev->si_drv1;
+
+	mtx_lock(&sc->mfi_io_lock);
 	sc->mfi_flags |= MFI_FLAGS_OPEN;
+	mtx_unlock(&sc->mfi_io_lock);
 
 	return (0);
 }
@@ -1750,18 +1776,21 @@ static int
 mfi_close(struct cdev *dev, int flags, int fmt, d_thread_t *td)
 {
 	struct mfi_softc *sc;
-	struct mfi_aen *mfi_aen_entry;
+	struct mfi_aen *mfi_aen_entry, *tmp;
 
 	sc = dev->si_drv1;
+
+	mtx_lock(&sc->mfi_io_lock);
 	sc->mfi_flags &= ~MFI_FLAGS_OPEN;
 
-	TAILQ_FOREACH(mfi_aen_entry, &sc->mfi_aen_pids, aen_link) {
+	TAILQ_FOREACH_SAFE(mfi_aen_entry, &sc->mfi_aen_pids, aen_link, tmp) {
 		if (mfi_aen_entry->p == curproc) {
 			TAILQ_REMOVE(&sc->mfi_aen_pids, mfi_aen_entry,
 			    aen_link);
 			free(mfi_aen_entry, M_MFIBUF);
 		}
 	}
+	mtx_unlock(&sc->mfi_io_lock);
 	return (0);
 }
 
@@ -1834,6 +1863,7 @@ mfi_ioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_thread_t *td)
 			break;
 		}
 	default:
+		device_printf(sc->mfi_dev, "IOCTL 0x%lx not handled\n", cmd);
 		error = ENOENT;
 		break;
 	}
@@ -1913,21 +1943,25 @@ mfi_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_threa
 			*sense_ptr = cm->cm_sense_busaddr;
 		}
 
+		mtx_lock(&sc->mfi_io_lock);
 		if ((error = mfi_mapcmd(sc, cm)) != 0) {
 			device_printf(sc->mfi_dev,
 			    "Controller info buffer map failed");
+			mtx_unlock(&sc->mfi_io_lock);
 			goto out;
 		}
 
 		if ((error = mfi_polled_command(sc, cm)) != 0) {
 			device_printf(sc->mfi_dev,
 			    "Controller polled failed");
+			mtx_unlock(&sc->mfi_io_lock);
 			goto out;
 		}
 
 		bus_dmamap_sync(sc->mfi_buffer_dmat, cm->cm_dmamap,
 				BUS_DMASYNC_POSTREAD);
 		bus_dmamap_unload(sc->mfi_buffer_dmat, cm->cm_dmamap);
+		mtx_unlock(&sc->mfi_io_lock);
 
 		temp = data;
 		for (i = 0; i < l_ioc.lioc_sge_count; i++) {
@@ -1967,6 +2001,14 @@ mfi_linux_ioctl_int(struct cdev *dev, u_long cmd, caddr_t arg, int flag, d_threa
 			goto out;
 		}
 
+		if (cm->cm_frame->header.cmd_status == MFI_STAT_OK) {
+			switch (cm->cm_frame->dcmd.opcode) {
+			case MFI_DCMD_CFG_CLEAR:
+			case MFI_DCMD_CFG_ADD:
+				/* mfi_ldrescan(sc); */
+				break;
+			}
+		}
 out:
 		if (data)
 			free(data, M_MFIBUF);
@@ -1984,6 +2026,7 @@ out:
 		printf("AEN IMPLEMENTED for pid %d\n", curproc->p_pid);
 		mfi_aen_entry = malloc(sizeof(struct mfi_aen), M_MFIBUF,
 		    M_WAITOK);
+		mtx_lock(&sc->mfi_io_lock);
 		if (mfi_aen_entry != NULL) {
 			mfi_aen_entry->p = curproc;
 			TAILQ_INSERT_TAIL(&sc->mfi_aen_pids, mfi_aen_entry,
@@ -1997,6 +2040,7 @@ out:
 			    aen_link);
 			free(mfi_aen_entry, M_MFIBUF);
 		}
+		mtx_unlock(&sc->mfi_io_lock);
 
 		return (error);
 	default:
@@ -2017,8 +2061,10 @@ mfi_poll(struct cdev *dev, int poll_events, struct thread *td)
 	sc = dev->si_drv1;
 
 	if (poll_events & (POLLIN | POLLRDNORM)) {
-		if (sc->mfi_aen_triggered != 0)
+		if (sc->mfi_aen_triggered != 0) {
 			revents |= poll_events & (POLLIN | POLLRDNORM);
+			sc->mfi_aen_triggered = 0;
+		}
 		if (sc->mfi_aen_triggered == 0 && sc->mfi_aen_cm == NULL) {
 			revents |= POLLERR;
 		}
@@ -2028,13 +2074,58 @@ mfi_poll(struct cdev *dev, int poll_events, struct thread *td)
 		if (poll_events & (POLLIN | POLLRDNORM)) {
 			sc->mfi_poll_waiting = 1;
 			selrecord(td, &sc->mfi_select);
-			sc->mfi_poll_waiting = 0;
 		}
 	}
 
 	return revents;
 }
 
+
+static void
+mfi_dump_all(void)
+{
+	struct mfi_softc *sc;
+	struct mfi_command *cm;
+	devclass_t dc;
+	time_t deadline;
+	int timedout;
+	int i;
+
+	dc = devclass_find("mfi");
+	if (dc == NULL) {
+		printf("No mfi dev class\n");
+		return;
+	}
+
+	for (i = 0; ; i++) {
+		sc = devclass_get_softc(dc, i);
+		if (sc == NULL)
+			break;
+		device_printf(sc->mfi_dev, "Dumping\n\n");
+		timedout = 0;
+		deadline = time_uptime - MFI_CMD_TIMEOUT;
+		mtx_lock(&sc->mfi_io_lock);
+		TAILQ_FOREACH(cm, &sc->mfi_busy, cm_link) {
+			if (cm->cm_timestamp < deadline) {
+				device_printf(sc->mfi_dev,
+				    "COMMAND %p TIMEOUT AFTER %d SECONDS\n", cm,
+				    (int)(time_uptime - cm->cm_timestamp));
+				MFI_PRINT_CMD(cm);
+				timedout++;
+			}
+		}
+
+#if 0
+		if (timedout)
+			MFI_DUMP_CMDS(SC);
+#endif
+
+		mtx_unlock(&sc->mfi_io_lock);
+	}
+
+	return;
+}
+
 static void
 mfi_timeout(void *data)
 {
@@ -2048,11 +2139,12 @@ mfi_timeout(void *data)
 	TAILQ_FOREACH(cm, &sc->mfi_busy, cm_link) {
 		if (sc->mfi_aen_cm == cm)
 			continue;
-		if (cm->cm_timestamp < deadline) {
+		if ((sc->mfi_aen_cm != cm) && (cm->cm_timestamp < deadline)) {
 			device_printf(sc->mfi_dev,
 			    "COMMAND %p TIMEOUT AFTER %d SECONDS\n", cm,
 			    (int)(time_uptime - cm->cm_timestamp));
 			MFI_PRINT_CMD(cm);
+			MFI_VALIDATE_CMD(sc, cm);
 			timedout++;
 		}
 	}
@@ -2067,5 +2159,7 @@ mfi_timeout(void *data)
 	callout_reset(&sc->mfi_watchdog_callout, MFI_CMD_TIMEOUT * hz,
 	    mfi_timeout, sc);
 
+	if (0)
+		mfi_dump_all();
 	return;
 }
diff --git a/sys/dev/mfi/mfi_debug.c b/sys/dev/mfi/mfi_debug.c
index 5cd2d35..515e6ed 100644
--- a/sys/dev/mfi/mfi_debug.c
+++ b/sys/dev/mfi/mfi_debug.c
@@ -46,6 +46,9 @@ __FBSDID("$FreeBSD$");
 #include <machine/resource.h>
 #include <machine/bus.h>
 
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
 #include <dev/mfi/mfireg.h>
 #include <dev/mfi/mfi_ioctl.h>
 #include <dev/mfi/mfivar.h>
@@ -225,4 +228,36 @@ mfi_dump_cmds(struct mfi_softc *sc)
 		mfi_print_generic_frame(sc, &sc->mfi_commands[i]);
 }
 
+void
+mfi_validate_sg(struct mfi_softc *sc, struct mfi_command *cm,
+	const char *function, int line)
+{
+	struct mfi_frame_header *hdr;
+	int i;
+	uint32_t count = 0, data_len;
+
+	hdr = &cm->cm_frame->header;
+	count = 0;
+	for (i = 0; i < hdr->sg_count; i++) {
+		count += cm->cm_sg->sg32[i].len;
+	}
+	/*
+	count++;
+	*/
+	data_len = hdr->data_len;
+	switch (hdr->cmd) {
+	case MFI_CMD_LD_READ:
+	case MFI_CMD_LD_WRITE:
+		data_len = data_len * 512;
+	case MFI_CMD_DCMD:
+		if (count != data_len) {
+			device_printf(sc->mfi_dev,
+			    "%s %d COMMAND %p S/G count bad %d %d %d 0x%jx\n",
+			    function, line, cm, count, data_len, cm->cm_len,
+			    (intmax_t)pmap_kextract((vm_offset_t)cm->cm_data));
+			MFI_PRINT_CMD(cm);
+		}
+	}
+}
+
 #endif
diff --git a/sys/dev/mfi/mfi_disk.c b/sys/dev/mfi/mfi_disk.c
index 909adcf..35de4df 100644
--- a/sys/dev/mfi/mfi_disk.c
+++ b/sys/dev/mfi/mfi_disk.c
@@ -103,6 +103,7 @@ mfi_disk_attach(device_t dev)
 	struct mfi_ld *ld;
 	uint64_t sectors;
 	uint32_t secsize;
+	char *state;
 
 	sc = device_get_softc(dev);
 	ld = device_get_ivars(dev);
@@ -117,8 +118,27 @@ mfi_disk_attach(device_t dev)
 	secsize = MFI_SECTOR_LEN;
 	TAILQ_INSERT_TAIL(&sc->ld_controller->mfi_ld_tqh, ld, ld_link);
 
-	device_printf(dev, "%juMB (%ju sectors) RAID\n",
-	    sectors / (1024 * 1024 / secsize), sectors);
+	switch (ld->ld_info->ld_config.params.state) {
+	case MFI_LD_STATE_OFFLINE:
+		state = "offline";
+		break;
+	case MFI_LD_STATE_PARTIALLY_DEGRADED:
+		state = "partially degraded";
+		break;
+	case MFI_LD_STATE_DEGRADED:
+		state = "degraded";
+		break;
+	case MFI_LD_STATE_OPTIMAL:
+		state = "optimal";
+		break;
+	default:
+		state = "unknown";
+		break;
+	}
+	device_printf(dev, "%juMB (%ju sectors) RAID volume '%s' is %s\n",
+		      sectors / (1024 * 1024 / secsize), sectors,
+		      ld->ld_info->ld_config.properties.name,
+		      state);
 
 	sc->ld_disk = disk_alloc();
 	sc->ld_disk->d_drv1 = sc;
diff --git a/sys/dev/mfi/mfireg.h b/sys/dev/mfi/mfireg.h
index 48a1382..d9df2c5 100644
--- a/sys/dev/mfi/mfireg.h
+++ b/sys/dev/mfi/mfireg.h
@@ -101,6 +101,8 @@ typedef enum {
 /* Direct commands */
 typedef enum {
 	MFI_DCMD_CTRL_GETINFO =		0x01010000,
+	MFI_DCMD_CTRL_MFC_DEFAULTS_GET =0x010e0201,
+	MFI_DCMD_CTRL_MFC_DEFAULTS_SET =0x010e0202,
 	MFI_DCMD_CTRL_FLUSHCACHE =	0x01101000,
 	MFI_DCMD_CTRL_SHUTDOWN =	0x01050000,
 	MFI_DCMD_CTRL_EVENT_GETINFO =	0x01040100,
@@ -110,6 +112,9 @@ typedef enum {
 	MFI_DCMD_LD_GET_INFO =		0x03020000,
 	MFI_DCMD_LD_GET_PROP =		0x03030000,
 	MFI_DCMD_LD_SET_PROP =		0x03040000,
+	MFI_DCMD_CFG_READ =		0x04010000,
+	MFI_DCMD_CFG_ADD =		0x04020000,
+	MFI_DCMD_CFG_CLEAR =		0x04030000,
 	MFI_DCMD_CLUSTER =		0x08000000,
 	MFI_DCMD_CLUSTER_RESET_ALL =	0x08010100,
 	MFI_DCMD_CLUSTER_RESET_LD =	0x08010200
@@ -244,6 +249,22 @@ typedef enum {
 	MR_EVT_ARGS_ECC
 } mfi_evt_args;
 
+typedef enum {
+	MR_LD_CACHE_WRITE_BACK =	0x01,
+	MR_LD_CACHE_WRITE_ADAPTIVE =	0x02,
+	MR_LD_CACHE_READ_AHEAD =	0x04,
+	MR_LD_CACHE_READ_ADAPTIVE =	0x08,
+	MR_LD_CACHE_WRITE_CACHE_BAD_BBU=0x10,
+	MR_LD_CACHE_ALLOW_WRITE_CACHE =	0x20,
+	MR_LD_CACHE_ALLOW_READ_CACHE =	0x40
+} mfi_ld_cache;
+
+typedef enum {
+	MR_PD_CACHE_UNCHANGED  =	0,
+	MR_PD_CACHE_ENABLE =		1,
+	MR_PD_CACHE_DISABLE =		2
+} mfi_pd_cache;
+
 /*
  * Other propertities and definitions
  */
@@ -456,6 +477,51 @@ struct mfi_info_component {
 	char		 build_time[16];
 } __packed;
 
+/* Controller default settings */
+struct mfi_defaults {
+	uint64_t	sas_addr;
+	uint8_t		phy_polarity;
+	uint8_t		background_rate;
+	uint8_t		stripe_size;
+	uint8_t		flush_time;
+	uint8_t		write_back;
+	uint8_t		read_ahead;
+	uint8_t		cache_when_bbu_bad;
+	uint8_t		cached_io;
+	uint8_t		smart_mode;
+	uint8_t		alarm_disable;
+	uint8_t		coercion;
+	uint8_t		zrc_config;
+	uint8_t		dirty_led_shows_drive_activity;
+	uint8_t		bios_continue_on_error;
+	uint8_t		spindown_mode;
+	uint8_t		allowed_device_types;
+	uint8_t		allow_mix_in_enclosure;
+	uint8_t		allow_mix_in_ld;
+	uint8_t		allow_sata_in_cluster;
+	uint8_t		max_chained_enclosures;
+	uint8_t		disable_ctrl_r;
+	uint8_t		enabel_web_bios;
+	uint8_t		phy_polarity_split;
+	uint8_t		direct_pd_mapping;
+	uint8_t		bios_enumerate_lds;
+	uint8_t		restored_hot_spare_on_insertion;
+	uint8_t		expose_enclosure_devices;
+	uint8_t		maintain_pd_fail_history;
+	uint8_t		resv[28];
+} __packed;
+
+/* Controller default settings */
+struct mfi_bios_data {
+	uint16_t	boot_target_id;
+	uint8_t		do_not_int_13;
+	uint8_t		continue_on_error;
+	uint8_t		verbose;
+	uint8_t		geometry;
+	uint8_t		expose_all_drives;
+	uint8_t		reserved[56];
+	uint8_t		check_sum;
+} __packed;
 
 /* SAS (?) controller info, returned from MFI_DCMD_CTRL_GETINFO. */
 struct mfi_ctrl_info {
@@ -721,17 +787,117 @@ struct mfi_evt_list {
 	struct mfi_evt_detail	event[1];
 } __packed;
 
-struct mfi_ldref {
-	uint8_t		target_id;
-	uint8_t		reserved;
-	uint16_t	seq;
+union mfi_pd_ref {
+	struct {
+		uint16_t	device_id;
+		uint16_t	seq_num;
+	} v;
+	uint32_t	ref;
+} __packed;
+
+union mfi_pd_ddf_type {
+	struct {
+		union {
+			struct {
+				uint16_t	forced_pd_guid	: 1;
+				uint16_t	in_vd		: 1;
+				uint16_t	is_global_spare	: 1;
+				uint16_t	is_spare	: 1;
+				uint16_t	is_foreign	: 1;
+				uint16_t	reserved	: 7;
+				uint16_t	intf		: 4;
+			} pd_type;
+			uint16_t	type;
+		} v;
+		uint16_t		reserved;
+	} ddf;
+	struct {
+		uint32_t		reserved;
+	} non_disk;
+	uint32_t			type;
+} __packed;
+
+struct mfi_pd_progress {
+	struct {
+		uint32_t		rbld	: 1;
+		uint32_t		patrol	: 1;
+		uint32_t		clear	: 1;
+		uint32_t		reserved: 29;
+	} active;
+	struct mfi_progress		rbld;
+	struct mfi_progress		patrol;
+	struct mfi_progress		clear;
+	struct mfi_progress		reserved[4];
+} __packed;
+
+struct mfi_pd_info {
+	union mfi_pd_ref		ref;
+	uint8_t				inquiry_data[96];
+	uint8_t				vpd_page83[64];
+	uint8_t				not_supported;
+	uint8_t				scsi_dev_type;
+	uint8_t				connected_port_bitmap;
+	uint8_t				device_speed;
+	uint32_t			media_err_count;
+	uint32_t			other_err_count;
+	uint32_t			pred_fail_count;
+	uint32_t			last_pred_fail_event_seq_num;
+	uint16_t			fw_state;
+	uint8_t				disable_for_removal;
+	uint8_t				link_speed;
+	union mfi_pd_ddf_type		state;
+	struct {
+		uint8_t			count;
+		uint8_t			is_path_broken;
+		uint8_t			reserved[6];
+		uint64_t		sas_addr[4];
+	} path_info;
+	uint64_t			raw_size;
+	uint64_t			non_coerced_size;
+	uint64_t			coerced_size;
+	uint16_t			encl_device_id;
+	uint8_t				encl_index;
+	uint8_t				slot_number;
+	struct mfi_pd_progress		prog_info;
+	uint8_t				bad_block_table_full;
+	uint8_t				unusable_in_current_config;
+	uint8_t				vpd_page83_ext[64];
+	uint8_t				reserved[512-358];
+} __packed;
+
+struct mfi_pd_address {
+	uint16_t		device_id;
+	uint16_t		encl_device_id;
+	uint8_t			encl_index;
+	uint8_t			slot_number;
+	uint8_t			scsi_dev_type;
+	uint8_t			connect_port_bitmap;
+	uint64_t		sas_addr[2];
+} __packed;
+
+struct mfi_pd_list {
+	uint32_t		size;
+	uint32_t		count;
+	uint8_t			data;
+	/*
+	struct mfi_pd_address	addr[];
+	*/
+} __packed;
+
+union mfi_ld_ref {
+	struct {
+		uint8_t		target_id;
+		uint8_t		reserved;
+		uint16_t	seq;
+	} v;
+	uint32_t		ref;
 } __packed;
 
 struct mfi_ld_list {
 	uint32_t		ld_count;
 	uint32_t		reserved1;
 	struct {
-		struct mfi_ldref	ld;
+		union mfi_ld_ref	ld;
 		uint8_t		state;
 		uint8_t		reserved2[3];
 		uint64_t	size;
@@ -753,7 +919,7 @@ enum mfi_ld_state {
 };
 
 struct mfi_ld_props {
-	struct mfi_ldref	ld;
+	union mfi_ld_ref	ld;
 	char			name[16];
 	uint8_t			default_cache_policy;
 	uint8_t			access_policy;
@@ -814,4 +980,57 @@ struct mfi_ld_info {
 	uint8_t			reserved2[16];
 } __packed;
 
+union mfi_spare_type {
+	struct {
+		uint8_t		is_dedicate		:1;
+		uint8_t		is_revertable		:1;
+		uint8_t		is_encl_affinity	:1;
+		uint8_t		reserved		:5;
+	} v;
+	uint8_t		type;
+} __packed;
+
+#define MAX_ARRAYS 16
+struct mfi_spare {
+	union mfi_pd_ref	ref;
+	union mfi_spare_type	spare_type;
+	uint8_t			reserved[2];
+	uint8_t			array_count;
+	uint16_t		array_refd[MAX_ARRAYS];
+} __packed;
+
+#define MAX_ROW_SIZE 32
+struct mfi_array {
+	uint64_t			size;
+	uint8_t				num_drives;
+	uint8_t				reserved;
+	uint16_t			array_ref;
+	uint8_t				pad[20];
+	struct {
+		union mfi_pd_ref	ref;
+		uint16_t		fw_state;
+		struct {
+			uint8_t		pd;
+			uint8_t		slot;
+		} encl;
+	} pd[MAX_ROW_SIZE];
+} __packed;
+
+struct mfi_config_data {
+	uint32_t		size;
+	uint16_t		array_count;
+	uint16_t		array_size;
+	uint16_t		log_drv_count;
+	uint16_t		log_drv_size;
+	uint16_t		spares_count;
+	uint16_t		spares_size;
+	uint8_t			reserved[16];
+	uint8_t			data;
+	/*
+	struct mfi_array	array[];
+	struct mfi_ld_config	ld[];
+	struct mfi_spare	spare[];
+	*/
+} __packed;
+
 #endif /* _MFIREG_H */
diff --git a/sys/dev/mfi/mfivar.h b/sys/dev/mfi/mfivar.h
index 1017cbf..7028cc4 100644
--- a/sys/dev/mfi/mfivar.h
+++ b/sys/dev/mfi/mfivar.h
@@ -92,6 +92,7 @@ struct mfi_softc {
 #define MFI_FLAGS_SG64		(1<<0)
 #define MFI_FLAGS_QFRZN		(1<<1)
 #define MFI_FLAGS_OPEN		(1<<2)
+#define MFI_FLAGS_STOP		(1<<3)
 
 	struct mfi_hwcomms		*mfi_comms;
 	TAILQ_HEAD(,mfi_command)	mfi_free;
@@ -327,11 +328,14 @@ MALLOC_DECLARE(M_MFIBUF);
 #ifdef MFI_DEBUG
 extern void mfi_print_cmd(struct mfi_command *cm);
 extern void mfi_dump_cmds(struct mfi_softc *sc);
+extern void mfi_validate_sg(struct mfi_softc *, struct mfi_command *, const char *, int );
 #define MFI_PRINT_CMD(cm)	mfi_print_cmd(cm)
-#define MFI_DUMP_CMDS(sc)	mfi_dump_cmds(sc);
+#define MFI_DUMP_CMDS(sc)	mfi_dump_cmds(sc)
+#define MFI_VALIDATE_CMD(sc, cm) mfi_validate_sg(sc, cm, __FUNCTION__, __LINE__)
 #else
 #define MFI_PRINT_CMD(cm)
 #define MFI_DUMP_CMDS(sc)
+#define MFI_VALIDATE_CMD(sc, cm)
 #endif
 
 #endif /* _MFIVAR_H */