Add a command debugging module and a periodic watchdog timer.

Sponsored by: IronPort

3 files changed, 287 insertions, 0 deletions

diff --git a/sys/dev/mfi/mfi.c b/sys/dev/mfi/mfi.c
index 8c97ea3..b8fab65 100644
--- a/sys/dev/mfi/mfi.c
+++ b/sys/dev/mfi/mfi.c
@@ -81,6 +81,7 @@ static int	mfi_send_frame(struct mfi_softc *, struct mfi_command *);
 static void	mfi_complete(struct mfi_softc *, struct mfi_command *);
 static int	mfi_abort(struct mfi_softc *, struct mfi_command *);
 static int	mfi_linux_ioctl_int(struct cdev *, u_long, caddr_t, int, d_thread_t *);
+static void	mfi_timeout(void *);
 
 
 SYSCTL_NODE(_hw, OID_AUTO, mfi, CTLFLAG_RD, 0, "MFI driver parameters");
@@ -389,6 +390,11 @@ mfi_attach(struct mfi_softc *sc)
 	if (sc->mfi_cdev != NULL)
 		sc->mfi_cdev->si_drv1 = sc;
 
+	/* Start the timeout watchdog */
+	callout_init(&sc->mfi_watchdog_callout, 1);
+	callout_reset(&sc->mfi_watchdog_callout, MFI_CMD_TIMEOUT * hz,
+	    mfi_timeout, sc);
+
 	return (0);
 }
 
@@ -416,6 +422,7 @@ mfi_alloc_commands(struct mfi_softc *sc)
 		cm->cm_sense = &sc->mfi_sense[i];
 		cm->cm_sense_busaddr= sc->mfi_sense_busaddr + MFI_SENSE_LEN * i;
 		cm->cm_sc = sc;
+		cm->cm_index = i;
 		if (bus_dmamap_create(sc->mfi_buffer_dmat, 0,
 		    &cm->cm_dmamap) == 0)
 			mfi_release_command(cm);
@@ -701,6 +708,8 @@ mfi_free(struct mfi_softc *sc)
 	struct mfi_command *cm;
 	int i;
 
+	callout_drain(&sc->mfi_watchdog_callout);
+
 	if (sc->mfi_cdev != NULL)
 		destroy_dev(sc->mfi_cdev);
 
@@ -1530,6 +1539,7 @@ mfi_mapcmd(struct mfi_softc *sc, struct mfi_command *cm)
 			return (0);
 		}
 	} else {
+		cm->cm_timestamp = time_uptime;
 		mfi_enqueue_busy(cm);
 		error = mfi_send_frame(sc, cm);
 	}
@@ -1591,6 +1601,7 @@ mfi_data_cb(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 
 	/* The caller will take care of delivering polled commands */
 	if ((cm->cm_flags & MFI_CMD_POLLED) == 0) {
+		cm->cm_timestamp = time_uptime;
 		mfi_enqueue_busy(cm);
 		mfi_send_frame(sc, cm);
 	}
@@ -2023,3 +2034,36 @@ mfi_poll(struct cdev *dev, int poll_events, struct thread *td)
 
 	return revents;
 }
+
+static void
+mfi_timeout(void *data)
+{
+	struct mfi_softc *sc = (struct mfi_softc *)data;
+	struct mfi_command *cm;
+	time_t deadline;
+	int timedout = 0;
+
+	deadline = time_uptime - MFI_CMD_TIMEOUT;
+	mtx_lock(&sc->mfi_io_lock);
+	TAILQ_FOREACH(cm, &sc->mfi_busy, cm_link) {
+		if (cm->cm_timestamp < deadline) {
+			device_printf(sc->mfi_dev,
+			    "COMMAND %p TIMEOUT AFTER %d SECONDS\n", cm,
+			    (int)(time_uptime - cm->cm_timestamp));
+			MFI_PRINT_CMD(cm);
+			timedout++;
+		}
+	}
+
+#if 0
+	if (timedout)
+		MFI_DUMP_CMDS(SC);
+#endif
+
+	mtx_unlock(&sc->mfi_io_lock);
+
+	callout_reset(&sc->mfi_watchdog_callout, MFI_CMD_TIMEOUT * hz,
+	    mfi_timeout, sc);
+
+	return;
+}
diff --git a/sys/dev/mfi/mfi_debug.c b/sys/dev/mfi/mfi_debug.c
new file mode 100644
index 0000000..9120f08
--- /dev/null
+++ b/sys/dev/mfi/mfi_debug.c
@@ -0,0 +1,228 @@
+/*-
+ * Copyright (c) 2006 IronPort Systems
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <opt_mfi.h>
+
+#ifdef MFI_DEBUG
+
+#include <sys/param.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/conf.h>
+#include <sys/bus.h>
+#include <sys/bio.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/malloc.h>
+#include <sys/selinfo.h>
+#include <sys/taskqueue.h>
+#include <sys/uio.h>
+#include <machine/resource.h>
+#include <machine/bus.h>
+
+#include <dev/mfi/mfireg.h>
+#include <dev/mfi/mfi_ioctl.h>
+#include <dev/mfi/mfivar.h>
+
+static void
+mfi_print_frame_flags(device_t dev, uint32_t flags)
+{
+	device_printf(dev, "flags=%b\n", flags,
+	    "\20"
+	    "\1NOPOST"
+	    "\2SGL64"
+	    "\3SENSE64"
+	    "\4WRITE"
+	    "\5READ");
+}
+
+static void
+mfi_print_sgl(struct mfi_frame_header *hdr, union mfi_sgl *sgl, int count)
+{
+	int i, columns = 0;
+
+	printf("SG List:\n");
+	for (i = 0; i < count; i++) {
+		if (hdr->flags & MFI_FRAME_SGL64) {
+			printf("0x%lx:%06d ", (u_long)sgl->sg64[i].addr,
+			    sgl->sg64[i].len);
+			columns += 26;
+			if (columns > 77) {
+				printf("\n");
+				columns = 0;
+			}
+		} else {
+			printf("0x%x:%06d ", sgl->sg32[i].addr,
+			    sgl->sg32[i].len);
+			columns += 18;
+			if (columns > 71) {
+				printf("\n");
+				columns = 0;
+			}
+		}
+	}
+	if (columns != 0)
+		printf("\n");
+
+}
+
+static void
+mfi_print_ldio(struct mfi_softc *sc, device_t dev, struct mfi_command *cm)
+{
+	struct mfi_io_frame *io;
+	struct mfi_frame_header *hdr;
+
+	io = &cm->cm_frame->io;
+	hdr = &io->header;
+
+	device_printf(dev, "cmd=%s target_id=%d sg_count=%d data_len=%d "
+	    "lba=%d\n", (hdr->cmd == MFI_CMD_LD_READ) ? "LD_READ":"LD_WRITE",
+	     hdr->target_id, hdr->sg_count, hdr->data_len, io->lba_lo);
+	mfi_print_frame_flags(dev, hdr->flags);
+	mfi_print_sgl(hdr, &io->sgl, hdr->sg_count);
+
+}
+
+static void
+mfi_print_dcmd(struct mfi_softc *sc, device_t dev, struct mfi_command *cm)
+{
+	struct mfi_dcmd_frame *dcmd;
+	struct mfi_frame_header *hdr;
+	const char *opcode;
+
+	dcmd = &cm->cm_frame->dcmd;
+	hdr = &dcmd->header;
+
+	switch (dcmd->opcode) {
+	case MFI_DCMD_CTRL_GETINFO:
+		opcode = "CTRL_GETINFO";
+		break;
+	case MFI_DCMD_CTRL_FLUSHCACHE:
+		opcode = "CTRL_FLUSHCACHE";
+		break;
+	case MFI_DCMD_CTRL_SHUTDOWN:
+		opcode = "CTRL_SHUTDOWN";
+		break;
+	case MFI_DCMD_CTRL_EVENT_GETINFO:
+		opcode = "EVENT_GETINFO";
+		break;
+	case MFI_DCMD_CTRL_EVENT_GET:
+		opcode = "EVENT_GET";
+		break;
+	case MFI_DCMD_CTRL_EVENT_WAIT:
+		opcode = "EVENT_WAIT";
+		break;
+	case MFI_DCMD_LD_GET_LIST:
+		opcode = "LD_GET_LIST";
+		break;
+	case MFI_DCMD_LD_GET_INFO:
+		opcode = "LD_GET_INFO";
+		break;
+	case MFI_DCMD_LD_GET_PROP:
+		opcode = "LD_GET_PROP";
+		break;
+	case MFI_DCMD_LD_SET_PROP:
+		opcode = "LD_SET_PROP";
+		break;
+	case MFI_DCMD_CLUSTER:
+		opcode = "CLUSTER";
+		break;
+	case MFI_DCMD_CLUSTER_RESET_ALL:
+		opcode = "CLUSTER_RESET_ALL";
+		break;
+	case MFI_DCMD_CLUSTER_RESET_LD:
+		opcode = "CLUSTER_RESET_LD";
+		break;
+	default:
+		opcode = "UNKNOWN";
+		break;
+	}
+
+	device_printf(dev, "cmd=MFI_CMD_DCMD opcode=%s data_len=%d\n",
+	    opcode, hdr->data_len);
+	mfi_print_frame_flags(dev, hdr->flags);
+	mfi_print_sgl(hdr, &dcmd->sgl, hdr->sg_count);
+
+}
+
+static void
+mfi_print_generic_frame(struct mfi_softc *sc, struct mfi_command *cm)
+{
+	hexdump(cm->cm_frame, cm->cm_total_frame_size, NULL, HD_OMIT_CHARS);
+}
+
+void
+mfi_print_cmd(struct mfi_command *cm)
+{
+	device_t dev;
+	struct mfi_softc *sc;
+
+	sc = cm->cm_sc;
+	dev = sc->mfi_dev;
+
+	device_printf(dev, "cm=%p index=%d total_frame_size=%d "
+	    "extra_frames=%d\n", cm, cm->cm_index, cm->cm_total_frame_size,
+	    cm->cm_extra_frames);
+	device_printf(dev, "flags=%b\n", cm->cm_flags,
+	    "\20"
+	    "\1MAPPED"
+	    "\2DATAIN"
+	    "\3DATAOUT"
+	    "\4COMPLETED"
+	    "\5POLLED"
+	    "\6Q_FREE"
+	    "\7Q_READY"
+	    "\10Q_BUSY");
+
+	switch (cm->cm_frame->header.cmd) {
+	case MFI_CMD_DCMD:
+		mfi_print_dcmd(sc, dev, cm);
+		break;
+	case MFI_CMD_LD_READ:
+	case MFI_CMD_LD_WRITE:
+		mfi_print_ldio(sc, dev, cm);
+		break;
+	default:
+		mfi_print_generic_frame(sc, cm);
+		break;
+	}
+
+	return;
+}
+
+void
+mfi_dump_cmds(struct mfi_softc *sc)
+{
+	int i;
+
+	for (i = 0; i < sc->mfi_total_cmds; i++)
+		mfi_print_generic_frame(sc, &sc->mfi_commands[i]);
+}
+
+#endif
diff --git a/sys/dev/mfi/mfivar.h b/sys/dev/mfi/mfivar.h
index 232904f..1017cbf 100644
--- a/sys/dev/mfi/mfivar.h
+++ b/sys/dev/mfi/mfivar.h
@@ -46,6 +46,7 @@ struct mfi_softc;
 
 struct mfi_command {
 	TAILQ_ENTRY(mfi_command) cm_link;
+	time_t			cm_timestamp;
 	struct mfi_softc	*cm_sc;
 	union mfi_frame		*cm_frame;
 	uint32_t		cm_frame_busaddr;
@@ -70,6 +71,7 @@ struct mfi_command {
 	int			cm_aen_abort;
 	void			(* cm_complete)(struct mfi_command *cm);
 	void			*cm_private;
+	int			cm_index;
 };
 
 struct mfi_ld {
@@ -170,6 +172,7 @@ struct mfi_softc {
 	eventhandler_tag		mfi_eh;
 	struct cdev			*mfi_cdev;
 
+	struct callout			mfi_watchdog_callout;
 	struct mtx			mfi_io_lock;
 };
 
@@ -319,4 +322,16 @@ mfi_print_sense(struct mfi_softc *sc, void *sense)
 
 MALLOC_DECLARE(M_MFIBUF);
 
+#define MFI_CMD_TIMEOUT 30
+
+#ifdef MFI_DEBUG
+extern void mfi_print_cmd(struct mfi_command *cm);
+extern void mfi_dump_cmds(struct mfi_softc *sc);
+#define MFI_PRINT_CMD(cm)	mfi_print_cmd(cm)
+#define MFI_DUMP_CMDS(sc)	mfi_dump_cmds(sc);
+#else
+#define MFI_PRINT_CMD(cm)
+#define MFI_DUMP_CMDS(sc)
+#endif
+
 #endif /* _MFIVAR_H */