Push things closer to path failover by implementing loop down and

gone device timers and zombie state entries. There are tunables
that can be used to select a number of parameters.

loop_down_limit - how long to wait for loop to come back up before
declaring
all devices dead (default 300 seconds)

gone_device_time- how long to wait for a device that has appeared
to leave the loop or fabric to reappear (default 30 seconds)

Internal tunables include (which should be externalized):

quick_boot_time- how long to wait when booting for loop to come up

change_is_bad- whether or not to accept devices with the same
WWNN/WWPN that reappear at a different PortID as being the 'same'
device.

Keen students of some of the subtle issues here will ask how
one can keep devices from being re-accepted at all (the answer
is to set a gone_device_time to zero- that effectively would
be the same thing).

8 files changed, 940 insertions, 495 deletions

diff --git a/sys/dev/isp/isp.c b/sys/dev/isp/isp.c
index 4f5537f..d7e811c 100644
--- a/sys/dev/isp/isp.c
+++ b/sys/dev/isp/isp.c
@@ -60,6 +60,9 @@ __FBSDID("$FreeBSD$");
  */
 
 #define	MBOX_DELAY_COUNT	1000000 / 100
+#define	ISP_MARK_PORTDB(a, b)	\
+    isp_prt(isp, ISP_LOGSANCFG, "line %d: markportdb", __LINE__); \
+    isp_mark_portdb(a, b)
 
 /*
  * Local static data
@@ -109,7 +112,6 @@ static void isp_scsi_init(ispsoftc_t *);
 static void isp_scsi_channel_init(ispsoftc_t *, int);
 static void isp_fibre_init(ispsoftc_t *);
 static void isp_fibre_init_2400(ispsoftc_t *);
-static void isp_dump_portdb(ispsoftc_t *);
 static void isp_mark_portdb(ispsoftc_t *, int);
 static void isp_plogx_24xx(ispsoftc_t *, uint16_t, uint32_t, int *);
 static int isp_port_login(ispsoftc_t *, uint16_t, uint32_t);
@@ -117,7 +119,7 @@ static void isp_port_logout(ispsoftc_t *, uint16_t, uint32_t);
 static int isp_getpdb(ispsoftc_t *, uint16_t, isp_pdb_t *, int);
 static uint64_t isp_get_portname(ispsoftc_t *, int, int);
 static int isp_fclink_test(ispsoftc_t *, int);
-static const char *isp2100_fw_statename(int);
+static const char *ispfc_fw_statename(int);
 static int isp_pdb_sync(ispsoftc_t *);
 static int isp_scan_loop(ispsoftc_t *);
 static int isp_gid_ft_sns(ispsoftc_t *);
@@ -1173,7 +1175,7 @@ isp_init(ispsoftc_t *isp)
 		/*
 		 * Do this *before* initializing the firmware.
 		 */
-		isp_mark_portdb(isp, 0);
+		ISP_MARK_PORTDB(isp, 0);
 		FCPARAM(isp)->isp_fwstate = FW_CONFIG_WAIT;
 		FCPARAM(isp)->isp_loopstate = LOOP_NIL;
 
@@ -2047,53 +2049,6 @@ isp_fibre_init_2400(ispsoftc_t *isp)
 	isp->isp_state = ISP_INITSTATE;
 }
 
-/*
- * Fibre Channel Support- get the port database for the id.
- */
-static void
-isp_dump_portdb(ispsoftc_t *isp)
-{
-	fcparam *fcp = (fcparam *) isp->isp_param;
-	int i;
-
-	for (i = 0; i < MAX_FC_TARG; i++) {
-		char mb[4];
-		const char *dbs[8] = {
-			"NIL ",
-			"PROB",
-			"DEAD",
-			"CHGD",
-			"NEW ",
-			"PVLD",
-			"????",
-			"VLD "
-		};
-		const char *roles[4] = {
-			" UNK", " TGT", " INI", "TINI"
-		};
-		fcportdb_t *lp = &fcp->portdb[i];
-
-		if (lp->state == FC_PORTDB_STATE_NIL) {
-			continue;
-		}
-		if (lp->ini_map_idx) {
-			SNPRINTF(mb, sizeof (mb), "%3d",
-			    ((int) lp->ini_map_idx) - 1);
-		} else {
-			SNPRINTF(mb, sizeof (mb), "---");
-		}
-		isp_prt(isp, ISP_LOGALL, "%d: %s al%d tgt %s %s 0x%06x =>%s"
-		    " 0x%06x; WWNN 0x%08x%08x WWPN 0x%08x%08x", i,
-		    dbs[lp->state], lp->autologin, mb,
-		    roles[lp->roles], lp->portid,
-		    roles[lp->new_roles], lp->new_portid,
-		    (uint32_t) (lp->node_wwn >> 32),
-		    (uint32_t) (lp->node_wwn),
-		    (uint32_t) (lp->port_wwn >> 32),
-		    (uint32_t) (lp->port_wwn));
-	}
-}
-
 static void
 isp_mark_portdb(ispsoftc_t *isp, int onprobation)
 {
@@ -2101,7 +2056,6 @@ isp_mark_portdb(ispsoftc_t *isp, int onprobation)
 	int i;
 
 	for (i = 0; i < MAX_FC_TARG; i++) {
-		fcp->isp_ini_map[i] = 0;
 		if (onprobation == 0) {
 			MEMZERO(&fcp->portdb[i], sizeof (fcportdb_t));
 		} else {
@@ -2113,6 +2067,8 @@ isp_mark_portdb(ispsoftc_t *isp, int onprobation)
 				fcp->portdb[i].state =
 					FC_PORTDB_STATE_PROBATIONAL;
 				break;
+			case FC_PORTDB_STATE_ZOMBIE:
+				break;
 			case FC_PORTDB_STATE_NIL:
 			default:
 				MEMZERO(&fcp->portdb[i], sizeof (fcportdb_t));
@@ -2167,6 +2123,7 @@ isp_plogx_24xx(ispsoftc_t *isp, uint16_t handle, uint32_t portid, int *log_ret)
 	mbs.param[6] = DMA_WD3(FCPARAM(isp)->isp_scdma);
 	mbs.param[7] = DMA_WD2(FCPARAM(isp)->isp_scdma);
 	mbs.logval = MBLOGALL;
+	mbs.timeout = 250000;
 	MEMORYBARRIER(isp, SYNC_SFORDEV, 0, QENTRY_LEN);
 	isp_mboxcmd(isp, &mbs);
 	if (mbs.param[0] != MBOX_COMMAND_COMPLETE) {
@@ -2276,6 +2233,7 @@ isp_port_login(ispsoftc_t *isp, uint16_t handle, uint32_t portid)
 	mbs.param[3] = portid;
 
 	mbs.logval = MBLOGNONE;
+	mbs.timeout = 250000;
 	isp_mboxcmd(isp, &mbs);
 
 	switch (mbs.param[0]) {
@@ -2463,8 +2421,8 @@ isp_fclink_test(ispsoftc_t *isp, int usdelay)
 
 	fcp = isp->isp_param;
 
-	isp_prt(isp, ISP_LOGDEBUG0, "FC Link Test Entry");
-	isp_mark_portdb(isp, 1);
+	isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0, "FC Link Test Entry");
+	ISP_MARK_PORTDB(isp, 1);
 
 	/*
 	 * Wait up to N microseconds for F/W to go to a ready state.
@@ -2479,9 +2437,10 @@ isp_fclink_test(ispsoftc_t *isp, int usdelay)
 		GET_NANOTIME(&hra);
 		isp_fw_state(isp);
 		if (lwfs != fcp->isp_fwstate) {
-			isp_prt(isp, ISP_LOGINFO, "Firmware State <%s->%s>",
-			    isp2100_fw_statename((int)lwfs),
-			    isp2100_fw_statename((int)fcp->isp_fwstate));
+			isp_prt(isp, ISP_LOGCONFIG|ISP_LOGSANCFG,
+			    "Firmware State <%s->%s>",
+			    ispfc_fw_statename((int)lwfs),
+			    ispfc_fw_statename((int)fcp->isp_fwstate));
 			lwfs = fcp->isp_fwstate;
 		}
 		if (fcp->isp_fwstate == FW_READY) {
@@ -2533,7 +2492,7 @@ isp_fclink_test(ispsoftc_t *isp, int usdelay)
 	 * If we haven't gone to 'ready' state, return.
 	 */
 	if (fcp->isp_fwstate != FW_READY) {
-		isp_prt(isp, ISP_LOGDEBUG0,
+		isp_prt(isp, ISP_LOGSANCFG,
 		    "isp_fclink_test: not at FW_READY state");
 		return (-1);
 	}
@@ -2645,19 +2604,19 @@ not_on_fabric:
 	/*
 	 * Announce ourselves, too.
 	 */
-	isp_prt(isp, ISP_LOGCONFIG, topology, fcp->isp_portid,
+	isp_prt(isp, ISP_LOGSANCFG|ISP_LOGCONFIG, topology, fcp->isp_portid,
 	    fcp->isp_loopid, toponames[fcp->isp_topo]);
-	isp_prt(isp, ISP_LOGCONFIG, ourwwn,
+	isp_prt(isp, ISP_LOGSANCFG|ISP_LOGCONFIG, ourwwn,
 	    (uint32_t) (ISP_NODEWWN(isp) >> 32),
 	    (uint32_t) ISP_NODEWWN(isp),
 	    (uint32_t) (ISP_PORTWWN(isp) >> 32),
 	    (uint32_t) ISP_PORTWWN(isp));
-	isp_prt(isp, ISP_LOGDEBUG0, "FC Link Test Complete");
+	isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0, "FC Link Test Complete");
 	return (0);
 }
 
 static const char *
-isp2100_fw_statename(int state)
+ispfc_fw_statename(int state)
 {
 	switch(state) {
 	case FW_CONFIG_WAIT:	return "Config Wait";
@@ -2736,6 +2695,8 @@ isp_pdb_sync(ispsoftc_t *isp)
 		}
 	}
 
+	isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0, "Synchronizing PDBs");
+
 	fcp->isp_loopstate = LOOP_SYNCING_PDB;
 
 	for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
@@ -2757,12 +2718,11 @@ isp_pdb_sync(ispsoftc_t *isp)
 		switch (lp->state) {
 		case FC_PORTDB_STATE_PROBATIONAL:
 		case FC_PORTDB_STATE_DEAD:
-			isp_async(isp, ISPASYNC_DEV_GONE, lp);
-			if (lp->ini_map_idx) {
-				fcp->isp_ini_map[lp->ini_map_idx-1] = 0;
-				lp->ini_map_idx = 0;
-			}
+			/*
+			 * It's up to the outer layers to clear isp_ini_map.
+			 */
 			lp->state = FC_PORTDB_STATE_NIL;
+			isp_async(isp, ISPASYNC_DEV_GONE, lp);
 			if (lp->autologin == 0) {
 				if (IS_24XX(isp)) {
 					int action =
@@ -2782,68 +2742,57 @@ isp_pdb_sync(ispsoftc_t *isp)
 			}
 			lp->new_roles = 0;
 			lp->new_portid = 0;
+			/*
+			 * Note that we might come out of this with our state
+			 * set to FC_PORTDB_STATE_ZOMBIE.
+			 */
 			break;
 		case FC_PORTDB_STATE_NEW:
 			/*
-			 * If *we* have a new target dole and *it* has a target
-			 * role, assign a new target id to it.
+			 * It's up to the outer layers to assign a virtual
+			 * target id in isp_ini_map (if any).
 			 */
 			lp->portid = lp->new_portid;
 			lp->roles = lp->new_roles;
 			lp->state = FC_PORTDB_STATE_VALID;
-			if ((isp->isp_role & ISP_ROLE_INITIATOR) &&
-			    (lp->roles & (SVC3_TGT_ROLE >> SVC3_ROLE_SHIFT))) {
-				int i, t = dbidx;
-				for (i = 0; i < MAX_FC_TARG; i++) {
-					if (i < FL_ID || i > SNS_ID) {
-						if (fcp->isp_ini_map[t] == 0) {
-							break;
-						}
-					}
-					if (++t == MAX_FC_TARG) {
-						t = 0;
-					}
-				}
-				if (i < MAX_FC_TARG) {
-					fcp->isp_ini_map[t] = dbidx + 1;
-					lp->ini_map_idx = t + 1;
-				} else {
-					isp_prt(isp, ISP_LOGWARN,
-					    "out of target ids");
-				}
-			}
 			isp_async(isp, ISPASYNC_DEV_ARRIVED, lp);
 			lp->new_roles = 0;
 			lp->new_portid = 0;
+			lp->reserved = 0;
+			lp->new_reserved = 0;
 			break;
 		case FC_PORTDB_STATE_CHANGED:
-			lp->portid = lp->new_portid;
-			lp->roles = lp->new_roles;
+/*
+ * XXXX FIX THIS
+ */
 			lp->state = FC_PORTDB_STATE_VALID;
-			if (lp->ini_map_idx) {
-				int t = lp->ini_map_idx - 1;
-				fcp->isp_ini_map[t] = dbidx + 1;
-			}
 			isp_async(isp, ISPASYNC_DEV_CHANGED, lp);
 			lp->new_roles = 0;
 			lp->new_portid = 0;
+			lp->reserved = 0;
+			lp->new_reserved = 0;
 			break;
 		case FC_PORTDB_STATE_PENDING_VALID:
 			lp->portid = lp->new_portid;
 			lp->roles = lp->new_roles;
-			lp->state = FC_PORTDB_STATE_VALID;
 			if (lp->ini_map_idx) {
 				int t = lp->ini_map_idx - 1;
 				fcp->isp_ini_map[t] = dbidx + 1;
 			}
+			lp->state = FC_PORTDB_STATE_VALID;
 			isp_async(isp, ISPASYNC_DEV_STAYED, lp);
 			if (dbidx != FL_ID) {
 				lp->new_roles = 0;
 				lp->new_portid = 0;
 			}
+			lp->reserved = 0;
+			lp->new_reserved = 0;
+			break;
+		case FC_PORTDB_STATE_ZOMBIE:
 			break;
 		default:
-			isp_prt(isp, ISP_LOGERR, "eh? state %d for idx %d",
+			isp_prt(isp, ISP_LOGWARN,
+			    "isp_scan_loop: state %d for idx %d",
 			    lp->state, dbidx);
 			isp_dump_portdb(isp);
 		}
@@ -2869,7 +2818,7 @@ isp_scan_loop(ispsoftc_t *isp)
 	fcparam *fcp = isp->isp_param;
 	int i;
 	isp_pdb_t pdb;
-	uint16_t dbidx, lim = 0;
+	uint16_t handle, lim = 0;
 
 	if (fcp->isp_fwstate < FW_READY ||
 	    fcp->isp_loopstate < LOOP_PDB_RCVD) {
@@ -2906,17 +2855,18 @@ isp_scan_loop(ispsoftc_t *isp)
 	}
 
 	fcp->isp_loopstate = LOOP_SCANNING_LOOP;
-	isp_prt(isp, ISP_LOGDEBUG0, "scanning loop 0..%d", lim-1);
+
+	isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0, "FC scan loop 0..%d", lim-1);
 
 
 	/*
 	 * Run through the list and get the port database info for each one.
 	 */
-	for (dbidx = 0; dbidx < lim; dbidx++) {
+	for (handle = 0; handle < lim; handle++) {
 		/*
 		 * But don't even try for ourselves...
 	 	 */
-		if (dbidx == fcp->isp_loopid) {
+		if (handle == fcp->isp_loopid) {
 			continue;
 		}
 
@@ -2925,7 +2875,7 @@ isp_scan_loop(ispsoftc_t *isp)
 		 * known to hang. This trick gets around that problem.
 		 */
 		if (IS_2100(isp) || IS_2200(isp)) {
-			uint64_t node_wwn = isp_get_portname(isp, dbidx, 1);
+			uint64_t node_wwn = isp_get_portname(isp, handle, 1);
 			if (fcp->isp_loopstate < LOOP_SCANNING_LOOP) {
 				return (-1);
 			}
@@ -2937,29 +2887,29 @@ isp_scan_loop(ispsoftc_t *isp)
 		/*
 		 * Get the port database entity for this index.
 		 */
-		if (isp_getpdb(isp, dbidx, &pdb, 1) != 0) {
+		if (isp_getpdb(isp, handle, &pdb, 1) != 0) {
 			if (fcp->isp_loopstate < LOOP_SCANNING_LOOP) {
-				isp_mark_portdb(isp, 1);
+				ISP_MARK_PORTDB(isp, 1);
 				return (-1);
 			}
 			continue;
 		}
 
 		if (fcp->isp_loopstate < LOOP_SCANNING_LOOP) {
-			isp_mark_portdb(isp, 1);
+			ISP_MARK_PORTDB(isp, 1);
 			return (-1);
 		}
 
 		/*
 		 * On *very* old 2100 firmware we would end up sometimes
 		 * with the firmware returning the port database entry
-		 * for something else. We used to restart locally, but
-		 * now we punt.
+		 * for something else. We used to restart this, but
+		 * now we just punt.
 		 */
-		if (IS_2100(isp) && pdb.handle != dbidx) {
+		if (IS_2100(isp) && pdb.handle != handle) {
 			isp_prt(isp, ISP_LOGWARN,
 			    "giving up on synchronizing the port database");
-			isp_mark_portdb(isp, 1);
+			ISP_MARK_PORTDB(isp, 1);
 			return (-1);
 		}
 
@@ -2978,8 +2928,7 @@ isp_scan_loop(ispsoftc_t *isp)
 		 * which shift on a loop.
 		 */
 		if (tmp.node_wwn == 0 || tmp.port_wwn == 0 || tmp.portid == 0) {
-			isp_prt(isp, ISP_LOGWARN,
-			    "bad pdb entry at loop %d", dbidx);
+			isp_prt(isp, ISP_LOGWARN, "bad pdb @ loop %d", handle);
 			isp_dump_portdb(isp);
 			continue;
 		}
@@ -3002,22 +2951,27 @@ isp_scan_loop(ispsoftc_t *isp)
 
 			/*
 			 * Okay- we've found a non-nil entry that matches.
-			 * Check to make sure it's probational.
+			 * Check to make sure it's probational or a zombie.
 			 */
-			if (lp->state != FC_PORTDB_STATE_PROBATIONAL) {
+			if (lp->state != FC_PORTDB_STATE_PROBATIONAL &&
+			    lp->state != FC_PORTDB_STATE_ZOMBIE) {
 				isp_prt(isp, ISP_LOGERR,
-				    "portdb entry %d not probational (0x%x)",
+				    "[%d] not probational/zombie (0x%x)",
 				    i, lp->state);
 				isp_dump_portdb(isp);
-				isp_mark_portdb(isp, 1);
+				ISP_MARK_PORTDB(isp, 1);
 				return (-1);
 			}
 
+			/*
+			 * Mark the device as something the f/w logs into
+			 * automatically.
+			 */
 			lp->autologin = 1;
+
 			/*
-			 * Check to make sure it's really the same
-			 * device and update the initiator map before
-			 * we mark it as pending valid.
+			 * Check to make see if really still the same
+			 * device. If it is, we mark it pending valid.
 			 */
 			if (lp->portid == tmp.portid &&
 			    lp->handle == tmp.handle &&
@@ -3025,12 +2979,15 @@ isp_scan_loop(ispsoftc_t *isp)
 				lp->new_portid = tmp.portid;
 				lp->new_roles = tmp.roles;
 				lp->state = FC_PORTDB_STATE_PENDING_VALID;
+				isp_prt(isp, ISP_LOGSANCFG,
+				    "Loop Port 0x%06x@0x%x Pending Valid",
+				    tmp.portid, tmp.handle);
 				break;
 			}
 		
 			/*
-			 * We can wipe out the old handle value here because
-			 * it's no longer valid.
+			 * We can wipe out the old handle value
+			 * here because it's no longer valid.
 			 */
 			lp->handle = tmp.handle;
 
@@ -3038,6 +2995,9 @@ isp_scan_loop(ispsoftc_t *isp)
 			 * Claim that this has changed and let somebody else
 			 * decide what to do.
 			 */
+			isp_prt(isp, ISP_LOGSANCFG,
+			    "Loop Port 0x%06x@0x%x changed",
+			    tmp.portid, tmp.handle);
 			lp->state = FC_PORTDB_STATE_CHANGED;
 			lp->new_portid = tmp.portid;
 			lp->new_roles = tmp.roles;
@@ -3061,23 +3021,23 @@ isp_scan_loop(ispsoftc_t *isp)
 			}
 		}
 		if (i == MAX_FC_TARG) {
-			isp_prt(isp, ISP_LOGERR,
-			    "could not find slot for new entry");
+			isp_prt(isp, ISP_LOGERR, "out of portdb entries");
 			continue;
 		}
 		lp = &fcp->portdb[i];
 
+		MEMZERO(lp, sizeof (fcportdb_t));
 		lp->autologin = 1;
 		lp->state = FC_PORTDB_STATE_NEW;
-		lp->portid = 0;
-		lp->roles = 0;
 		lp->new_portid = tmp.portid;
 		lp->new_roles = tmp.roles;
 		lp->handle = tmp.handle;
 		lp->port_wwn = tmp.port_wwn;
 		lp->node_wwn = tmp.node_wwn;
+		isp_prt(isp, ISP_LOGSANCFG,
+		    "Loop Port 0x%06x@0x%x is New Entry",
+		    tmp.portid, tmp.handle);
 	}
-
 	fcp->isp_loopstate = LOOP_LSCAN_DONE;
 	return (0);
 }
@@ -3258,7 +3218,7 @@ isp_scan_fabric(ispsoftc_t *isp)
 	int portidx, portlim, r;
 	sns_gid_ft_rsp_t *rs0, *rs1;
 
-	isp_prt(isp, ISP_LOGDEBUG0, "FC Scan Fabric");
+	isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0, "FC Scan Fabric");
 	if (fcp->isp_fwstate != FW_READY ||
 	    fcp->isp_loopstate < LOOP_LSCAN_DONE) {
 		return (-1);
@@ -3268,7 +3228,8 @@ isp_scan_fabric(ispsoftc_t *isp)
 	}
 	if (fcp->isp_topo != TOPO_FL_PORT && fcp->isp_topo != TOPO_F_PORT) {
 		fcp->isp_loopstate = LOOP_FSCAN_DONE;
-		isp_prt(isp, ISP_LOGDEBUG0, "FC Scan Fabric Done (no fabric)");
+		isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+		    "FC Scan Fabric Done (no fabric)");
 		return (0);
 	}
 
@@ -3303,7 +3264,7 @@ isp_scan_fabric(ispsoftc_t *isp)
 		int level;
 		if (rs1->snscb_cthdr.ct_reason == 9 &&
 		    rs1->snscb_cthdr.ct_explanation == 7) {
-			level = ISP_LOGDEBUG0;
+			level = ISP_LOGSANCFG|ISP_LOGDEBUG0;
 		} else {
 			level = ISP_LOGWARN;
 		}
@@ -3348,8 +3309,8 @@ isp_scan_fabric(ispsoftc_t *isp)
 		    "fabric too big for scratch area: increase ISP2100_SCRLEN");
 	}
 	portlim = portidx + 1;
-	isp_prt(isp, ISP_LOGDEBUG0, "got %d ports back from name server",
-	    portlim);
+	isp_prt(isp, ISP_LOGSANCFG,
+	    "got %d ports back from name server", portlim);
 
 	for (portidx = 0; portidx < portlim; portidx++) {
 		int npidx;
@@ -3373,7 +3334,7 @@ isp_scan_fabric(ispsoftc_t *isp)
 			rs1->snscb_ports[npidx].portid[0] = 0;
 			rs1->snscb_ports[npidx].portid[1] = 0;
 			rs1->snscb_ports[npidx].portid[2] = 0;
-			isp_prt(isp, ISP_LOGDEBUG0,
+			isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
 			    "removing duplicate PortID 0x%x entry from list",
 			    portid);
 		}
@@ -3406,7 +3367,7 @@ isp_scan_fabric(ispsoftc_t *isp)
 		    ((rs1->snscb_ports[portidx].portid[2]));
 
 		if (portid == 0) {
-			isp_prt(isp, ISP_LOGDEBUG0,
+			isp_prt(isp, ISP_LOGSANCFG,
 			    "skipping null PortID at idx %d", portidx);
 			continue;
 		}
@@ -3415,15 +3376,18 @@ isp_scan_fabric(ispsoftc_t *isp)
 		 * Skip ourselves...
 		 */
 		if (portid == fcp->isp_portid) {
-			isp_prt(isp, ISP_LOGDEBUG0,
+			isp_prt(isp, ISP_LOGSANCFG,
 			    "skip ourselves @ PortID 0x%06x", portid);
 			continue;
 		}
-		isp_prt(isp, ISP_LOGDEBUG0, "Fabric Port 0x%06x", portid);
+		isp_prt(isp, ISP_LOGSANCFG,
+		    "Checking Fabric Port 0x%06x", portid);
 
 		/*
 		 * We now search our Port Database for any
-		 * probational entries with this PortID.
+		 * probational entries with this PortID. We don't
+		 * look for zombies here- only probational
+		 * entries (we've already logged out of zombies).
 		 */
 		for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
 			lp = &fcp->portdb[dbidx];
@@ -3464,12 +3428,15 @@ isp_scan_fabric(ispsoftc_t *isp)
 			r = isp_getpdb(isp, lp->handle, &pdb, 0);
 			if (fcp->isp_loopstate != LOOP_SCANNING_FABRIC) {
 				FC_SCRATCH_RELEASE(isp);
-				isp_mark_portdb(isp, 1);
+				ISP_MARK_PORTDB(isp, 1);
 				return (-1);
 			}
 			if (r != 0) {
 				lp->new_portid = portid;
 				lp->state = FC_PORTDB_STATE_DEAD;
+				isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+				    "Fabric Port 0x%06x considered dead",
+				    portid);
 				continue;
 			}
 
@@ -3486,8 +3453,8 @@ isp_scan_fabric(ispsoftc_t *isp)
 			    pdb.portid != portid ||
 			    wwpn != lp->port_wwn ||
 			    wwnn != lp->node_wwn) {
-				isp_prt(isp, ISP_LOGDEBUG0, fconf, dbidx,
-				    pdb.handle, pdb.portid,
+				isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+				    fconf, dbidx, pdb.handle, pdb.portid,
 				    (uint32_t) (wwnn >> 32), (uint32_t) wwnn,
 				    (uint32_t) (wwpn >> 32), (uint32_t) wwpn,
 				    lp->handle, portid,
@@ -3510,7 +3477,7 @@ isp_scan_fabric(ispsoftc_t *isp)
 					if (fcp->isp_loopstate !=
 					    LOOP_SCANNING_FABRIC) {
 						FC_SCRATCH_RELEASE(isp);
-						isp_mark_portdb(isp, 1);
+						ISP_MARK_PORTDB(isp, 1);
 						return (-1);
 					}
 					continue;
@@ -3548,8 +3515,13 @@ isp_scan_fabric(ispsoftc_t *isp)
 			lp->new_roles = nr;
 			if (pdb.portid != lp->portid || nr != lp->roles ||
 			    handle_changed) {
+				isp_prt(isp, ISP_LOGSANCFG,
+				    "Fabric Port 0x%06x changed", portid);
 				lp->state = FC_PORTDB_STATE_CHANGED;
 			} else {
+				isp_prt(isp, ISP_LOGSANCFG,
+				    "Fabric Port 0x%06x Now Pending Valid",
+				    portid);
 				lp->state = FC_PORTDB_STATE_PENDING_VALID;
 			}
 			continue;
@@ -3557,49 +3529,54 @@ isp_scan_fabric(ispsoftc_t *isp)
 
 		/*
 		 * Ah- a new entry. Search the database again for all non-NIL
-		 * entries to make sure we never ever make a database entry
-		 * with the same port id.
+		 * entries to make sure we never ever make a new database entry
+		 * with the same port id. While we're at it, mark where the
+		 * last free entry was.
 		 */
-		for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
-			if (dbidx >= FL_ID && dbidx <= SNS_ID) {
+	
+		dbidx = MAX_FC_TARG;
+		for (lp = fcp->portdb; lp < &fcp->portdb[MAX_FC_TARG]; lp++) {
+			if (lp >= &fcp->portdb[FL_ID] &&
+			    lp <= &fcp->portdb[SNS_ID]) {
 				continue;
 			}
-			if (fcp->portdb[dbidx].state == FC_PORTDB_STATE_NIL) {
+			if (lp->state == FC_PORTDB_STATE_NIL) {
+				if (dbidx == MAX_FC_TARG) {
+					dbidx = lp - fcp->portdb;
+				}
 				continue;
 			}
-			if (fcp->portdb[dbidx].portid == portid) {
+			if (lp->state == FC_PORTDB_STATE_ZOMBIE) {
+				continue;
+			}
+			if (lp->portid == portid) {
 				break;
 			}
 		}
 
-		if (dbidx != MAX_FC_TARG) {
+		if (lp < &fcp->portdb[MAX_FC_TARG]) {
 			isp_prt(isp, ISP_LOGWARN,
 			    "PortID 0x%06x already at %d handle %d state %d",
-			    portid, dbidx, fcp->portdb[dbidx].handle,
-			    fcp->portdb[dbidx].state);
+			    portid, dbidx, lp->handle, lp->state);
 			continue;
 		}
 
 		/*
-		 * Find an empty database entry for it.
+		 * We should have the index of the first free entry seen.
 		 */
-		for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
-			if (dbidx >= FL_ID && dbidx <= SNS_ID) {
-				continue;
-			}
-			if (fcp->portdb[dbidx].state == FC_PORTDB_STATE_NIL) {
-				break;
-			}
-		}
-
 		if (dbidx == MAX_FC_TARG) {
 			isp_prt(isp, ISP_LOGERR,
-			    "port database too small to login fabric device"
-			    "- increase MAX_FC_TARG");
+			    "port database too small to login PortID 0x%06x"
+			    "- increase MAX_FC_TARG", portid);
 			continue;
 		}
 
 		/*
+		 * Otherwise, point to our new home.
+		 */
+		lp = &fcp->portdb[dbidx];
+
+		/*
 		 * Try to see if we are logged into this device,
 		 * and maybe log into it.
 		 *
@@ -3609,7 +3586,7 @@ isp_scan_fabric(ispsoftc_t *isp)
 		if (isp_login_device(isp, portid, &pdb, &oldhandle)) {
 			if (fcp->isp_loopstate != LOOP_SCANNING_FABRIC) {
 				FC_SCRATCH_RELEASE(isp);
-				isp_mark_portdb(isp, 1);
+				ISP_MARK_PORTDB(isp, 1);
 				return (-1);
 			}
 			continue;
@@ -3625,13 +3602,11 @@ isp_scan_fabric(ispsoftc_t *isp)
 		 * that we do not make more than one entry that has the same
 		 * WWNN/WWPN duple
 		 */
-		lp = &fcp->portdb[dbidx];
-
 		for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
-			if (fcp->portdb[dbidx].state == FC_PORTDB_STATE_NIL) {
+			if (dbidx >= FL_ID && dbidx <= SNS_ID) {
 				continue;
 			}
-			if (dbidx >= FL_ID && dbidx <= SNS_ID) {
+			if (fcp->portdb[dbidx].state == FC_PORTDB_STATE_NIL) {
 				continue;
 			}
 			if (fcp->portdb[dbidx].node_wwn == wwnn &&
@@ -3640,33 +3615,61 @@ isp_scan_fabric(ispsoftc_t *isp)
 			}
 		}
 
-		if (dbidx != MAX_FC_TARG) {
+		if (dbidx == MAX_FC_TARG) {
+			MEMZERO(lp, sizeof (fcportdb_t));
+			lp->handle = handle;
+			lp->node_wwn = wwnn;
+			lp->port_wwn = wwpn;
+			lp->new_portid = portid;
+			lp->new_roles = nr;
+			lp->state = FC_PORTDB_STATE_NEW;
+			isp_prt(isp, ISP_LOGSANCFG,
+			    "Fabric Port 0x%06x is New Entry", portid);
+			continue;
+		}
+
+    		if (fcp->portdb[dbidx].state != FC_PORTDB_STATE_ZOMBIE) {
 			isp_prt(isp, ISP_LOGWARN,
 			    "PortID 0x%x 0x%08x%08x/0x%08x%08x %ld already at "
-			    "idx %d", portid,
+			    "idx %d, state 0x%x", portid,
 			    (uint32_t) (wwnn >> 32), (uint32_t) wwnn,
 			    (uint32_t) (wwpn >> 32), (uint32_t) wwpn,
-			    (long) (lp - fcp->portdb), dbidx);
+			    (long) (lp - fcp->portdb), dbidx,
+			    fcp->portdb[dbidx].state);
 			continue;
 		}
 
+		/*
+		 * We found a zombie entry that matches us.
+		 * Revive it. We know that WWN and WWPN
+		 * are the same. For fabric devices, we
+		 * don't care that handle is different
+		 * as we assign that. If role or portid
+		 * are different, it maybe a changed device.
+		 */
+		lp = &fcp->portdb[dbidx];
 		lp->handle = handle;
-		lp->ini_map_idx = 0;
-		lp->node_wwn = wwnn;
-		lp->port_wwn = wwpn;
 		lp->new_portid = portid;
 		lp->new_roles = nr;
-		lp->state = FC_PORTDB_STATE_NEW;
+		if (lp->portid != portid || lp->roles != nr) {
+			isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+			    "Zombie Fabric Port 0x%06x Now Changed", portid);
+			lp->state = FC_PORTDB_STATE_CHANGED;
+		} else {
+			isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+			    "Zombie Fabric Port 0x%06x Now Pending Valid",
+			    portid);
+			lp->state = FC_PORTDB_STATE_PENDING_VALID;
+		}
 	}
 
-
 	FC_SCRATCH_RELEASE(isp);
 	if (fcp->isp_loopstate != LOOP_SCANNING_FABRIC) {
-		isp_mark_portdb(isp, 1);
+		ISP_MARK_PORTDB(isp, 1);
 		return (-1);
 	}
 	fcp->isp_loopstate = LOOP_FSCAN_DONE;
-	isp_prt(isp, ISP_LOGDEBUG0, "FC Scan Fabric Done");
+	isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0, "FC Scan Fabric Done");
 	return (0);
 }
 
@@ -3736,7 +3739,7 @@ isp_login_device(ispsoftc_t *isp, uint32_t portid, isp_pdb_t *p, uint16_t *ohp)
 	}
 
 	if (i == lim) {
-		isp_prt(isp, ISP_LOGINFO, "PLOGI 0x%06x failed", portid);
+		isp_prt(isp, ISP_LOGWARN, "PLOGI 0x%06x failed", portid);
 		return (-1);
 	}
 
@@ -3894,14 +3897,16 @@ isp_register_fc4_type_24xx(ispsoftc_t *isp)
 	FC_SCRATCH_RELEASE(isp);
 
 	if (ct->ct_cmd_resp == LS_RJT) {
-		isp_prt(isp, ISP_LOGWARN, "Register FC4 Type rejected");
+		isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+		    "Register FC4 Type rejected");
 		return (-1);
 	} else if (ct->ct_cmd_resp == LS_ACC) {
-		isp_prt(isp, ISP_LOGDEBUG0, "Register FC4 Type accepted");
+		isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+		    "Register FC4 Type accepted");
 		return(0);
 	} else {
 		isp_prt(isp, ISP_LOGWARN,
-		    "Register FC4 Type: %x", ct->ct_cmd_resp);
+		    "Register FC4 Type: 0x%x", ct->ct_cmd_resp);
 		return (-1);
 	}
 }
@@ -3950,7 +3955,7 @@ isp_start(XS_T *xs)
 	ispreq_t *reqp, *qep;
 	void *cdbp;
 	uint16_t *tptr;
-	int target, i;
+	int target, i, hdlidx = 0;
 
 	XS_INITERR(xs);
 	isp = XS_ISP(xs);
@@ -3996,6 +4001,9 @@ isp_start(XS_T *xs)
 	if (IS_FC(isp)) {
 		fcparam *fcp = isp->isp_param;
 
+		/*
+		 * Try again later.
+		 */
 		if (fcp->isp_fwstate != FW_READY ||
 		    fcp->isp_loopstate != LOOP_READY) {
 			return (CMD_RQLATER);
@@ -4006,14 +4014,21 @@ isp_start(XS_T *xs)
 			return (CMD_COMPLETE);
 		}
 
-		i = fcp->isp_ini_map[XS_TGT(xs)];
+		hdlidx = fcp->isp_ini_map[XS_TGT(xs)] - 1;
 		isp_prt(isp, ISP_LOGDEBUG1, "XS_TGT(xs)=%d- handle value %d",
-		    XS_TGT(xs), i);
-		if (i < 1 || i >= MAX_FC_TARG) {
+		    XS_TGT(xs), hdlidx);
+		if (hdlidx < 0 || hdlidx >= MAX_FC_TARG) {
+			XS_SETERR(xs, HBA_SELTIMEOUT);
+			return (CMD_COMPLETE);
+		}
+		if (fcp->portdb[hdlidx].state == FC_PORTDB_STATE_ZOMBIE) {
+			return (CMD_RQLATER);
+		}
+		if (fcp->portdb[hdlidx].state != FC_PORTDB_STATE_VALID) {
 			XS_SETERR(xs, HBA_SELTIMEOUT);
 			return (CMD_COMPLETE);
 		}
-		target = fcp->portdb[i - 1].handle;
+		target = fcp->portdb[hdlidx].handle;
 	}
 
 	/*
@@ -4141,9 +4156,7 @@ isp_start(XS_T *xs)
 	} else if (IS_24XX(isp)) {
 		fcportdb_t *lp;
 
-		i = FCPARAM(isp)->isp_ini_map[XS_TGT(xs)] - 1;
-		lp = &FCPARAM(isp)->portdb[i];
-
+		lp = &FCPARAM(isp)->portdb[hdlidx];
 		((ispreqt7_t *)reqp)->req_nphdl = target;
 		((ispreqt7_t *)reqp)->req_tidlo = lp->portid;
 		((ispreqt7_t *)reqp)->req_tidhi = lp->portid >> 16;
@@ -4637,6 +4650,8 @@ again:
 		isphdr_t *hp;
 		int buddaboom, etype, scsi_status, completion_status;
 		int req_status_flags, req_state_flags;
+		uint8_t *snsp, *resp;
+		uint32_t rlen, slen;
 		long resid;
 		uint16_t oop;
 
@@ -4801,13 +4816,37 @@ again:
 			XS_SETERR(xs, HBA_BOTCH);
 		}
 
+		resp = NULL;
+		rlen = 0;
+		snsp = NULL;
+		slen = 0;
+		if (IS_24XX(isp) && (scsi_status & RQCS_RV) != 0) {
+			resp = ((isp24xx_statusreq_t *)sp)->req_rsp_sense;
+			rlen = ((isp24xx_statusreq_t *)sp)->req_response_len;
+		} else if (IS_FC(isp) && (scsi_status & RQCS_RV) != 0) {
+			resp = sp->req_response;
+			rlen = sp->req_response_len;
+		}
 		if (IS_FC(isp) && (scsi_status & RQCS_SV) != 0) {
 			/*
 			 * Fibre Channel F/W doesn't say we got status
 			 * if there's Sense Data instead. I guess they
 			 * think it goes w/o saying.
 			 */
-			req_state_flags |= RQSF_GOT_STATUS;
+			req_state_flags |= RQSF_GOT_STATUS|RQSF_GOT_SENSE;
+			if (IS_24XX(isp)) {
+				snsp =
+				    ((isp24xx_statusreq_t *)sp)->req_rsp_sense;
+				snsp += rlen;
+				slen =
+				    ((isp24xx_statusreq_t *)sp)->req_sense_len;
+			} else {
+				snsp = sp->req_sense_data;
+				slen = sp->req_sense_len;
+			}
+		} else if (IS_SCSI(isp) && (req_state_flags & RQSF_GOT_SENSE)) {
+			snsp = sp->req_sense_data;
+			slen = sp->req_sense_len;
 		}
 		if (req_state_flags & RQSF_GOT_STATUS) {
 			*XS_STSP(xs) = scsi_status & 0xff;
@@ -4815,8 +4854,14 @@ again:
 
 		switch (etype) {
 		case RQSTYPE_RESPONSE:
-			/* XXX won't work for 24xx */
 			XS_SET_STATE_STAT(isp, xs, sp);
+			if (resp) {
+				isp_prt(isp, ISP_LOGWARN,
+				    "%d.%d FCP RESPONSE: 0x%x",
+				    XS_TGT(xs), XS_LUN(xs),
+				    resp[FCP_RSPNS_CODE_OFFSET]);
+				XS_SETERR(xs, HBA_BOTCH);
+			}
 			if (IS_24XX(isp)) {
 				isp_parse_status_24xx(isp,
 				    (isp24xx_statusreq_t *)sp, xs, &resid);
@@ -4829,11 +4874,6 @@ again:
 			}
 			if (IS_SCSI(isp)) {
 				XS_RESID(xs) = resid;
-				if ((req_state_flags & RQSF_GOT_STATUS) &&
-				    (*XS_STSP(xs) == SCSI_CHECK) &&
-				    (req_state_flags & RQSF_GOT_SENSE)) {
-					XS_SAVE_SENSE(xs, sp);
-				}
 				/*
 				 * A new synchronous rate was negotiated for
 				 * this target. Mark state such that we'll go
@@ -4855,13 +4895,9 @@ again:
 				} else {
 					XS_RESID(xs) = 0;
 				}
-				if ((req_state_flags & RQSF_GOT_STATUS) &&
-				    (*XS_STSP(xs) == SCSI_CHECK) &&
-				    (scsi_status & RQCS_SV)) {
-					XS_SAVE_SENSE(xs, sp);
-					/* solely for the benefit of debug */
-					req_state_flags |= RQSF_GOT_SENSE;
-				}
+			}
+			if (snsp && slen) {
+				XS_SAVE_SENSE(xs, snsp, slen);
 			}
 			isp_prt(isp, ISP_LOGDEBUG2,
 			   "asked for %ld got raw resid %ld settled for %ld",
@@ -5175,7 +5211,7 @@ isp_parse_async(ispsoftc_t *isp, uint16_t mbox)
 		FCPARAM(isp)->isp_fwstate = FW_CONFIG_WAIT;
 		FCPARAM(isp)->isp_loopstate = LOOP_LIP_RCVD;
 		isp->isp_sendmarker = 1;
-		isp_mark_portdb(isp, 1);
+		ISP_MARK_PORTDB(isp, 1);
 		isp_async(isp, ISPASYNC_LIP, NULL);
 #ifdef	ISP_TARGET_MODE
 		if (isp_target_async(isp, bus, mbox)) {
@@ -5211,7 +5247,7 @@ isp_parse_async(ispsoftc_t *isp, uint16_t mbox)
 		isp->isp_sendmarker = 1;
 		FCPARAM(isp)->isp_fwstate = FW_CONFIG_WAIT;
 		FCPARAM(isp)->isp_loopstate = LOOP_LIP_RCVD;
-		isp_mark_portdb(isp, 1);
+		ISP_MARK_PORTDB(isp, 1);
 		isp_async(isp, ISPASYNC_LOOP_UP, NULL);
 #ifdef	ISP_TARGET_MODE
 		if (isp_target_async(isp, bus, mbox)) {
@@ -5224,7 +5260,7 @@ isp_parse_async(ispsoftc_t *isp, uint16_t mbox)
 		isp->isp_sendmarker = 1;
 		FCPARAM(isp)->isp_fwstate = FW_CONFIG_WAIT;
 		FCPARAM(isp)->isp_loopstate = LOOP_NIL;
-		isp_mark_portdb(isp, 1);
+		ISP_MARK_PORTDB(isp, 1);
 		isp_async(isp, ISPASYNC_LOOP_DOWN, NULL);
 #ifdef	ISP_TARGET_MODE
 		if (isp_target_async(isp, bus, mbox)) {
@@ -5237,7 +5273,7 @@ isp_parse_async(ispsoftc_t *isp, uint16_t mbox)
 		isp->isp_sendmarker = 1;
 		FCPARAM(isp)->isp_fwstate = FW_CONFIG_WAIT;
 		FCPARAM(isp)->isp_loopstate = LOOP_NIL;
-		isp_mark_portdb(isp, 1);
+		ISP_MARK_PORTDB(isp, 1);
 		isp_async(isp, ISPASYNC_LOOP_RESET, NULL);
 #ifdef	ISP_TARGET_MODE
 		if (isp_target_async(isp, bus, mbox)) {
@@ -5249,7 +5285,7 @@ isp_parse_async(ispsoftc_t *isp, uint16_t mbox)
 	case ASYNC_PDB_CHANGED:
 		isp->isp_sendmarker = 1;
 		FCPARAM(isp)->isp_loopstate = LOOP_PDB_RCVD;
-		isp_mark_portdb(isp, 1);
+		ISP_MARK_PORTDB(isp, 1);
 		isp_async(isp, ISPASYNC_CHANGE_NOTIFY, ISPASYNC_CHANGE_PDB);
 		break;
 
@@ -5259,12 +5295,12 @@ isp_parse_async(ispsoftc_t *isp, uint16_t mbox)
 		} else {
 			FCPARAM(isp)->isp_loopstate = LOOP_PDB_RCVD;
 		}
-		isp_mark_portdb(isp, 1);
+		ISP_MARK_PORTDB(isp, 1);
 		isp_async(isp, ISPASYNC_CHANGE_NOTIFY, ISPASYNC_CHANGE_SNS);
 		break;
 
 	case ASYNC_PTPMODE:
-		isp_mark_portdb(isp, 1);
+		ISP_MARK_PORTDB(isp, 1);
 		isp->isp_sendmarker = 1;
 		FCPARAM(isp)->isp_fwstate = FW_CONFIG_WAIT;
 		FCPARAM(isp)->isp_loopstate = LOOP_LIP_RCVD;
@@ -5279,7 +5315,7 @@ isp_parse_async(ispsoftc_t *isp, uint16_t mbox)
 
 	case ASYNC_CONNMODE:
 		mbox = ISP_READ(isp, OUTMAILBOX1);
-		isp_mark_portdb(isp, 1);
+		ISP_MARK_PORTDB(isp, 1);
 		switch (mbox) {
 		case ISP_CONN_LOOP:
 			isp_prt(isp, ISP_LOGINFO,
@@ -5771,14 +5807,6 @@ isp_parse_status(ispsoftc_t *isp, ispstatusreq_t *sp, XS_T *xs, long *rp)
 			mbs.logval = MBLOGALL;
 			isp_mboxcmd_qnw(isp, &mbs, 1);
 		}
-
-		/*
-		 * Probably overkill.
-		 */
-		isp->isp_sendmarker = 1;
-		FCPARAM(isp)->isp_loopstate = LOOP_PDB_RCVD;
-		isp_mark_portdb(isp, 1);
-		isp_async(isp, ISPASYNC_CHANGE_NOTIFY, ISPASYNC_CHANGE_OTHER);
 		if (XS_NOERR(xs)) {
 			XS_SETERR(xs, HBA_SELTIMEOUT);
 		}
@@ -5929,14 +5957,6 @@ isp_parse_status_24xx(ispsoftc_t *isp, isp24xx_statusreq_t *sp,
 			mbs.logval = MBLOGALL;
 			isp_mboxcmd_qnw(isp, &mbs, 1);
 		}
-
-		/*
-		 * Probably overkill.
-		 */
-		isp->isp_sendmarker = 1;
-		FCPARAM(isp)->isp_loopstate = LOOP_PDB_RCVD;
-		isp_mark_portdb(isp, 1);
-		isp_async(isp, ISPASYNC_CHANGE_NOTIFY, ISPASYNC_CHANGE_OTHER);
 		if (XS_NOERR(xs)) {
 			XS_SETERR(xs, HBA_SELTIMEOUT);
 		}
@@ -7168,7 +7188,7 @@ isp_reinit(ispsoftc_t *isp)
 	uint32_t tmp;
 
 	if (IS_FC(isp)) {
-		isp_mark_portdb(isp, 0);
+		ISP_MARK_PORTDB(isp, 0);
 	}
 	isp_reset(isp);
 	if (isp->isp_state != ISP_RESETSTATE) {
diff --git a/sys/dev/isp/isp_freebsd.c b/sys/dev/isp/isp_freebsd.c
index 2750969..e6ac264 100644
--- a/sys/dev/isp/isp_freebsd.c
+++ b/sys/dev/isp/isp_freebsd.c
@@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$");
 #if	__FreeBSD_version >= 500000
 #include <sys/sysctl.h>
 #endif
+#include <cam/cam_periph.h>
 
 
 MODULE_VERSION(isp, 1);
@@ -48,13 +49,22 @@ MODULE_DEPEND(isp, cam, 1, 1, 1);
 int isp_announced = 0;
 int isp_fabric_hysteresis = 5;
 int isp_loop_down_limit = 300;	/* default loop down limit */
-int isp_quickboot_time = 5;	/* don't wait more than N secs for loop up */
+int isp_change_is_bad = 0;	/* "changed" devices are bad */
+int isp_quickboot_time = 15;	/* don't wait more than N secs for loop up */
+int isp_gone_device_time = 30;	/* grace time before reporting device lost */
+static const char *roles[4] = {
+    "(none)", "Target", "Initiator", "Target/Initiator"
+};
+static const char prom3[] =
+    "PortID 0x%06x Departed from Target %u because of %s";
 
+static void isp_freeze_loopdown(ispsoftc_t *, char *);
 static d_ioctl_t ispioctl;
 static void isp_intr_enable(void *);
 static void isp_cam_async(void *, uint32_t, struct cam_path *, void *);
 static void isp_poll(struct cam_sim *);
 static timeout_t isp_watchdog;
+static timeout_t isp_ldt;
 static void isp_kthread(void *);
 static void isp_action(struct cam_sim *, union ccb *);
 
@@ -171,7 +181,6 @@ isp_attach(ispsoftc_t *isp)
 	if (IS_FC(isp)) {
 		ISPLOCK_2_CAMLOCK(isp);
 #if __FreeBSD_version >= 500000  
-		/* XXX: LOCK VIOLATION */
 		cv_init(&isp->isp_osinfo.kthread_cv, "isp_kthread_cv");
 		if (kthread_create(isp_kthread, isp, &isp->isp_osinfo.kproc,
 		    RFHIGHPID, 0, "%s: fc_thrd",
@@ -189,6 +198,17 @@ isp_attach(ispsoftc_t *isp)
 			return;
 		}
 		CAMLOCK_2_ISPLOCK(isp);
+		/*
+		 * We start by being "loop down" if we have an initiator role
+		 */
+		if (isp->isp_role & ISP_ROLE_INITIATOR) {
+			isp_freeze_loopdown(isp, "isp_attach");
+			isp->isp_osinfo.ldt =
+			    timeout(isp_ldt, isp, isp_quickboot_time * hz);
+			isp->isp_osinfo.ldt_running = 1;
+			isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+			   "Starting Initial Loop Down Timer");
+		}
 	}
 
 
@@ -259,7 +279,7 @@ isp_attach(ispsoftc_t *isp)
 	isp_sysctl_update(isp);
 }
 
-static __inline void
+static void
 isp_freeze_loopdown(ispsoftc_t *isp, char *msg)
 {
 	if (isp->isp_osinfo.simqfrozen == 0) {
@@ -632,12 +652,9 @@ isp_sysctl_update(ispsoftc_t *isp)
 	struct sysctl_oid *tree = device_get_sysctl_tree(isp->isp_osinfo.dev);
 
 	if (IS_SCSI(isp)) {
-		isp->isp_osinfo.sysctl_info.spi.iid = DEFAULT_IID(isp);
-		SYSCTL_ADD_INT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO, "iid",
-		    CTLFLAG_RD, &isp->isp_osinfo.sysctl_info.spi.iid, 0,
-		    "Initiator ID");
 		return;
 	}
+
 	snprintf(isp->isp_osinfo.sysctl_info.fc.wwnn,
 	    sizeof (isp->isp_osinfo.sysctl_info.fc.wwnn), "0x%08x%08x",
 	    (uint32_t) (ISP_NODEWWN(isp) >> 32), (uint32_t) ISP_NODEWWN(isp));
@@ -649,9 +666,20 @@ isp_sysctl_update(ispsoftc_t *isp)
 	SYSCTL_ADD_STRING(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 	       "wwnn", CTLFLAG_RD, isp->isp_osinfo.sysctl_info.fc.wwnn, 0,
 	       "World Wide Node Name");
+
 	SYSCTL_ADD_STRING(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
 	       "wwpn", CTLFLAG_RD, isp->isp_osinfo.sysctl_info.fc.wwpn, 0,
 	       "World Wide Port Name");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+	    "loop_down_limit",
+	    CTLFLAG_RW, &isp->isp_osinfo.loop_down_limit, 0,
+	    "How long to wait for loop to come back up");
+
+	SYSCTL_ADD_UINT(ctx, SYSCTL_CHILDREN(tree), OID_AUTO,
+	    "gone_device_time",
+	    CTLFLAG_RW, &isp->isp_osinfo.gone_device_time, 0,
+	    "How long to wait for a device to reappear");
 }
 #endif
 
@@ -2016,11 +2044,11 @@ isp_poll(struct cam_sim *sim)
 }
 
 
-static void
-isp_watchdog(void *arg)
+static int isp_watchdog_work(ispsoftc_t *, XS_T *);
+
+static int
+isp_watchdog_work(ispsoftc_t *isp, XS_T *xs)
 {
-	XS_T *xs = arg;
-	ispsoftc_t *isp = XS_ISP(xs);
 	uint32_t handle;
 
 	/*
@@ -2038,14 +2066,14 @@ isp_watchdog(void *arg)
 			isp_prt(isp, ISP_LOGDEBUG1,
 			    "watchdog found done cmd (handle 0x%x)", handle);
 			ISP_UNLOCK(isp);
-			return;
+			return (1);;
 		}
 
 		if (XS_CMD_WDOG_P(xs)) {
 			isp_prt(isp, ISP_LOGDEBUG2,
 			    "recursive watchdog (handle 0x%x)", handle);
 			ISP_UNLOCK(isp);
-			return;
+			return (1);
 		}
 
 		XS_CMD_S_WDOG(xs);
@@ -2055,13 +2083,15 @@ isp_watchdog(void *arg)
 		if (XS_CMD_DONE_P(xs)) {
 			isp_prt(isp, ISP_LOGDEBUG2,
 			    "watchdog cleanup for handle 0x%x", handle);
+			ISPLOCK_2_CAMLOCK(isp);
 			xpt_done((union ccb *) xs);
+			CAMLOCK_2_ISPLOCK(isp);
 		} else if (XS_CMD_GRACE_P(xs)) {
 			/*
 			 * Make sure the command is *really* dead before we
 			 * release the handle (and DMA resources) for reuse.
 			 */
-			(void) isp_control(isp, ISPCTL_ABORT_CMD, arg);
+			(void) isp_control(isp, ISPCTL_ABORT_CMD, xs);
 
 			/*
 			 * After this point, the comamnd is really dead.
@@ -2075,26 +2105,195 @@ isp_watchdog(void *arg)
 			    "watchdog timeout for handle 0x%x", handle);
 			XS_SETERR(xs, CAM_CMD_TIMEOUT);
 			XS_CMD_C_WDOG(xs);
+			ISPLOCK_2_CAMLOCK(isp);
 			isp_done(xs);
+			CAMLOCK_2_ISPLOCK(isp);
 		} else {
 			XS_CMD_C_WDOG(xs);
 			xs->ccb_h.timeout_ch = timeout(isp_watchdog, xs, hz);
 			XS_CMD_S_GRACE(xs);
 			isp->isp_sendmarker |= 1 << XS_CHANNEL(xs);
 		}
+		ISP_UNLOCK(isp);
+		return (1);
+	}
+	ISP_UNLOCK(isp);
+	return (0);
+}
+
+static void
+isp_watchdog(void *arg)
+{
+	ispsoftc_t *isp;
+	XS_T *xs = arg;
+	for (isp = isplist; isp != NULL; isp = isp->isp_osinfo.next) {
+		if (isp_watchdog_work(isp, xs)) {
+			break;
+		}
+	}
+	if (isp == NULL) {
+		printf("isp_watchdog: nobody had %p active\n", arg);
+	}
+}
+
+
+#if __FreeBSD_version >= 500000  
+#define	isp_make_here(isp, tgt)	isp_announce(isp, tgt, AC_FOUND_DEVICE)
+#define	isp_make_gone(isp, tgt)	isp_announce(isp, tgt, AC_LOST_DEVICE)
+
+/*
+ * Support function for Announcement
+ */
+static void
+isp_announce(ispsoftc_t *isp, int tgt, int action)
+{
+	struct cam_path *tmppath;
+	ISPLOCK_2_CAMLOCK(isp);
+	if (xpt_create_path(&tmppath, NULL, cam_sim_path(isp->isp_sim), tgt,
+	    CAM_LUN_WILDCARD) == CAM_REQ_CMP) {
+		xpt_async(action, tmppath, NULL);
+		xpt_free_path(tmppath);
+	}
+	CAMLOCK_2_ISPLOCK(isp);
+}
+#else
+#define	isp_make_here(isp, tgt)	do { ; } while (0)
+#define	isp_make_gone(isp, tgt)	do { ; } while (0)
+#endif
+
+
+/*
+ * Gone Device Timer Function- when we have decided that a device has gone
+ * away, we wait a specific period of time prior to telling the OS it has
+ * gone away.
+ *
+ * This timer function fires once a second and then scans the port database
+ * for devices that are marked dead but still have a virtual target assigned.
+ * We decrement a counter for that port database entry, and when it hits zero,
+ * we tell the OS the device has gone away.
+ */
+static void
+isp_gdt(void *arg)
+{
+	ispsoftc_t *isp = arg;
+	fcportdb_t *lp;
+	int dbidx, tgt, more_to_do = 0;
+
+	isp_prt(isp, ISP_LOGDEBUG0, "GDT timer expired");
+	ISP_LOCK(isp);
+	for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
+		lp = &FCPARAM(isp)->portdb[dbidx];
+
+		if (lp->state != FC_PORTDB_STATE_ZOMBIE) {
+			continue;
+		}
+		if (lp->ini_map_idx == 0) {
+			continue;
+		}
+		if (lp->new_reserved == 0) {
+			continue;
+		}
+		lp->new_reserved -= 1;
+		if (lp->new_reserved != 0) {
+			more_to_do++;
+			continue;
+		}
+		tgt = lp->ini_map_idx - 1;
+		FCPARAM(isp)->isp_ini_map[tgt] = 0;
+		lp->ini_map_idx = 0;
+		lp->state = FC_PORTDB_STATE_NIL;
+		isp_prt(isp, ISP_LOGCONFIG, prom3, lp->portid, tgt,
+		    "Gone Device Timeout");
+		isp_make_gone(isp, tgt);
+	}
+	if (more_to_do) {
+		isp->isp_osinfo.gdt = timeout(isp_gdt, isp, hz);
 	} else {
-		isp_prt(isp, ISP_LOGDEBUG2, "watchdog with no command");
+		isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+		    "stopping Gone Device Timer");
+		isp->isp_osinfo.gdt_running = 0;
 	}
 	ISP_UNLOCK(isp);
 }
 
+/*
+ * Loop Down Timer Function- when loop goes down, a timer is started and
+ * and after it expires we come here and take all probational devices that
+ * the OS knows about and the tell the OS that they've gone away.
+ * 
+ * We don't clear the devices out of our port database because, when loop
+ * come back up, we have to do some actual cleanup with the chip at that
+ * point (implicit PLOGO, e.g., to get the chip's port database state right).
+ */
 static void
-isp_kthread(void *arg)
+isp_ldt(void *arg)
 {
 	ispsoftc_t *isp = arg;
-	int slp;
+	fcportdb_t *lp;
+	int dbidx, tgt;
+
+	isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0, "Loop Down Timer expired");
+	ISP_LOCK(isp);
+
+	/*
+	 * Notify to the OS all targets who we now consider have departed.
+	 */
+	for (dbidx = 0; dbidx < MAX_FC_TARG; dbidx++) {
+		lp = &FCPARAM(isp)->portdb[dbidx];
+
+		if (lp->state != FC_PORTDB_STATE_PROBATIONAL) {
+			continue;
+		}
+		if (lp->ini_map_idx == 0) {
+			continue;
+		}
+
+		/*
+		 * XXX: CLEAN UP AND COMPLETE ANY PENDING COMMANDS FIRST!
+		 */
+
+		/*
+		 * Mark that we've announced that this device is gone....
+		 */
+		lp->reserved = 1;
+
+		/*
+		 * but *don't* change the state of the entry. Just clear
+		 * any target id stuff and announce to CAM that the
+		 * device is gone. This way any necessary PLOGO stuff
+		 * will happen when loop comes back up.
+		 */
 
+		tgt = lp->ini_map_idx - 1;
+		FCPARAM(isp)->isp_ini_map[tgt] = 0;
+		lp->ini_map_idx = 0;
+		isp_prt(isp, ISP_LOGCONFIG, prom3, lp->portid, tgt,
+		    "Loop Down Timeout");
+		isp_make_gone(isp, tgt);
+	}
 
+	/*
+	 * The loop down timer has expired. Wake up the kthread
+	 * to notice that fact (or make it false).
+	 */
+	isp->isp_osinfo.loop_down_time = isp->isp_osinfo.loop_down_limit+1;
+#if __FreeBSD_version < 500000  
+	wakeup(&isp->isp_osinfo.kproc);
+#else
+#ifdef	ISP_SMPLOCK
+	cv_signal(&isp->isp_osinfo.kthread_cv);
+#else
+	wakeup(&isp->isp_osinfo.kthread_cv);
+#endif
+#endif
+	ISP_UNLOCK(isp);
+}
+
+static void
+isp_kthread(void *arg)
+{
+	ispsoftc_t *isp = arg;
+	int slp = 0;
 #if __FreeBSD_version < 500000  
         int s;
 
@@ -2111,19 +2310,25 @@ isp_kthread(void *arg)
 	 * gotten good fibre channel state.
 	 */
 	for (;;) {
-		int wasfrozen, lb;
+		int wasfrozen, lb, lim;
 
-		isp_prt(isp, ISP_LOGDEBUG0, "kthread: checking FC state");
+		isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+		    "isp_kthread: checking FC state");
+		isp->isp_osinfo.mbox_sleep_ok = 1;
 		lb = isp_fc_runstate(isp, 250000);
+		isp->isp_osinfo.mbox_sleep_ok = 0;
 		if (lb) {
-			unsigned int inc = 1;
+			/*
+			 * Increment loop down time by the last sleep interval
+			 */
+			isp->isp_osinfo.loop_down_time += slp;
 
 			if (lb < 0) {
-				isp_prt(isp, ISP_LOGDEBUG0,
+				isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
 				    "kthread: FC loop not up (down count %d)",
 				    isp->isp_osinfo.loop_down_time);
 			} else {
-				isp_prt(isp, ISP_LOGDEBUG0,
+				isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
 				    "kthread: FC got to %d (down count %d)",
 				    lb, isp->isp_osinfo.loop_down_time);
 			}
@@ -2131,33 +2336,33 @@ isp_kthread(void *arg)
 
 			/*
 			 * If we've never seen loop up and we've waited longer
-			 * than quickboot time, give up and go to sleep until
-			 * loop comes up. Otherwise, increment the loop down
-			 * time and figure out how long to sleep to the next
-			 * check. 
+			 * than quickboot time, or we've seen loop up but we've
+			 * waited longer than loop_down_limit, give up and go
+			 * to sleep until loop comes up.
 			 */
-			if (FCPARAM(isp)->loop_seen_once == 0 &&
-			    isp->isp_osinfo.loop_down_time >=
-			    isp_quickboot_time) {
-				isp->isp_osinfo.loop_down_time = 0xffff;
-				slp = 0;
-			} else if (isp->isp_osinfo.loop_down_time > 30) {
-				inc = 30;
-				slp = 30 * hz;
-			} else if (isp->isp_osinfo.loop_down_time > 1) {
-				slp = hz;
+			if (FCPARAM(isp)->loop_seen_once == 0) {
+				lim = isp_quickboot_time;
 			} else {
-				slp = 1;
+				lim = isp->isp_osinfo.loop_down_limit;
 			}
-
-			inc += isp->isp_osinfo.loop_down_time;
-			if (inc < 0xffff) {
-				isp->isp_osinfo.loop_down_time = inc;
+			if (isp->isp_osinfo.loop_down_time >= lim) {
+				isp_freeze_loopdown(isp, "loop limit hit");
+				slp = 0;
+			} else if (isp->isp_osinfo.loop_down_time < 10) {
+				slp = 1;
+			} else if (isp->isp_osinfo.loop_down_time < 30) {
+				slp = 5;
+			} else if (isp->isp_osinfo.loop_down_time < 60) {
+				slp = 10;
+			} else if (isp->isp_osinfo.loop_down_time < 120) {
+				slp = 20;
 			} else {
-				isp->isp_osinfo.loop_down_time = 0xfffe;
+				slp = 30;
 			}
+
 		} else {
-			isp_prt(isp, ISP_LOGDEBUG0, "kthread: FC state OK");
+			isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+			    "isp_kthread: FC state OK");
 			isp->isp_osinfo.loop_down_time = 0;
 			slp = 0;
 		}
@@ -2171,18 +2376,24 @@ isp_kthread(void *arg)
 		wasfrozen = isp->isp_osinfo.simqfrozen & SIMQFRZ_LOOPDOWN;
 		isp->isp_osinfo.simqfrozen &= ~SIMQFRZ_LOOPDOWN;
 		if (wasfrozen && isp->isp_osinfo.simqfrozen == 0) {
-			isp_prt(isp, ISP_LOGDEBUG0, "kthread: releasing simq");
+			isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+			    "isp_kthread: releasing simq");
 			ISPLOCK_2_CAMLOCK(isp);
 			xpt_release_simq(isp->isp_sim, 1);
 			CAMLOCK_2_ISPLOCK(isp);
 		}
+		isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+		    "isp_kthread: sleep time %d", slp);
 #if __FreeBSD_version < 500000  
-		tsleep(&isp->isp_osinfo.kproc, PRIBIO, "ispf", slp);
+		tsleep(&isp->isp_osinfo.kproc, PRIBIO, "ispf",
+		    slp * hz);
 #else
 #ifdef	ISP_SMPLOCK
-		cv_timed_wait(&isp->isp_osinfo.kthread_cv, &isp->isp_lock, slp);
+		cv_timed_wait(&isp->isp_osinfo.kthread_cv, &isp->isp_lock,
+		    slp * hz);
 #else
-		(void) tsleep(&isp->isp_osinfo.kthread_cv, PRIBIO, "ispf", slp);
+		(void) tsleep(&isp->isp_osinfo.kthread_cv, PRIBIO, "ispf",
+		    slp * hz);
 #endif
 #endif
 		/*
@@ -2193,6 +2404,9 @@ isp_kthread(void *arg)
 		 * to settle.
 		 */
 		if (slp == 0 && isp->isp_osinfo.hysteresis) {
+			isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+			    "isp_kthread: sleep hysteresis tick time %d",
+			    isp->isp_osinfo.hysteresis * hz);
 			(void) tsleep(&isp_fabric_hysteresis, PRIBIO, "ispT",
 			    (isp->isp_osinfo.hysteresis * hz));
 		}
@@ -2202,7 +2416,7 @@ isp_kthread(void *arg)
 static void
 isp_action(struct cam_sim *sim, union ccb *ccb)
 {
-	int bus, tgt, error;
+	int bus, tgt, error, lim;
 	ispsoftc_t *isp;
 	struct ccb_trans_settings *cts;
 
@@ -2262,59 +2476,63 @@ isp_action(struct cam_sim *sim, union ccb *ccb)
 		error = isp_start((XS_T *) ccb);
 		switch (error) {
 		case CMD_QUEUED:
+			XS_CMD_S_CLEAR(ccb);
+			ISPLOCK_2_CAMLOCK(isp);
 			ccb->ccb_h.status |= CAM_SIM_QUEUED;
 			if (ccb->ccb_h.timeout != CAM_TIME_INFINITY) {
-				uint64_t ticks = (uint64_t) hz;
-				if (ccb->ccb_h.timeout == CAM_TIME_DEFAULT)
-					ticks = 60 * 1000 * ticks;
-				else
-					ticks = ccb->ccb_h.timeout * hz;
-				ticks = ((ticks + 999) / 1000) + hz + hz;
-				if (ticks >= 0x80000000) {
-					isp_prt(isp, ISP_LOGERR,
-					    "timeout overflow");
-					ticks = 0x7fffffff;
+				int ms = ccb->ccb_h.timeout;
+				if (ms == CAM_TIME_DEFAULT) {
+					ms = 60*1000;
 				}
-				ccb->ccb_h.timeout_ch = timeout(isp_watchdog,
-				    (caddr_t)ccb, (int)ticks);
+				ccb->ccb_h.timeout_ch =
+				    timeout(isp_watchdog, ccb, isp_mstohz(ms));
 			} else {
 				callout_handle_init(&ccb->ccb_h.timeout_ch);
 			}
-			ISPLOCK_2_CAMLOCK(isp);
 			break;
 		case CMD_RQLATER:
 			/*
 			 * This can only happen for Fibre Channel
 			 */
 			KASSERT((IS_FC(isp)), ("CMD_RQLATER for FC only"));
+
 			/*
-			 * If we've exceeded the loop down limit start
-			 * failing commands.
+			 * Handle initial and subsequent loop down cases
 			 */
-			if (isp->isp_osinfo.loop_down_time > 
-			    isp->isp_osinfo.loop_down_limit) {
+			if (FCPARAM(isp)->loop_seen_once == 0) {
+				lim = isp_quickboot_time;
+			} else {
+				lim = isp->isp_osinfo.loop_down_limit;
+			}
+			if (isp->isp_osinfo.loop_down_time >= lim) {
+				isp_prt(isp, ISP_LOGDEBUG0,
+				    "%d.%d downtime (%d) > lim (%d)",
+				    XS_TGT(ccb), XS_LUN(ccb),
+				    isp->isp_osinfo.loop_down_time, lim);
+				ccb->ccb_h.status =
+				    CAM_SEL_TIMEOUT|CAM_DEV_QFRZN;
+				xpt_freeze_devq(ccb->ccb_h.path, 1);
 				ISPLOCK_2_CAMLOCK(isp);
-				XS_SETERR(ccb, CAM_SEL_TIMEOUT);
 				xpt_done(ccb);
 				break;
 			}
-#if __FreeBSD_version < 500000  
-			wakeup(&isp->isp_osinfo.kproc);
-#else
-#ifdef	ISP_SMPLOCK
-			cv_signal(&isp->isp_osinfo.kthread_cv);
-#else
-			wakeup(&isp->isp_osinfo.kthread_cv);
-#endif
-#endif
-			isp_freeze_loopdown(isp, "isp_action(RQLATER)");
-			XS_SETERR(ccb, CAM_REQUEUE_REQ);
+			isp_prt(isp, ISP_LOGDEBUG0,
+			    "%d.%d retry later", XS_TGT(ccb), XS_LUN(ccb));
+			/*
+			 * Otherwise, retry in a while.
+			 */
 			ISPLOCK_2_CAMLOCK(isp);
+			cam_freeze_devq(ccb->ccb_h.path);
+			cam_release_devq(ccb->ccb_h.path,
+			    RELSIM_RELEASE_AFTER_TIMEOUT, 0, 1000, 0);
+			XS_SETERR(ccb, CAM_REQUEUE_REQ);
 			xpt_done(ccb);
 			break;
 		case CMD_EAGAIN:
-			XS_SETERR(ccb, CAM_REQUEUE_REQ);
 			ISPLOCK_2_CAMLOCK(isp);
+			cam_freeze_devq(ccb->ccb_h.path);
+			cam_release_devq(ccb->ccb_h.path,
+			    RELSIM_RELEASE_AFTER_TIMEOUT, 0, 250, 0);
 			xpt_done(ccb);
 			break;
 		case CMD_COMPLETE:
@@ -2322,12 +2540,12 @@ isp_action(struct cam_sim *sim, union ccb *ccb)
 			ISPLOCK_2_CAMLOCK(isp);
 			break;
 		default:
+			ISPLOCK_2_CAMLOCK(isp);
 			isp_prt(isp, ISP_LOGERR,
 			    "What's this? 0x%x at %d in file %s",
 			    error, __LINE__, __FILE__);
 			XS_SETERR(ccb, CAM_REQ_CMP_ERR);
 			xpt_done(ccb);
-			ISPLOCK_2_CAMLOCK(isp);
 		}
 		break;
 
@@ -2739,6 +2957,7 @@ isp_action(struct cam_sim *sim, union ccb *ccb)
 }
 
 #define	ISPDDB	(CAM_DEBUG_INFO|CAM_DEBUG_TRACE|CAM_DEBUG_CDB)
+
 void
 isp_done(struct ccb_scsiio *sccb)
 {
@@ -2760,13 +2979,13 @@ isp_done(struct ccb_scsiio *sccb)
 
 	sccb->ccb_h.status &= ~CAM_SIM_QUEUED;
 	if ((sccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
+		isp_prt(isp, ISP_LOGDEBUG0,
+		    "target %d lun %d CAM status 0x%x SCSI status 0x%x",
+		    XS_TGT(sccb), XS_LUN(sccb), sccb->ccb_h.status,
+		    sccb->scsi_status);
 		if ((sccb->ccb_h.status & CAM_DEV_QFRZN) == 0) {
 			sccb->ccb_h.status |= CAM_DEV_QFRZN;
 			xpt_freeze_devq(sccb->ccb_h.path, 1);
-			isp_prt(isp, ISP_LOGDEBUG0,
-			    "freeze devq %d.%d cam sts %x scsi sts %x",
-			    sccb->ccb_h.target_id, sccb->ccb_h.target_lun,
-			    sccb->ccb_h.status, sccb->scsi_status);
 		}
 	}
 
@@ -2779,7 +2998,7 @@ isp_done(struct ccb_scsiio *sccb)
 
 	XS_CMD_S_DONE(sccb);
 	if (XS_CMD_WDOG_P(sccb) == 0) {
-		untimeout(isp_watchdog, (caddr_t)sccb, sccb->ccb_h.timeout_ch);
+		untimeout(isp_watchdog, sccb, sccb->ccb_h.timeout_ch);
 		if (XS_CMD_GRACE_P(sccb)) {
 			isp_prt(isp, ISP_LOGDEBUG2,
 			    "finished command on borrowed time");
@@ -2795,15 +3014,13 @@ int
 isp_async(ispsoftc_t *isp, ispasync_t cmd, void *arg)
 {
 	int bus, rv = 0;
-	static const char *roles[4] = {
-	    "(none)", "Target", "Initiator", "Target/Initiator"
-	};
 	static const char prom[] =
 	    "PortID 0x%06x handle 0x%x role %s %s\n"
-            "      WWNN 0x%08x%08x WWPN 0x%08x%08x";
+	    "      WWNN 0x%08x%08x WWPN 0x%08x%08x";
 	static const char prom2[] =
 	    "PortID 0x%06x handle 0x%x role %s %s tgt %u\n"
-            "      WWNN 0x%08x%08x WWPN 0x%08x%08x";
+	    "      WWNN 0x%08x%08x WWPN 0x%08x%08x";
+	char *msg = NULL;
 	target_id_t tgt;
 	fcportdb_t *lp;
 	struct cam_path *tmppath;
@@ -2890,22 +3107,30 @@ isp_async(ispsoftc_t *isp, ispasync_t cmd, void *arg)
 		}
 		break;
 	case ISPASYNC_LIP:
-		if (isp->isp_path) {
-			isp_freeze_loopdown(isp, "ISPASYNC_LIP");
+		if (msg == NULL) {
+			msg = "LIP Received";
 		}
-		isp_prt(isp, ISP_LOGINFO, "LIP Received");
-		break;
+		/* FALLTHROUGH */
 	case ISPASYNC_LOOP_RESET:
-		if (isp->isp_path) {
-			isp_freeze_loopdown(isp, "ISPASYNC_LOOP_RESET");
+		if (msg == NULL) {
+			msg = "LOOP Reset";
 		}
-		isp_prt(isp, ISP_LOGINFO, "Loop Reset Received");
-		break;
+		/* FALLTHROUGH */
 	case ISPASYNC_LOOP_DOWN:
+		if (msg == NULL) {
+			msg = "LOOP Down";
+		}
 		if (isp->isp_path) {
-			isp_freeze_loopdown(isp, "ISPASYNC_LOOP_DOWN");
+			isp_freeze_loopdown(isp, msg);
 		}
-		isp_prt(isp, ISP_LOGINFO, "Loop DOWN");
+		if (isp->isp_osinfo.ldt_running == 0) {
+			isp->isp_osinfo.ldt = timeout(isp_ldt, isp,
+			    isp->isp_osinfo.loop_down_limit * hz);
+			isp->isp_osinfo.ldt_running = 1;
+			isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+			   "starting Loop Down Timer");
+		}
+		isp_prt(isp, ISP_LOGINFO, msg);
 		break;
 	case ISPASYNC_LOOP_UP:
 		/*
@@ -2918,32 +3143,42 @@ isp_async(ispsoftc_t *isp, ispasync_t cmd, void *arg)
 		break;
 	case ISPASYNC_DEV_ARRIVED:
 		lp = arg;
-
+		lp->reserved = 0;
+		if ((isp->isp_role & ISP_ROLE_INITIATOR) &&
+		    (lp->roles & (SVC3_TGT_ROLE >> SVC3_ROLE_SHIFT))) {
+			int dbidx = lp - FCPARAM(isp)->portdb;
+			int i;
+
+			for (i = 0; i < MAX_FC_TARG; i++) {
+				if (i >= FL_ID && i <= SNS_ID) {
+					continue;
+				}
+				if (FCPARAM(isp)->isp_ini_map[i] == 0) {
+					break;
+				}
+			}
+			if (i < MAX_FC_TARG) {
+				FCPARAM(isp)->isp_ini_map[i] = dbidx + 1;
+				lp->ini_map_idx = i + 1;
+			} else {
+				isp_prt(isp, ISP_LOGWARN, "out of target ids");
+				isp_dump_portdb(isp);
+			}
+		}
 		if (lp->ini_map_idx) {
 			tgt = lp->ini_map_idx - 1;
 			isp_prt(isp, ISP_LOGCONFIG, prom2,
 			    lp->portid, lp->handle,
-		            roles[lp->roles & 0x3], "arrived at", tgt,
+		            roles[lp->roles], "arrived at", tgt,
 		    	    (uint32_t) (lp->node_wwn >> 32),
 			    (uint32_t) lp->node_wwn,
 		    	    (uint32_t) (lp->port_wwn >> 32),
 			    (uint32_t) lp->port_wwn);
-#if __FreeBSD_version >= 500000  
-			ISPLOCK_2_CAMLOCK(isp);
-			if (xpt_create_path(&tmppath, NULL,
-			    cam_sim_path(isp->isp_sim), tgt,
-			    CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
-				CAMLOCK_2_ISPLOCK(isp);
-				break;
-			}
-			xpt_async(AC_FOUND_DEVICE, tmppath, NULL);
-			xpt_free_path(tmppath);
-			CAMLOCK_2_ISPLOCK(isp);
-#endif
+			isp_make_here(isp, tgt);
 		} else {
 			isp_prt(isp, ISP_LOGCONFIG, prom,
 			    lp->portid, lp->handle,
-		            roles[lp->roles & 0x3], "arrived",
+		            roles[lp->roles], "arrived",
 		    	    (uint32_t) (lp->node_wwn >> 32),
 			    (uint32_t) lp->node_wwn,
 		    	    (uint32_t) (lp->port_wwn >> 32),
@@ -2952,23 +3187,49 @@ isp_async(ispsoftc_t *isp, ispasync_t cmd, void *arg)
 		break;
 	case ISPASYNC_DEV_CHANGED:
 		lp = arg;
-		if (lp->ini_map_idx) {
-			tgt = lp->ini_map_idx - 1;
-			isp_prt(isp, ISP_LOGCONFIG, prom2,
-			    lp->portid, lp->handle,
-		    	    roles[lp->roles & 0x3], "changed at", tgt,
-			    (uint32_t) (lp->node_wwn >> 32),
-			    (uint32_t) lp->node_wwn,
-		    	    (uint32_t) (lp->port_wwn >> 32),
-			    (uint32_t) lp->port_wwn);
+		if (isp_change_is_bad) {
+			lp->state = FC_PORTDB_STATE_NIL;
+			if (lp->ini_map_idx) {
+				tgt = lp->ini_map_idx - 1;
+				FCPARAM(isp)->isp_ini_map[tgt] = 0;
+				lp->ini_map_idx = 0;
+				isp_prt(isp, ISP_LOGCONFIG, prom3,
+				    lp->portid, tgt, "change is bad");
+				isp_make_gone(isp, tgt);
+			} else {
+				isp_prt(isp, ISP_LOGCONFIG, prom,
+				    lp->portid, lp->handle,
+				    roles[lp->roles],
+				    "changed and departed",
+				    (uint32_t) (lp->node_wwn >> 32),
+				    (uint32_t) lp->node_wwn,
+				    (uint32_t) (lp->port_wwn >> 32),
+				    (uint32_t) lp->port_wwn);
+			}
 		} else {
-			isp_prt(isp, ISP_LOGCONFIG, prom,
-			    lp->portid, lp->handle,
-		    	    roles[lp->roles & 0x3], "changed",
-			    (uint32_t) (lp->node_wwn >> 32),
-			    (uint32_t) lp->node_wwn,
-		    	    (uint32_t) (lp->port_wwn >> 32),
-			    (uint32_t) lp->port_wwn);
+			lp->portid = lp->new_portid;
+			lp->roles = lp->new_roles;
+			if (lp->ini_map_idx) {
+				int t = lp->ini_map_idx - 1;
+				FCPARAM(isp)->isp_ini_map[t] =
+				    (lp - FCPARAM(isp)->portdb) + 1;
+				tgt = lp->ini_map_idx - 1;
+				isp_prt(isp, ISP_LOGCONFIG, prom2,
+				    lp->portid, lp->handle,
+				    roles[lp->roles], "changed at", tgt,
+				    (uint32_t) (lp->node_wwn >> 32),
+				    (uint32_t) lp->node_wwn,
+				    (uint32_t) (lp->port_wwn >> 32),
+				    (uint32_t) lp->port_wwn);
+			} else {
+				isp_prt(isp, ISP_LOGCONFIG, prom,
+				    lp->portid, lp->handle,
+				    roles[lp->roles], "changed",
+				    (uint32_t) (lp->node_wwn >> 32),
+				    (uint32_t) lp->node_wwn,
+				    (uint32_t) (lp->port_wwn >> 32),
+				    (uint32_t) lp->port_wwn);
+			}
 		}
 		break;
 	case ISPASYNC_DEV_STAYED:
@@ -2977,7 +3238,7 @@ isp_async(ispsoftc_t *isp, ispasync_t cmd, void *arg)
 			tgt = lp->ini_map_idx - 1;
 			isp_prt(isp, ISP_LOGCONFIG, prom2,
 			    lp->portid, lp->handle,
-		    	    roles[lp->roles & 0x3], "stayed at", tgt,
+		    	    roles[lp->roles], "stayed at", tgt,
 			    (uint32_t) (lp->node_wwn >> 32),
 			    (uint32_t) lp->node_wwn,
 		    	    (uint32_t) (lp->port_wwn >> 32),
@@ -2985,7 +3246,7 @@ isp_async(ispsoftc_t *isp, ispasync_t cmd, void *arg)
 		} else {
 			isp_prt(isp, ISP_LOGCONFIG, prom,
 			    lp->portid, lp->handle,
-		    	    roles[lp->roles & 0x3], "stayed",
+		    	    roles[lp->roles], "stayed",
 			    (uint32_t) (lp->node_wwn >> 32),
 			    (uint32_t) lp->node_wwn,
 		    	    (uint32_t) (lp->port_wwn >> 32),
@@ -2994,34 +3255,39 @@ isp_async(ispsoftc_t *isp, ispasync_t cmd, void *arg)
 		break;
 	case ISPASYNC_DEV_GONE:
 		lp = arg;
-		if (lp->ini_map_idx) {
+		/*
+		 * If this has a virtual target and we haven't marked it
+		 * that we're going to have isp_gdt tell the OS it's gone,
+		 * set the isp_gdt timer running on it.
+		 *
+		 * If it isn't marked that isp_gdt is going to get rid of it,
+		 * announce that it's gone.
+		 */
+		if (lp->ini_map_idx && lp->reserved == 0) {
+			lp->reserved = 1;
+			lp->new_reserved = isp->isp_osinfo.gone_device_time;
+			lp->state = FC_PORTDB_STATE_ZOMBIE;
+			if (isp->isp_osinfo.gdt_running == 0) {
+				isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+				    "starting Gone Device Timer");
+				isp->isp_osinfo.gdt = timeout(isp_gdt, isp, hz);
+				isp->isp_osinfo.gdt_running = 1;
+			}
 			tgt = lp->ini_map_idx - 1;
 			isp_prt(isp, ISP_LOGCONFIG, prom2,
 			    lp->portid, lp->handle,
-		            roles[lp->roles & 0x3], "departed from", tgt,
+		            roles[lp->roles], "gone zombie at", tgt,
 		    	    (uint32_t) (lp->node_wwn >> 32),
 			    (uint32_t) lp->node_wwn,
 		    	    (uint32_t) (lp->port_wwn >> 32),
 			    (uint32_t) lp->port_wwn);
-#if __FreeBSD_version >= 500000  
-			ISPLOCK_2_CAMLOCK(isp);
-			if (xpt_create_path(&tmppath, NULL,
-			    cam_sim_path(isp->isp_sim), tgt,
-			    CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
-				CAMLOCK_2_ISPLOCK(isp);
-				break;
-			}
-			xpt_async(AC_LOST_DEVICE, tmppath, NULL);
-			xpt_free_path(tmppath);
-			CAMLOCK_2_ISPLOCK(isp);
-#endif
-		} else {
+		} else if (lp->reserved == 0) {
 			isp_prt(isp, ISP_LOGCONFIG, prom,
 			    lp->portid, lp->handle,
-		            roles[lp->roles & 0x3], "departed",
-		    	    (uint32_t) (lp->node_wwn >> 32),
+			    roles[lp->roles], "departed",
+			    (uint32_t) (lp->node_wwn >> 32),
 			    (uint32_t) lp->node_wwn,
-		    	    (uint32_t) (lp->port_wwn >> 32),
+			    (uint32_t) (lp->port_wwn >> 32),
 			    (uint32_t) lp->port_wwn);
 		}
 		break;
@@ -3035,6 +3301,16 @@ isp_async(ispsoftc_t *isp, ispasync_t cmd, void *arg)
 		} else {
 			msg = "Other Change Notify";
 		}
+		/*
+		 * If the loop down timer is running, cancel it.
+		 */
+		if (isp->isp_osinfo.ldt_running) {
+			isp_prt(isp, ISP_LOGSANCFG|ISP_LOGDEBUG0,
+			   "Stopping Loop Down Timer");
+			isp->isp_osinfo.ldt_running = 0;
+			untimeout(isp_ldt, isp, isp->isp_osinfo.ldt);
+			callout_handle_init(&isp->isp_osinfo.ldt);
+		}
 		isp_prt(isp, ISP_LOGINFO, msg);
 		isp_freeze_loopdown(isp, msg);
 #if __FreeBSD_version < 500000  
@@ -3154,3 +3430,106 @@ isp_prt(ispsoftc_t *isp, int level, const char *fmt, ...)
 	va_end(ap);
 	printf("\n");
 }
+
+uint64_t
+isp_nanotime_sub(struct timespec *b, struct timespec *a)
+{
+	uint64_t elapsed;
+	struct timespec x = *b;
+	timespecsub(&x, a);
+	elapsed = GET_NANOSEC(&x);
+	if (elapsed == 0)
+		elapsed++;
+	return (elapsed);
+}
+
+int
+isp_mbox_acquire(ispsoftc_t *isp)
+{
+	if (isp->isp_osinfo.mboxbsy) {
+		return (1);
+	} else {
+		isp->isp_osinfo.mboxcmd_done = 0;
+		isp->isp_osinfo.mboxbsy = 1;
+		return (0);
+	}
+}
+
+void
+isp_mbox_wait_complete(ispsoftc_t *isp, mbreg_t *mbp)
+{
+	int usecs = mbp->timeout;
+	int j;
+
+	if (usecs == 0) {
+		usecs = MBCMD_DEFAULT_TIMEOUT;
+	}
+	if (isp->isp_mbxwrk0) {
+		usecs *= isp->isp_mbxwrk0;
+	}
+	if (isp->isp_osinfo.mbox_sleep_ok) {
+		int ms = usecs / 1000;
+		isp->isp_osinfo.mbox_sleep_ok = 0;
+		isp->isp_osinfo.mbox_sleeping = 1;
+#if __FreeBSD_version < 500000  || !defined(ISP_SMPLOCK)
+		tsleep(&isp->isp_mbxworkp, PRIBIO, "ispmbx_sleep",
+		    isp_mstohz(ms));
+#else
+		msleep(&isp->isp_mbxworkp, &isp->isp_mtx, PRIBIO,
+		    "ispmbx_sleep", isp_mstohz(ms));
+#endif
+		isp->isp_osinfo.mbox_sleep_ok = 1;
+		isp->isp_osinfo.mbox_sleeping = 0;
+	} else {
+		for (j = 0; j < usecs; j += 100) {
+			uint32_t isr;
+			uint16_t sema, mbox;
+			if (isp->isp_osinfo.mboxcmd_done) {
+				break;
+			}
+			if (ISP_READ_ISR(isp, &isr, &sema, &mbox)) {
+				isp_intr(isp, isr, sema, mbox);
+				if (isp->isp_osinfo.mboxcmd_done) {
+					break;
+				}
+			}
+			USEC_DELAY(100);
+		}
+	}
+	if (isp->isp_osinfo.mboxcmd_done == 0) {
+		isp_prt(isp, ISP_LOGWARN,
+		    "%s Mailbox Command (0x%x) Timeout",
+		    isp->isp_osinfo.mbox_sleep_ok? "Interrupting" : "Polled",
+		    isp->isp_lastmbxcmd);
+		mbp->param[0] = MBOX_TIMEOUT;
+		isp->isp_osinfo.mboxcmd_done = 1;
+	}
+}
+
+void
+isp_mbox_notify_done(ispsoftc_t *isp)
+{
+	if (isp->isp_osinfo.mbox_sleeping) {
+		wakeup(&isp->isp_mbxworkp);
+	}
+	isp->isp_osinfo.mboxcmd_done = 1;
+}
+
+void
+isp_mbox_release(ispsoftc_t *isp)
+{
+	isp->isp_osinfo.mboxbsy = 0;
+}
+
+int
+isp_mstohz(int ms)
+{
+	struct timeval t;
+	t.tv_sec = ms / 1000;
+	t.tv_usec = (ms % 1000) * 1000;
+	ms = tvtohz(&t);
+	if (ms < 0) {
+		ms = 0x7fffffff;
+	}
+	return (ms);
+}
diff --git a/sys/dev/isp/isp_freebsd.h b/sys/dev/isp/isp_freebsd.h
index 2a96d5d..e41b0c8 100644
--- a/sys/dev/isp/isp_freebsd.h
+++ b/sys/dev/isp/isp_freebsd.h
@@ -149,16 +149,22 @@ struct isposinfo {
 	struct cam_sim		*sim2;
 	struct cam_path		*path2;
 	struct intr_config_hook	ehook;
-	uint16_t		loop_down_time;
-	uint16_t		loop_down_limit;
+	uint32_t		loop_down_time;
+	uint32_t		loop_down_limit;
+	uint32_t		gone_device_time;
 	uint32_t		: 5,
 		simqfrozen	: 3,
 		hysteresis	: 8,
-				: 4,
+		gdt_running	: 1,
+		ldt_running	: 1,
 		disabled	: 1,
 		fcbsy		: 1,
+		mbox_sleeping	: 1,
+		mbox_sleep_ok	: 1,
 		mboxcmd_done	: 1,
 		mboxbsy		: 1;
+	struct callout_handle 	ldt;	/* loop down timer */
+	struct callout_handle	gdt;	/* gone device timer */
 #if __FreeBSD_version >= 500000  
 	struct firmware *	fw;
 	struct mtx		lock;
@@ -168,9 +174,6 @@ struct isposinfo {
 			char wwnn[17];
 			char wwpn[17];
 		} fc;
-		struct {
-			int iid;
-		} spi;
 	} sysctl_info;
 #endif
 	struct proc		*kproc;
@@ -204,10 +207,17 @@ struct isposinfo {
 #define	CAMLOCK_2_ISPLOCK(isp)	\
 	mtx_unlock(&Giant); mtx_lock(&(isp)->isp_lock)
 #else
+#if __FreeBSD_version < 500000
 #define	ISP_LOCK(x)		do { } while (0)
 #define	ISP_UNLOCK(x)		do { } while (0)
 #define	ISPLOCK_2_CAMLOCK(isp)	do { } while (0)
 #define	CAMLOCK_2_ISPLOCK(isp)	do { } while (0)
+#else
+#define	ISP_LOCK(x)		GIANT_REQUIRED
+#define	ISP_UNLOCK(x)		do { } while (0)
+#define	ISPLOCK_2_CAMLOCK(isp)	do { } while (0)
+#define	CAMLOCK_2_ISPLOCK(isp)	GIANT_REQUIRED
+#endif
 #endif
 
 /*
@@ -225,7 +235,7 @@ struct isposinfo {
 #define	NANOTIME_T		struct timespec
 #define	GET_NANOTIME		nanotime
 #define	GET_NANOSEC(x)		((x)->tv_sec * 1000000000 + (x)->tv_nsec)
-#define	NANOTIME_SUB		nanotime_sub
+#define	NANOTIME_SUB		isp_nanotime_sub
 
 #define	MAXISPREQUEST(isp)	((IS_FC(isp) || IS_ULTRA2(isp))? 1024 : 256)
 
@@ -247,7 +257,7 @@ default:							\
 
 #define	MBOX_ACQUIRE			isp_mbox_acquire
 #define	MBOX_WAIT_COMPLETE		isp_mbox_wait_complete
-#define	MBOX_NOTIFY_COMPLETE(isp)	isp->isp_osinfo.mboxcmd_done = 1
+#define	MBOX_NOTIFY_COMPLETE		isp_mbox_notify_done
 #define	MBOX_RELEASE			isp_mbox_release
 
 #define	FC_SCRATCH_ACQUIRE(isp)						\
@@ -326,10 +336,9 @@ default:							\
 #define	XS_INITERR(ccb)		\
 	XS_SETERR(ccb, CAM_REQ_INPROG), (ccb)->ccb_h.spriv_field0 = 0
 
-#define	XS_SAVE_SENSE(xs, sp)				\
-	(xs)->ccb_h.status |= CAM_AUTOSNS_VALID,	\
-	memcpy(&(xs)->sense_data, sp->req_sense_data,	\
-	    imin(XS_SNSLEN(xs), sp->req_sense_len))
+#define	XS_SAVE_SENSE(xs, sense_ptr, sense_len)		\
+	(xs)->ccb_h.status |= CAM_AUTOSNS_VALID;	\
+	memcpy(&(xs)->sense_data, sense_ptr, imin(XS_SNSLEN(xs), sense_len))
 
 #define	XS_SET_STATE_STAT(a, b, c)
 
@@ -404,8 +413,6 @@ default:							\
 #include <dev/isp/isp_tpublic.h>
 #endif
 
-void isp_prt(ispsoftc_t *, int level, const char *, ...)
-	__printflike(3, 4);
 /*
  * isp_osinfo definiitions && shorthand
  */
@@ -431,6 +438,7 @@ extern void isp_uninit(ispsoftc_t *);
 extern int isp_announced;
 extern int isp_fabric_hysteresis;
 extern int isp_loop_down_limit;
+extern int isp_gone_device_time;
 extern int isp_quickboot_time;
 
 /*
@@ -456,100 +464,22 @@ extern int isp_quickboot_time;
 #define	XS_CMD_S_CLEAR(sccb)	(sccb)->ccb_h.spriv_field0 = 0
 
 /*
- * Platform specific inline functions
+ * Platform Library Functions
  */
+void isp_prt(ispsoftc_t *, int level, const char *, ...) __printflike(3, 4);
+uint64_t isp_nanotime_sub(struct timespec *, struct timespec *);
+int isp_mbox_acquire(ispsoftc_t *);
+void isp_mbox_wait_complete(ispsoftc_t *, mbreg_t *);
+void isp_mbox_notify_done(ispsoftc_t *);
+void isp_mbox_release(ispsoftc_t *);
+int isp_mstohz(int);
 
-static __inline int isp_mbox_acquire(ispsoftc_t *);
-static __inline void isp_mbox_wait_complete(ispsoftc_t *, mbreg_t *);
-static __inline void isp_mbox_release(ispsoftc_t *);
-
-static __inline int
-isp_mbox_acquire(ispsoftc_t *isp)
-{
-	if (isp->isp_osinfo.mboxbsy) {
-		return (1);
-	} else {
-		isp->isp_osinfo.mboxcmd_done = 0;
-		isp->isp_osinfo.mboxbsy = 1;
-		return (0);
-	}
-}
-
-static __inline void
-isp_mbox_wait_complete(ispsoftc_t *isp, mbreg_t *mbp)
-{
-	int lim = mbp->timeout;
-	int j;
-
-	if (lim == 0) {
-		lim = MBCMD_DEFAULT_TIMEOUT;
-	}
-	if (isp->isp_mbxwrk0) {
-		lim *= isp->isp_mbxwrk0;
-	}
-	for (j = 0; j < lim; j += 100) {
-		uint32_t isr;
-		uint16_t sema, mbox;
-		if (isp->isp_osinfo.mboxcmd_done) {
-			break;
-		}
-		if (ISP_READ_ISR(isp, &isr, &sema, &mbox)) {
-			isp_intr(isp, isr, sema, mbox);
-			if (isp->isp_osinfo.mboxcmd_done) {
-				break;
-			}
-		}
-		USEC_DELAY(100);
-	}
-	if (isp->isp_osinfo.mboxcmd_done == 0) {
-		isp_prt(isp, ISP_LOGWARN,
-		    "Polled Mailbox Command (0x%x) Timeout",
-		    isp->isp_lastmbxcmd);
-		mbp->param[0] = MBOX_TIMEOUT;
-		isp->isp_osinfo.mboxcmd_done = 1;
-	}
-}
-
-static __inline void
-isp_mbox_release(ispsoftc_t *isp)
-{
-	isp->isp_osinfo.mboxbsy = 0;
-}
-
-static __inline uint64_t nanotime_sub(struct timespec *, struct timespec *);
-static __inline uint64_t
-nanotime_sub(struct timespec *b, struct timespec *a)
-{
-	uint64_t elapsed;
-	struct timespec x = *b;
-	timespecsub(&x, a);
-	elapsed = GET_NANOSEC(&x);
-	if (elapsed == 0)
-		elapsed++;
-	return (elapsed);
-}
-
-static __inline char *strncat(char *, const char *, size_t);
-static __inline char *
-strncat(char *d, const char *s, size_t c)
-{
-        char *t = d;
-
-        if (c) {
-                while (*d)
-                        d++;
-                while ((*d++ = *s++)) {
-                        if (--c == 0) {
-                                *d = '\0';
-                                break;
-                        }
-                }
-        }
-        return (t);
-}
+/*
+ * Platform specific inline functions
+ */
 
 /*
- * ISP Library functions
+ * ISP General Library functions
  */
 
 #include <dev/isp/isp_library.h>
diff --git a/sys/dev/isp/isp_library.c b/sys/dev/isp/isp_library.c
index 072d726..9b086f6 100644
--- a/sys/dev/isp/isp_library.c
+++ b/sys/dev/isp/isp_library.c
@@ -197,13 +197,13 @@ isp_fc_runstate(ispsoftc_t *isp, int tval)
 	if (fcp->isp_fwstate < FW_READY ||
 	    fcp->isp_loopstate < LOOP_PDB_RCVD) {
 		if (isp_control(isp, ISPCTL_FCLINK_TEST, tptr) != 0) {
-			isp_prt(isp, ISP_LOGINFO,
+			isp_prt(isp, ISP_LOGSANCFG,
 			    "isp_fc_runstate: linktest failed");
 			return (-1);
 		}
 		if (fcp->isp_fwstate != FW_READY ||
 		    fcp->isp_loopstate < LOOP_PDB_RCVD) {
-			isp_prt(isp, ISP_LOGINFO,
+			isp_prt(isp, ISP_LOGSANCFG,
 				"isp_fc_runstate: f/w not ready");
 			return (-1);
 		}
@@ -212,27 +212,74 @@ isp_fc_runstate(ispsoftc_t *isp, int tval)
 		return (0);
 	}
 	if (isp_control(isp, ISPCTL_SCAN_LOOP, NULL) != 0) {
-		isp_prt(isp, ISP_LOGINFO,
+		isp_prt(isp, ISP_LOGSANCFG,
 		    "isp_fc_runstate: scan loop fails");
 		return (LOOP_PDB_RCVD);
 	}
 	if (isp_control(isp, ISPCTL_SCAN_FABRIC, NULL) != 0) {
-		isp_prt(isp, ISP_LOGINFO,
+		isp_prt(isp, ISP_LOGSANCFG,
 		    "isp_fc_runstate: scan fabric fails");
 		return (LOOP_LSCAN_DONE);
 	}
 	if (isp_control(isp, ISPCTL_PDB_SYNC, NULL) != 0) {
-		isp_prt(isp, ISP_LOGINFO, "isp_fc_runstate: pdb_sync fails");
+		isp_prt(isp, ISP_LOGSANCFG, "isp_fc_runstate: pdb_sync fails");
 		return (LOOP_FSCAN_DONE);
 	}
 	if (fcp->isp_fwstate != FW_READY || fcp->isp_loopstate != LOOP_READY) {
-		isp_prt(isp, ISP_LOGINFO,
+		isp_prt(isp, ISP_LOGSANCFG,
 		    "isp_fc_runstate: f/w not ready again");
 		return (-1);
 	}
 	return (0);
 }
 
+/*
+ * Fibre Channel Support- get the port database for the id.
+ */
+void
+isp_dump_portdb(ispsoftc_t *isp)
+{
+	fcparam *fcp = (fcparam *) isp->isp_param;
+	int i;
+
+	for (i = 0; i < MAX_FC_TARG; i++) {
+		char mb[4];
+		const char *dbs[8] = {
+			"NIL ",
+			"PROB",
+			"DEAD",
+			"CHGD",
+			"NEW ",
+			"PVLD",
+			"ZOMB",
+			"VLD "
+		};
+		const char *roles[4] = {
+			" UNK", " TGT", " INI", "TINI"
+		};
+		fcportdb_t *lp = &fcp->portdb[i];
+
+		if (lp->state == FC_PORTDB_STATE_NIL) {
+			continue;
+		}
+		if (lp->ini_map_idx) {
+			SNPRINTF(mb, sizeof (mb), "%3d",
+			    ((int) lp->ini_map_idx) - 1);
+		} else {
+			SNPRINTF(mb, sizeof (mb), "---");
+		}
+		isp_prt(isp, ISP_LOGALL, "%d: %s al%d tgt %s %s 0x%06x =>%s"
+		    " 0x%06x; WWNN 0x%08x%08x WWPN 0x%08x%08x", i,
+		    dbs[lp->state], lp->autologin, mb,
+		    roles[lp->roles], lp->portid,
+		    roles[lp->new_roles], lp->new_portid,
+		    (uint32_t) (lp->node_wwn >> 32),
+		    (uint32_t) (lp->node_wwn),
+		    (uint32_t) (lp->port_wwn >> 32),
+		    (uint32_t) (lp->port_wwn));
+	}
+}
+
 void
 isp_shutdown(ispsoftc_t *isp)
 {
diff --git a/sys/dev/isp/isp_library.h b/sys/dev/isp/isp_library.h
index 9447741..2c4b013 100644
--- a/sys/dev/isp/isp_library.h
+++ b/sys/dev/isp/isp_library.h
@@ -39,6 +39,7 @@ extern int isp_getrqentry(ispsoftc_t *, uint32_t *, uint32_t *, void **);
 extern void isp_print_qentry (ispsoftc_t *, char *, int, void *);
 extern void isp_print_bytes(ispsoftc_t *, char *, int, void *);
 extern int isp_fc_runstate(ispsoftc_t *, int);
+extern void isp_dump_portdb(ispsoftc_t *);
 extern void isp_shutdown(ispsoftc_t *);
 extern void isp_put_hdr(ispsoftc_t *, isphdr_t *, isphdr_t *);
 extern void isp_get_hdr(ispsoftc_t *, isphdr_t *, isphdr_t *);
diff --git a/sys/dev/isp/isp_pci.c b/sys/dev/isp/isp_pci.c
index db17cc7..2f9c9a5 100644
--- a/sys/dev/isp/isp_pci.c
+++ b/sys/dev/isp/isp_pci.c
@@ -432,6 +432,9 @@ isp_get_options(device_t dev, ispsoftc_t *isp)
 	uint64_t wwn;
 	int bitmap, unit;
 
+	callout_handle_init(&isp->isp_osinfo.ldt);
+	callout_handle_init(&isp->isp_osinfo.gdt);
+
 	unit = device_get_unit(dev);
 	if (getenv_int("isp_disable", &bitmap)) {
 		if (bitmap & (1 << unit)) {
@@ -520,6 +523,14 @@ isp_get_options(device_t dev, ispsoftc_t *isp)
 		isp->isp_osinfo.loop_down_limit = isp_loop_down_limit;
 	}
 
+	bitmap = 0;
+	(void) getenv_int("isp_gone_device_time", &bitmap);
+	if (bitmap >= 0 && bitmap < 0xffff) {
+		isp->isp_osinfo.gone_device_time = bitmap;
+	} else {
+		isp->isp_osinfo.gone_device_time = isp_gone_device_time;
+	}
+
 
 #ifdef	ISP_FW_CRASH_DUMP
 	bitmap = 0;
@@ -578,6 +589,10 @@ isp_get_options(device_t dev, ispsoftc_t *isp)
 {
 	int tval;
 	const char *sptr;
+
+	callout_handle_init(&isp->isp_osinfo.ldt);
+	callout_handle_init(&isp->isp_osinfo.gdt);
+
 	/*
 	 * Figure out if we're supposed to skip this one.
 	 */
@@ -733,7 +748,7 @@ isp_get_options(device_t dev, ispsoftc_t *isp)
 		isp->isp_osinfo.hysteresis = isp_fabric_hysteresis;
 	}
 
-	tval = 0;
+	tval = -1;
 	(void) resource_int_value(device_get_name(dev), device_get_unit(dev),
 	    "loop_down_limit", &tval);
 	if (tval >= 0 && tval < 0xffff) {
@@ -742,6 +757,14 @@ isp_get_options(device_t dev, ispsoftc_t *isp)
 		isp->isp_osinfo.loop_down_limit = isp_loop_down_limit;
 	}
 
+	tval = -1;
+	(void) resource_int_value(device_get_name(dev), device_get_unit(dev),
+	    "gone_device_time", &tval);
+	if (tval >= 0 && tval < 0xffff) {
+		isp->isp_osinfo.gone_device_time = tval;
+	} else {
+		isp->isp_osinfo.gone_device_time = isp_gone_device_time;
+	}
 }
 
 static void
@@ -795,7 +818,7 @@ isp_pci_attach(device_t dev)
 	isp = &pcs->pci_isp;
 
 	/*
-	 * Get Generic Options
+	 * Set and Get Generic Options
 	 */
 	isp_get_options(dev, isp);
 
@@ -814,7 +837,6 @@ isp_pci_attach(device_t dev)
 	 */
 	isp_get_pci_options(dev, &m1, &m2);
 
-
 	linesz = PCI_DFLT_LNSZ;
 	irq = regs = NULL;
 	rgd = rtp = iqd = 0;
diff --git a/sys/dev/isp/isp_stds.h b/sys/dev/isp/isp_stds.h
index a25f774..bfacd14 100644
--- a/sys/dev/isp/isp_stds.h
+++ b/sys/dev/isp/isp_stds.h
@@ -129,7 +129,7 @@ typedef struct {
 /*
  * RFT_ID Requet CT_IU
  *
- * Source: INCITS xxx-200x Generic Services- 5 Rev 8.5 Section 5.2.5.30
+ * Source: NCITS xxx-200x Generic Services- 5 Rev 8.5 Section 5.2.5.30
  */
 typedef struct {
 	ct_hdr_t	rftid_hdr;
@@ -138,6 +138,19 @@ typedef struct {
 	uint32_t	rftid_fc4types[8];
 } rft_id_t;
 
+/*
+ * FCP Response Code Definitions
+ * Source: NCITS T10, Project 1144D, Revision 07a (aka FCP2r07a)
+ */
+#define	FCP_RSPNS_CODE_OFFSET		3
+
+#define	FCP_RSPNS_TMF_DONE		0
+#define	FCP_RSPNS_DLBRSTX		1
+#define	FCP_RSPNS_BADCMND		2
+#define	FCP_RSPNS_EROFS			3
+#define	FCP_RSPNS_TMF_REJECT		4
+#define	FCP_RSPNS_TMF_FAILED		5
+
 
 /* unconverted miscellany */
 /*
diff --git a/sys/dev/isp/ispvar.h b/sys/dev/isp/ispvar.h
index fec8423..72d3da3 100644
--- a/sys/dev/isp/ispvar.h
+++ b/sys/dev/isp/ispvar.h
@@ -75,7 +75,7 @@ struct ispmdvec {
  * Overall parameters
  */
 #define	MAX_TARGETS		16
-#define	MAX_FC_TARG		256
+#define	MAX_FC_TARG		512
 #define	ISP_MAX_TARGETS(isp)	(IS_FC(isp)? MAX_FC_TARG : MAX_TARGETS)
 #define	ISP_MAX_LUNS(isp)	(isp)->isp_maxluns
 
@@ -303,14 +303,33 @@ typedef struct {
  *    value.
  */
 typedef struct {
+	/*
+	 * This is the handle that the firmware needs in order for us to
+	 * send commands to the device. For pre-24XX cards, this would be
+	 * the 'loopid'.
+	 */
 	uint16_t	handle;
+	/*
+	 * The ini_map_idx, if nonzero, is the system virtual target ID (+1)
+	 * as a cross-reference with the isp_ini_map.
+	 *
+	 * A device is 'autologin' if the firmware automatically logs into
+	 * it (re-logins as needed). Basically, local private loop devices.
+	 *
+	 * The state is the current state of thsi entry.
+	 *
+	 * Role is Initiator, Target, Both
+	 *
+	 * Portid is obvious, as or node && port WWNs. The new_role and
+	 * new_portid is for when we are pending a change.
+	 */
 	uint16_t	ini_map_idx	: 12,
 			autologin	: 1,	/* F/W does PLOGI/PLOGO */
 			state		: 3;
-	uint32_t			: 6,
+	uint32_t	reserved	: 6,
 			roles		: 2,
 			portid		: 24;
-	uint32_t			: 6,
+	uint32_t	new_reserved	: 6,
 			new_roles	: 2,
 			new_portid	: 24;
 	uint64_t	node_wwn;
@@ -323,6 +342,7 @@ typedef struct {
 #define	FC_PORTDB_STATE_CHANGED		3
 #define	FC_PORTDB_STATE_NEW		4
 #define	FC_PORTDB_STATE_PENDING_VALID	5
+#define	FC_PORTDB_STATE_ZOMBIE		6
 #define	FC_PORTDB_STATE_VALID		7
 
 /*
@@ -357,8 +377,21 @@ typedef struct {
 	uint16_t		isp_maxfrmlen;
 	uint64_t		isp_nodewwn;
 	uint64_t		isp_portwwn;
+
+	/*
+	 * Our Port Data Base
+	 */
 	fcportdb_t		portdb[MAX_FC_TARG];
+
+	/*
+	 * This maps system virtual 'target' id to a portdb entry.
+	 *
+	 * The mapping function is to take any non-zero entry and
+	 * subtract one to get the portdb index. This means that
+	 * entries which are zero are unmapped (i.e., don't exist).
+	 */
 	uint16_t		isp_ini_map[MAX_FC_TARG];
+
 	/*
 	 * Scratch DMA mapped in area to fetch Port Database stuff, etc.
 	 */
@@ -849,7 +882,7 @@ int isp_async(ispsoftc_t *, ispasync_t, void *);
 #define	ISP_LOGDEBUG1	0x20	/* log intermediate debug messages */
 #define	ISP_LOGDEBUG2	0x40	/* log most debug messages */
 #define	ISP_LOGDEBUG3	0x80	/* log high frequency debug messages */
-#define	ISP_LOGDEBUG4	0x100	/* log high frequency debug messages */
+#define	ISP_LOGSANCFG	0x100	/* log SAN configuration */
 #define	ISP_LOGTDEBUG0	0x200	/* log simple debug messages (target mode) */
 #define	ISP_LOGTDEBUG1	0x400	/* log intermediate debug messages (target) */
 #define	ISP_LOGTDEBUG2	0x800	/* log all debug messages (target) */
@@ -934,7 +967,7 @@ int isp_async(ispsoftc_t *, ispasync_t, void *);
  *	XS_NOERR(xs)	there is no error currently set
  *	XS_INITERR(xs)	initialize error state
  *
- *	XS_SAVE_SENSE(xs, sp)		save sense data
+ *	XS_SAVE_SENSE(xs, sp, len)	save sense data
  *
  *	XS_SET_STATE_STAT(isp, sp, xs)	platform dependent interpreter of
  *					response queue entry status bits