MFp4:

- Reduce code duplication in ATA XPT and PMP driver.
- Move PIO size setting from ada driver to ATA XPT. It is XPT business
to negotiate transfer details. ada driver is now stateless.
- Report PIO size to SIM. It is required for correct PATA SIM operation.
- Tune PMP scan timings. It workarounds some problems with SiI.
- If reset hapens during PMP initialization - restart it.
- Introduce early-initialized periph drivers, which are used during initial
scan process. Use it for xpt, probe, aprobe and pmp. It gives pmp chance
to finish scan before mountroot and numerate devices in right order.

10 files changed, 452 insertions, 517 deletions

diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c
index 259d49d..76ed3d2 100644
--- a/sys/cam/ata/ata_da.c
+++ b/sys/cam/ata/ata_da.c
@@ -63,8 +63,7 @@ __FBSDID("$FreeBSD$");
 #define ATA_MAX_28BIT_LBA               268435455UL
 
 typedef enum {
-	ADA_STATE_NORMAL,
-	ADA_STATE_SET_MULTI
+	ADA_STATE_NORMAL
 } ada_state;
 
 typedef enum {
@@ -84,7 +83,6 @@ typedef enum {
 } ada_quirks;
 
 typedef enum {
-	ADA_CCB_SET_MULTI	= 0x01,
 	ADA_CCB_BUFFER_IO	= 0x03,
 	ADA_CCB_WAITING		= 0x04,
 	ADA_CCB_DUMP		= 0x05,
@@ -112,7 +110,6 @@ struct ada_softc {
 	ada_quirks quirks;
 	int	 ordered_tag_count;
 	int	 outstanding_cmds;
-	int	 secsperint;
 	struct	 disk_params params;
 	struct	 disk *disk;
 	union	 ccb saved_ccb;
@@ -550,22 +547,6 @@ adaasync(void *callback_arg, u_int32_t code,
 				"due to status 0x%x\n", status);
 		break;
 	}
-	case AC_SENT_BDR:
-	case AC_BUS_RESET:
-	{
-		struct ada_softc *softc = (struct ada_softc *)periph->softc;
-
-		cam_periph_async(periph, code, path, arg);
-		if (softc->state != ADA_STATE_NORMAL)
-			break;
-		/*
-		 * Restore device configuration.
-		 */
-		softc->state = ADA_STATE_SET_MULTI;
-		cam_periph_acquire(periph);
-		xpt_schedule(periph, CAM_PRIORITY_DEV);
-		break;
-	}
 	default:
 		cam_periph_async(periph, code, path, arg);
 		break;
@@ -644,8 +625,7 @@ adaregister(struct cam_periph *periph, void *arg)
 	if (cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ &&
 	    cgd->ident_data.queue >= 31)
 		softc->flags |= ADA_FLAG_CAN_NCQ;
-	softc->secsperint = max(1, min(cgd->ident_data.sectors_intr, 16));
-	softc->state = ADA_STATE_SET_MULTI;
+	softc->state = ADA_STATE_NORMAL;
 
 	periph->softc = softc;
 
@@ -734,18 +714,10 @@ adaregister(struct cam_periph *periph, void *arg)
 	 * them and the only alternative would be to
 	 * not attach the device on failure.
 	 */
-	xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE,
+	xpt_register_async(AC_LOST_DEVICE,
 			   adaasync, periph, periph->path);
 
 	/*
-	 * Take an exclusive refcount on the periph while adastart is called
-	 * to finish the probe.  The reference will be dropped in adadone at
-	 * the end of probe.
-	 */
-	cam_periph_acquire(periph);
-	xpt_schedule(periph, /*priority*/5);
-
-	/*
 	 * Schedule a periodic event to occasionally send an
 	 * ordered tag to a device.
 	 */
@@ -901,21 +873,6 @@ adastart(struct cam_periph *periph, union ccb *start_ccb)
 		}
 		break;
 	}
-	case ADA_STATE_SET_MULTI:
-	{
-		cam_fill_ataio(ataio,
-		    ada_retry_count,
-		    adadone,
-		    CAM_DIR_NONE,
-		    0,
-		    NULL,
-		    0,
-		    ada_default_timeout*1000);
-
-		ata_28bit_cmd(ataio, ATA_SET_MULTI, 0, 0, softc->secsperint);
-		start_ccb->ccb_h.ccb_state = ADA_CCB_SET_MULTI;
-		xpt_action(start_ccb);
-	}
 	}
 }
 
@@ -1003,35 +960,6 @@ adadone(struct cam_periph *periph, union ccb *done_ccb)
 		wakeup(&done_ccb->ccb_h.cbfcnp);
 		return;
 	}
-	case ADA_CCB_SET_MULTI:
-	{
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-		} else {
-			int	error;
-
-			error = adaerror(done_ccb, 0, 0);
-			if (error == ERESTART) {
-				/* A retry was scheduled, so just return. */
-				return;
-			}
-		}
-		softc->state = ADA_STATE_NORMAL;
-		/*
-		 * Since our peripheral may be invalidated by an error
-		 * above or an external event, we must release our CCB
-		 * before releasing the probe lock on the peripheral.
-		 * The peripheral will only go away once the last lock
-		 * is removed, and we need it around for the CCB release
-		 * operation.
-		 */
-		xpt_release_ccb(done_ccb);
-		if (bioq_first(&softc->bio_queue) != NULL) {
-			/* Have more work to do, so ensure we stay scheduled */
-			xpt_schedule(periph, CAM_PRIORITY_NORMAL);
-		}
-		cam_periph_release_locked(periph);
-		return;
-	}
 	case ADA_CCB_DUMP:
 		/* No-op.  We're polling */
 		return;
diff --git a/sys/cam/ata/ata_pmp.c b/sys/cam/ata/ata_pmp.c
index 76b3c0a..ea38790 100644
--- a/sys/cam/ata/ata_pmp.c
+++ b/sys/cam/ata/ata_pmp.c
@@ -93,7 +93,9 @@ struct pmp_softc {
 	int			pm_step;
 	int			pm_try;
 	int			found;
+	int			reset;
 	int			frozen;
+	int			restart;
 	union			ccb saved_ccb;
 	struct task		sysctl_task;
 	struct sysctl_ctx_list	sysctl_ctx;
@@ -134,7 +136,8 @@ TUNABLE_INT("kern.cam.pmp.default_timeout", &pmp_default_timeout);
 static struct periph_driver pmpdriver =
 {
 	pmpinit, "pmp",
-	TAILQ_HEAD_INITIALIZER(pmpdriver.units), /* generation */ 0
+	TAILQ_HEAD_INITIALIZER(pmpdriver.units), /* generation */ 0,
+	CAM_PERIPH_DRV_EARLY
 };
 
 PERIPHDRIVER_DECLARE(pmp, pmpdriver);
@@ -292,14 +295,21 @@ pmpasync(void *callback_arg, u_int32_t code,
 	case AC_BUS_RESET:
 		softc = (struct pmp_softc *)periph->softc;
 		cam_periph_async(periph, code, path, arg);
-		if (softc->state != PMP_STATE_NORMAL)
+		if (code == AC_SCSI_AEN && softc->state != PMP_STATE_NORMAL &&
+		    softc->state != PMP_STATE_SCAN)
 			break;
-		pmpfreeze(periph, softc->found);
+		if (softc->state != PMP_STATE_SCAN)
+			pmpfreeze(periph, softc->found);
+		else
+			pmpfreeze(periph, softc->found & ~(1 << softc->pm_step));
 		if (code == AC_SENT_BDR || code == AC_BUS_RESET)
 			softc->found = 0; /* We have to reset everything. */
-		softc->state = PMP_STATE_PORTS;
-		cam_periph_acquire(periph);
-		xpt_schedule(periph, CAM_PRIORITY_DEV);
+		if (softc->state == PMP_STATE_NORMAL) {
+			softc->state = PMP_STATE_PORTS;
+			cam_periph_acquire(periph);
+			xpt_schedule(periph, CAM_PRIORITY_BUS);
+		} else
+			softc->restart = 1;
 		break;
 	default:
 		cam_periph_async(periph, code, path, arg);
@@ -395,7 +405,7 @@ pmpregister(struct cam_periph *periph, void *arg)
 	 * the end of probe.
 	 */
 	(void)cam_periph_acquire(periph);
-	xpt_schedule(periph, CAM_PRIORITY_DEV);
+	xpt_schedule(periph, CAM_PRIORITY_BUS);
 
 	return(CAM_REQ_CMP);
 }
@@ -408,6 +418,11 @@ pmpstart(struct cam_periph *periph, union ccb *start_ccb)
 
 	softc = (struct pmp_softc *)periph->softc;
 	ataio = &start_ccb->ataio;
+	
+	if (softc->restart) {
+		softc->restart = 0;
+		softc->state = PMP_STATE_PORTS;
+	}
 
 	switch (softc->state) {
 	case PMP_STATE_PORTS:
@@ -469,6 +484,7 @@ printf("PM RESET %d%s\n", softc->pm_step,
 		ata_pm_read_cmd(ataio, 0, softc->pm_step);
 		break;
 	case PMP_STATE_CLEAR:
+		softc->reset = 0;
 		cam_fill_ataio(ataio,
 		      pmp_retry_count,
 		      pmpdone,
@@ -492,7 +508,7 @@ pmpdone(struct cam_periph *periph, union ccb *done_ccb)
 	struct ccb_ataio *ataio;
 	union ccb *work_ccb;
 	struct cam_path *path, *dpath;
-	u_int32_t  priority;
+	u_int32_t  priority, res;
 
 	softc = (struct pmp_softc *)periph->softc;
 	ataio = &done_ccb->ataio;
@@ -502,193 +518,158 @@ pmpdone(struct cam_periph *periph, union ccb *done_ccb)
 	path = done_ccb->ccb_h.path;
 	priority = done_ccb->ccb_h.pinfo.priority;
 
-	switch (softc->state) {
-	case PMP_STATE_PORTS:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_ports = (done_ccb->ataio.res.lba_high << 24) +
-			    (done_ccb->ataio.res.lba_mid << 16) +
-			    (done_ccb->ataio.res.lba_low << 8) +
-			    done_ccb->ataio.res.sector_count;
-			/* This PM declares 6 ports, while only 5 of them are real.
-			 * Port 5 is enclosure management bridge port, which has implementation
-			 * problems, causing probe faults. Hide it for now. */
-			if (softc->pm_pid == 0x37261095 && softc->pm_ports == 6)
-				softc->pm_ports = 5;
-			/* This PM declares 7 ports, while only 5 of them are real.
-			 * Port 5 is some fake "Config  Disk" with 640 sectors size,
-			 * port 6 is enclosure management bridge port.
-			 * Both fake ports has implementation problems, causing
-			 * probe faults. Hide them for now. */
-			if (softc->pm_pid == 0x47261095 && softc->pm_ports == 7)
-				softc->pm_ports = 5;
-			printf("PM ports: %d\n", softc->pm_ports);
-			softc->state = PMP_STATE_CONFIG;
-			xpt_release_ccb(done_ccb);
-			xpt_schedule(periph, priority);
-			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
+	if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
+		if (cam_periph_error(done_ccb, 0, 0,
+		    &softc->saved_ccb) == ERESTART) {
 			return;
 		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+			cam_release_devq(done_ccb->ccb_h.path,
+			    /*relsim_flags*/0,
+			    /*reduction*/0,
+			    /*timeout*/0,
+			    /*getcount_only*/0);
+		}
+		goto done;
+	}
+
+	if (softc->restart) {
+		softc->restart = 0;
+		if (softc->state == PMP_STATE_SCAN) {
+			pmpfreeze(periph, 1 << softc->pm_step);
+			work_ccb = done_ccb;
+			done_ccb = (union ccb*)work_ccb->ccb_h.ppriv_ptr0;
+			/* Free the current request path- we're done with it. */
+		    	xpt_free_path(work_ccb->ccb_h.path);
+			xpt_free_ccb(work_ccb);
 		}
 		xpt_release_ccb(done_ccb);
-		break;
+		softc->state = PMP_STATE_PORTS;
+		xpt_schedule(periph, priority);
+		return;
+	}
+
+	switch (softc->state) {
+	case PMP_STATE_PORTS:
+		softc->pm_ports = (done_ccb->ataio.res.lba_high << 24) +
+		    (done_ccb->ataio.res.lba_mid << 16) +
+		    (done_ccb->ataio.res.lba_low << 8) +
+		    done_ccb->ataio.res.sector_count;
+		/* This PM declares 6 ports, while only 5 of them are real.
+		 * Port 5 is enclosure management bridge port, which has implementation
+		 * problems, causing probe faults. Hide it for now. */
+		if (softc->pm_pid == 0x37261095 && softc->pm_ports == 6)
+			softc->pm_ports = 5;
+		/* This PM declares 7 ports, while only 5 of them are real.
+		 * Port 5 is some fake "Config  Disk" with 640 sectors size,
+		 * port 6 is enclosure management bridge port.
+		 * Both fake ports has implementation problems, causing
+		 * probe faults. Hide them for now. */
+		if (softc->pm_pid == 0x47261095 && softc->pm_ports == 7)
+			softc->pm_ports = 5;
+		printf("PM ports: %d\n", softc->pm_ports);
+		softc->state = PMP_STATE_CONFIG;
+		xpt_release_ccb(done_ccb);
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_CONFIG:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_step = 0;
-			softc->state = PMP_STATE_RESET;
-			xpt_release_ccb(done_ccb);
-			xpt_schedule(periph, priority);
-			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
-		}
+		softc->pm_step = 0;
+		softc->state = PMP_STATE_RESET;
+		softc->reset |= ~softc->found;
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_RESET:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_step++;
-			if (softc->pm_step < softc->pm_ports) {
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			} else {
-				softc->pm_step = 0;
-				DELAY(5000);
-				printf("PM reset done\n");
-				softc->state = PMP_STATE_CONNECT;
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+		softc->pm_step++;
+		if (softc->pm_step >= softc->pm_ports) {
+			softc->pm_step = 0;
+			cam_freeze_devq(periph->path);
+			cam_release_devq(periph->path,
+			    RELSIM_RELEASE_AFTER_TIMEOUT,
+			    /*reduction*/0,
+			    /*timeout*/5,
+			    /*getcount_only*/0);
+			printf("PM reset done\n");
+			softc->state = PMP_STATE_CONNECT;
 		}
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_CONNECT:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_step++;
-			if (softc->pm_step < softc->pm_ports) {
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			} else {
-				softc->pm_step = 0;
-				softc->pm_try = 0;
-				printf("PM connect done\n");
-				softc->state = PMP_STATE_CHECK;
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+		softc->pm_step++;
+		if (softc->pm_step >= softc->pm_ports) {
+			softc->pm_step = 0;
+			softc->pm_try = 0;
+			cam_freeze_devq(periph->path);
+			cam_release_devq(periph->path,
+			    RELSIM_RELEASE_AFTER_TIMEOUT,
+			    /*reduction*/0,
+			    /*timeout*/10,
+			    /*getcount_only*/0);
+			printf("PM connect done\n");
+			softc->state = PMP_STATE_CHECK;
 		}
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_CHECK:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			int res = (done_ccb->ataio.res.lba_high << 24) +
-			    (done_ccb->ataio.res.lba_mid << 16) +
-			    (done_ccb->ataio.res.lba_low << 8) +
-			    done_ccb->ataio.res.sector_count;
-			if ((res & 0xf0f) == 0x103 && (res & 0x0f0) != 0) {
-				printf("PM status: %d - %08x\n", softc->pm_step, res);
-				softc->found |= (1 << softc->pm_step);
-				softc->pm_step++;
+		res = (done_ccb->ataio.res.lba_high << 24) +
+		    (done_ccb->ataio.res.lba_mid << 16) +
+		    (done_ccb->ataio.res.lba_low << 8) +
+		    done_ccb->ataio.res.sector_count;
+		if ((res & 0xf0f) == 0x103 && (res & 0x0f0) != 0) {
+			printf("PM status: %d - %08x\n", softc->pm_step, res);
+			softc->found |= (1 << softc->pm_step);
+			softc->pm_step++;
+		} else {
+			if (softc->pm_try < 10) {
+				cam_freeze_devq(periph->path);
+				cam_release_devq(periph->path,
+				    RELSIM_RELEASE_AFTER_TIMEOUT,
+				    /*reduction*/0,
+				    /*timeout*/10,
+				    /*getcount_only*/0);
+				softc->pm_try++;
 			} else {
-				if (softc->pm_try < 100) {
-					DELAY(10000);
-					softc->pm_try++;
-				} else {
-					printf("PM status: %d - %08x\n", softc->pm_step, res);
-					softc->found &= ~(1 << softc->pm_step);
-					if (xpt_create_path(&dpath, periph,
-					    done_ccb->ccb_h.path_id,
-					    softc->pm_step, 0) == CAM_REQ_CMP) {
-						xpt_async(AC_LOST_DEVICE, dpath, NULL);
-						xpt_free_path(dpath);
-					}
-					softc->pm_step++;
+				printf("PM status: %d - %08x\n", softc->pm_step, res);
+				softc->found &= ~(1 << softc->pm_step);
+				if (xpt_create_path(&dpath, periph,
+				    done_ccb->ccb_h.path_id,
+				    softc->pm_step, 0) == CAM_REQ_CMP) {
+					xpt_async(AC_LOST_DEVICE, dpath, NULL);
+					xpt_free_path(dpath);
 				}
+				softc->pm_step++;
 			}
-			if (softc->pm_step < softc->pm_ports) {
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			} else {
-				softc->pm_step = 0;
-				softc->state = PMP_STATE_CLEAR;
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
+		}
+		if (softc->pm_step >= softc->pm_ports) {
+			if (softc->reset & softc->found) {
+				cam_freeze_devq(periph->path);
+				cam_release_devq(periph->path,
+				    RELSIM_RELEASE_AFTER_TIMEOUT,
+				    /*reduction*/0,
+				    /*timeout*/1000,
+				    /*getcount_only*/0);
 			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+			softc->state = PMP_STATE_CLEAR;
+			softc->pm_step = 0;
 		}
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_CLEAR:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_step++;
-			if (softc->pm_step < softc->pm_ports) {
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			} else if (softc->found) {
-				softc->pm_step = 0;
-				softc->state = PMP_STATE_SCAN;
-				work_ccb = xpt_alloc_ccb_nowait();
-				if (work_ccb != NULL)
-					goto do_scan;
-				xpt_release_ccb(done_ccb);
-			}
-			break;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
+		softc->pm_step++;
+		if (softc->pm_step < softc->pm_ports) {
+			xpt_release_ccb(done_ccb);
+			xpt_schedule(periph, priority);
 			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+		} else if (softc->found) {
+			softc->pm_step = 0;
+			softc->state = PMP_STATE_SCAN;
+			work_ccb = xpt_alloc_ccb_nowait();
+			if (work_ccb != NULL)
+				goto do_scan;
+			xpt_release_ccb(done_ccb);
 		}
-		xpt_release_ccb(done_ccb);
 		break;
 	case PMP_STATE_SCAN:
 		work_ccb = done_ccb;
@@ -703,7 +684,6 @@ do_scan:
 		}
 		if (softc->pm_step >= softc->pm_ports) {
 			xpt_free_ccb(work_ccb);
-			xpt_release_ccb(done_ccb);
 			break;
 		}
 		if (xpt_create_path(&dpath, periph,
@@ -712,7 +692,6 @@ do_scan:
 			printf("pmpdone: xpt_create_path failed"
 			    ", bus scan halted\n");
 			xpt_free_ccb(work_ccb);
-			xpt_release_ccb(done_ccb);
 			break;
 		}
 		xpt_setup_ccb(&work_ccb->ccb_h, dpath,
@@ -727,6 +706,8 @@ do_scan:
 	default:
 		break;
 	}
+done:
+	xpt_release_ccb(done_ccb);
 	softc->state = PMP_STATE_NORMAL;
 	pmprelease(periph, -1);
 	cam_periph_release_locked(periph);
diff --git a/sys/cam/ata/ata_xpt.c b/sys/cam/ata/ata_xpt.c
index a693e2c..9e781cd 100644
--- a/sys/cam/ata/ata_xpt.c
+++ b/sys/cam/ata/ata_xpt.c
@@ -83,7 +83,8 @@ static periph_init_t probe_periph_init;
 static struct periph_driver probe_driver =
 {
 	probe_periph_init, "aprobe",
-	TAILQ_HEAD_INITIALIZER(probe_driver.units)
+	TAILQ_HEAD_INITIALIZER(probe_driver.units), /* generation */ 0,
+	CAM_PERIPH_DRV_EARLY
 };
 
 PERIPHDRIVER_DECLARE(aprobe, probe_driver);
@@ -92,6 +93,7 @@ typedef enum {
 	PROBE_RESET,
 	PROBE_IDENTIFY,
 	PROBE_SETMODE,
+	PROBE_SET_MULTI,
 	PROBE_INQUIRY,
 	PROBE_FULL_INQUIRY,
 	PROBE_PM_PID,
@@ -103,6 +105,7 @@ static char *probe_action_text[] = {
 	"PROBE_RESET",
 	"PROBE_IDENTIFY",
 	"PROBE_SETMODE",
+	"PROBE_SET_MULTI",
 	"PROBE_INQUIRY",
 	"PROBE_FULL_INQUIRY",
 	"PROBE_PM_PID",
@@ -282,12 +285,16 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 	struct ccb_ataio *ataio;
 	struct ccb_scsiio *csio;
 	probe_softc *softc;
+	struct cam_path *path;
+	struct ata_params *ident_buf;
 
 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probestart\n"));
 
 	softc = (probe_softc *)periph->softc;
+	path = start_ccb->ccb_h.path;
 	ataio = &start_ccb->ataio;
 	csio = &start_ccb->csio;
+	ident_buf = &periph->path->device->ident_data;
 
 	switch (softc->action) {
 	case PROBE_RESET:
@@ -302,10 +309,6 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 		ata_reset_cmd(ataio);
 		break;
 	case PROBE_IDENTIFY:
-	{
-		struct ata_params *ident_buf =
-		    &periph->path->device->ident_data;
-
 		if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
 			/* Prepare check that it is the same device. */
 			MD5_CTX context;
@@ -335,12 +338,7 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 		else
 			ata_28bit_cmd(ataio, ATA_ATAPI_IDENTIFY, 0, 0, 0);
 		break;
-	}
 	case PROBE_SETMODE:
-	{
-		struct ata_params *ident_buf =
-		    &periph->path->device->ident_data;
-
 		cam_fill_ataio(ataio,
 		      1,
 		      probedone,
@@ -352,6 +350,37 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 		ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_SETXFER, 0,
 		    ata_max_mode(ident_buf, ATA_UDMA6, ATA_UDMA6));
 		break;
+	case PROBE_SET_MULTI:
+	{
+		struct ccb_trans_settings cts;
+		u_int sectors;
+
+		sectors = max(1, min(ident_buf->sectors_intr & 0xff, 16));
+
+		/* Report bytecount to SIM. */
+		bzero(&cts, sizeof(cts));
+		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
+		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
+		cts.type = CTS_TYPE_CURRENT_SETTINGS;
+		if (path->device->transport == XPORT_ATA) {
+			cts.xport_specific.ata.bytecount = sectors * 512;
+			cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT;
+		} else {
+			cts.xport_specific.sata.bytecount = sectors * 512;
+			cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT;
+		}
+		xpt_action((union ccb *)&cts);
+
+		cam_fill_ataio(ataio,
+		    1,
+		    probedone,
+		    CAM_DIR_NONE,
+		    0,
+		    NULL,
+		    0,
+		    30*1000);
+		ata_28bit_cmd(ataio, ATA_SET_MULTI, 0, 0, sectors);
+		break;
 	}
 	case PROBE_INQUIRY:
 	case PROBE_FULL_INQUIRY:
@@ -406,7 +435,7 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 		ata_pm_read_cmd(ataio, 1, 15);
 		break;
 	case PROBE_INVALID:
-		CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_INFO,
+		CAM_DEBUG(path, CAM_DEBUG_INFO,
 		    ("probestart: invalid action state\n"));
 	default:
 		break;
@@ -552,101 +581,119 @@ probedone(struct cam_periph *periph, union ccb *done_ccb)
 	priority = done_ccb->ccb_h.pinfo.priority;
 	ident_buf = &path->device->ident_data;
 
-	switch (softc->action) {
-	case PROBE_RESET:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			int sign = (done_ccb->ataio.res.lba_high << 8) +
-			    done_ccb->ataio.res.lba_mid;
-			xpt_print(path, "SIGNATURE: %04x\n", sign);
-			if (sign == 0x0000 &&
-			    done_ccb->ccb_h.target_id != 15) {
-				path->device->protocol = PROTO_ATA;
-				PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
-			} else if (sign == 0x9669 &&
-			    done_ccb->ccb_h.target_id == 15) {
-				struct ccb_trans_settings cts;
-
-				/* Report SIM that PM is present. */
-				bzero(&cts, sizeof(cts));
-				xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
-				cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
-				cts.type = CTS_TYPE_CURRENT_SETTINGS;
-				cts.xport_specific.sata.pm_present = 1;
-				cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
-				xpt_action((union ccb *)&cts);
-				path->device->protocol = PROTO_SATAPM;
-				PROBE_SET_ACTION(softc, PROBE_PM_PID);
-			} else if (sign == 0xeb14 &&
-			    done_ccb->ccb_h.target_id != 15) {
-				path->device->protocol = PROTO_SCSI;
-				PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
-			} else {
-				if (done_ccb->ccb_h.target_id != 15) {
-					xpt_print(path,
-					    "Unexpected signature 0x%04x\n", sign);
-				}
-				goto device_fail;
-			}
-			xpt_release_ccb(done_ccb);
-			xpt_schedule(periph, priority);
-			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
+	if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
+device_fail:	if (cam_periph_error(done_ccb, 0, 0,
+		    &softc->saved_ccb) == ERESTART) {
 			return;
 		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
 			/* Don't wedge the queue */
 			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
 					 /*run_queue*/TRUE);
 		}
-		goto device_fail;
-	case PROBE_IDENTIFY:
+		/* Old PIO2 devices may not support mode setting. */
+		if (softc->action == PROBE_SETMODE &&
+		    ata_max_pmode(ident_buf) <= ATA_PIO2 &&
+		    (ident_buf->capabilities1 & ATA_SUPPORT_IORDY) == 0)
+			goto noerror;
+		/*
+		 * If we get to this point, we got an error status back
+		 * from the inquiry and the error status doesn't require
+		 * automatically retrying the command.  Therefore, the
+		 * inquiry failed.  If we had inquiry information before
+		 * for this device, but this latest inquiry command failed,
+		 * the device has probably gone away.  If this device isn't
+		 * already marked unconfigured, notify the peripheral
+		 * drivers that this device is no more.
+		 */
+		if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
+			xpt_async(AC_LOST_DEVICE, path, NULL);
+		found = 0;
+		goto done;
+	}
+noerror:
+	switch (softc->action) {
+	case PROBE_RESET:
 	{
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			int16_t *ptr;
+		int sign = (done_ccb->ataio.res.lba_high << 8) +
+		    done_ccb->ataio.res.lba_mid;
+		xpt_print(path, "SIGNATURE: %04x\n", sign);
+		if (sign == 0x0000 &&
+		    done_ccb->ccb_h.target_id != 15) {
+			path->device->protocol = PROTO_ATA;
+			PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
+		} else if (sign == 0x9669 &&
+		    done_ccb->ccb_h.target_id == 15) {
+			struct ccb_trans_settings cts;
 
-			for (ptr = (int16_t *)ident_buf;
-			     ptr < (int16_t *)ident_buf + sizeof(struct ata_params)/2; ptr++) {
-				*ptr = le16toh(*ptr);
-			}
-			if (strncmp(ident_buf->model, "FX", 2) &&
-			    strncmp(ident_buf->model, "NEC", 3) &&
-			    strncmp(ident_buf->model, "Pioneer", 7) &&
-			    strncmp(ident_buf->model, "SHARP", 5)) {
-				ata_bswap(ident_buf->model, sizeof(ident_buf->model));
-				ata_bswap(ident_buf->revision, sizeof(ident_buf->revision));
-				ata_bswap(ident_buf->serial, sizeof(ident_buf->serial));
-			}
-			ata_btrim(ident_buf->model, sizeof(ident_buf->model));
-			ata_bpack(ident_buf->model, ident_buf->model, sizeof(ident_buf->model));
-			ata_btrim(ident_buf->revision, sizeof(ident_buf->revision));
-			ata_bpack(ident_buf->revision, ident_buf->revision, sizeof(ident_buf->revision));
-			ata_btrim(ident_buf->serial, sizeof(ident_buf->serial));
-			ata_bpack(ident_buf->serial, ident_buf->serial, sizeof(ident_buf->serial));
-
-			if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
-				/* Check that it is the same device. */
-				MD5_CTX context;
-				u_int8_t digest[16];
-
-				MD5Init(&context);
-				MD5Update(&context,
-				    (unsigned char *)ident_buf->model,
-				    sizeof(ident_buf->model));
-				MD5Update(&context,
-				    (unsigned char *)ident_buf->revision,
-				    sizeof(ident_buf->revision));
-				MD5Update(&context,
-				    (unsigned char *)ident_buf->serial,
-				    sizeof(ident_buf->serial));
-				MD5Final(digest, &context);
-				if (bcmp(digest, softc->digest, sizeof(digest))) {
-					/* Device changed. */
-					xpt_async(AC_LOST_DEVICE, path, NULL);
-				}
-				xpt_release_ccb(done_ccb);
-				break;
+				/* Report SIM that PM is present. */
+			bzero(&cts, sizeof(cts));
+			xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
+			cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
+			cts.type = CTS_TYPE_CURRENT_SETTINGS;
+			cts.xport_specific.sata.pm_present = 1;
+			cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
+			xpt_action((union ccb *)&cts);
+			path->device->protocol = PROTO_SATAPM;
+			PROBE_SET_ACTION(softc, PROBE_PM_PID);
+		} else if (sign == 0xeb14 &&
+		    done_ccb->ccb_h.target_id != 15) {
+			path->device->protocol = PROTO_SCSI;
+			PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
+		} else {
+			if (done_ccb->ccb_h.target_id != 15) {
+				xpt_print(path,
+				    "Unexpected signature 0x%04x\n", sign);
 			}
+			goto device_fail;
+		}
+		xpt_release_ccb(done_ccb);
+		xpt_schedule(periph, priority);
+		return;
+	}
+	case PROBE_IDENTIFY:
+	{
+		int16_t *ptr;
+
+		for (ptr = (int16_t *)ident_buf;
+		     ptr < (int16_t *)ident_buf + sizeof(struct ata_params)/2; ptr++) {
+			*ptr = le16toh(*ptr);
+		}
+		if (strncmp(ident_buf->model, "FX", 2) &&
+		    strncmp(ident_buf->model, "NEC", 3) &&
+		    strncmp(ident_buf->model, "Pioneer", 7) &&
+		    strncmp(ident_buf->model, "SHARP", 5)) {
+			ata_bswap(ident_buf->model, sizeof(ident_buf->model));
+			ata_bswap(ident_buf->revision, sizeof(ident_buf->revision));
+			ata_bswap(ident_buf->serial, sizeof(ident_buf->serial));
+		}
+		ata_btrim(ident_buf->model, sizeof(ident_buf->model));
+		ata_bpack(ident_buf->model, ident_buf->model, sizeof(ident_buf->model));
+		ata_btrim(ident_buf->revision, sizeof(ident_buf->revision));
+		ata_bpack(ident_buf->revision, ident_buf->revision, sizeof(ident_buf->revision));
+		ata_btrim(ident_buf->serial, sizeof(ident_buf->serial));
+		ata_bpack(ident_buf->serial, ident_buf->serial, sizeof(ident_buf->serial));
 
+		if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
+			/* Check that it is the same device. */
+			MD5_CTX context;
+			u_int8_t digest[16];
+
+			MD5Init(&context);
+			MD5Update(&context,
+			    (unsigned char *)ident_buf->model,
+			    sizeof(ident_buf->model));
+			MD5Update(&context,
+			    (unsigned char *)ident_buf->revision,
+			    sizeof(ident_buf->revision));
+			MD5Update(&context,
+			    (unsigned char *)ident_buf->serial,
+			    sizeof(ident_buf->serial));
+			MD5Final(digest, &context);
+			if (bcmp(digest, softc->digest, sizeof(digest))) {
+				/* Device changed. */
+				xpt_async(AC_LOST_DEVICE, path, NULL);
+			}
+		} else {
 			/* Clean up from previous instance of this device */
 			if (path->device->serial_num != NULL) {
 				free(path->device->serial_num, M_CAMXPT);
@@ -655,7 +702,7 @@ probedone(struct cam_periph *periph, union ccb *done_ccb)
 			}
 			path->device->serial_num =
 				(u_int8_t *)malloc((sizeof(ident_buf->serial) + 1),
-						   M_CAMXPT, M_NOWAIT);
+					   M_CAMXPT, M_NOWAIT);
 			if (path->device->serial_num != NULL) {
 				bcopy(ident_buf->serial,
 				      path->device->serial_num,
@@ -667,188 +714,120 @@ probedone(struct cam_periph *periph, union ccb *done_ccb)
 			}
 
 			path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
-			ata_device_transport(path);
-			PROBE_SET_ACTION(softc, PROBE_SETMODE);
-			xpt_release_ccb(done_ccb);
-			xpt_schedule(periph, priority);
-			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
 		}
-device_fail:
-		/*
-		 * If we get to this point, we got an error status back
-		 * from the inquiry and the error status doesn't require
-		 * automatically retrying the command.  Therefore, the
-		 * inquiry failed.  If we had inquiry information before
-		 * for this device, but this latest inquiry command failed,
-		 * the device has probably gone away.  If this device isn't
-		 * already marked unconfigured, notify the peripheral
-		 * drivers that this device is no more.
-		 */
-		if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
-			xpt_async(AC_LOST_DEVICE, path, NULL);
-		found = 0;
+		ata_device_transport(path);
+		PROBE_SET_ACTION(softc, PROBE_SETMODE);
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	}
 	case PROBE_SETMODE:
-	{
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-modedone:		if (path->device->protocol == PROTO_ATA) {
-				path->device->flags &= ~CAM_DEV_UNCONFIGURED;
-				done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
-				xpt_action(done_ccb);
-				xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
-				    done_ccb);
-				xpt_release_ccb(done_ccb);
-				break;
-			} else {
-				PROBE_SET_ACTION(softc, PROBE_INQUIRY);
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
+		if (path->device->protocol == PROTO_ATA) {
+			PROBE_SET_ACTION(softc, PROBE_SET_MULTI);
+		} else {
+			PROBE_SET_ACTION(softc, PROBE_INQUIRY);
 		}
-		/* Old PIO2 devices may not support mode setting. */
-		if (ata_max_pmode(ident_buf) <= ATA_PIO2 &&
-		    (ident_buf->capabilities1 & ATA_SUPPORT_IORDY) == 0)
-			goto modedone;
-		goto device_fail;
-	}
+		xpt_release_ccb(done_ccb);
+		xpt_schedule(periph, priority);
+		return;
+	case PROBE_SET_MULTI:
+		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
+			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
+			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
+			xpt_action(done_ccb);
+			xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
+			    done_ccb);
+		}
+		break;
 	case PROBE_INQUIRY:
 	case PROBE_FULL_INQUIRY:
 	{
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			struct scsi_inquiry_data *inq_buf;
-			u_int8_t periph_qual;
+		struct scsi_inquiry_data *inq_buf;
+		u_int8_t periph_qual, len;
 
-			path->device->flags |= CAM_DEV_INQUIRY_DATA_VALID;
-			inq_buf = &path->device->inq_data;
+		path->device->flags |= CAM_DEV_INQUIRY_DATA_VALID;
+		inq_buf = &path->device->inq_data;
 
-			periph_qual = SID_QUAL(inq_buf);
+		periph_qual = SID_QUAL(inq_buf);
 
-			if (periph_qual == SID_QUAL_LU_CONNECTED) {
-				u_int8_t len;
+		if (periph_qual != SID_QUAL_LU_CONNECTED)
+			break;
 
-				/*
-				 * We conservatively request only
-				 * SHORT_INQUIRY_LEN bytes of inquiry
-				 * information during our first try
-				 * at sending an INQUIRY. If the device
-				 * has more information to give,
-				 * perform a second request specifying
-				 * the amount of information the device
-				 * is willing to give.
-				 */
-				len = inq_buf->additional_length
-				    + offsetof(struct scsi_inquiry_data,
-                                               additional_length) + 1;
-				if (softc->action == PROBE_INQUIRY
-				    && len > SHORT_INQUIRY_LENGTH) {
-					PROBE_SET_ACTION(softc, PROBE_FULL_INQUIRY);
-					xpt_release_ccb(done_ccb);
-					xpt_schedule(periph, priority);
-					return;
-				}
-
-				scsi_find_quirk(path->device);
-				ata_device_transport(path);
-				path->device->flags &= ~CAM_DEV_UNCONFIGURED;
-				done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
-				xpt_action(done_ccb);
-				xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
-				    done_ccb);
-				xpt_release_ccb(done_ccb);
-				break;
-			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
-		}
-		goto device_fail;
-	}
-	case PROBE_PM_PID:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) == 0)
-				bzero(ident_buf, sizeof(*ident_buf));
-			softc->pm_pid = (done_ccb->ataio.res.lba_high << 24) +
-			    (done_ccb->ataio.res.lba_mid << 16) +
-			    (done_ccb->ataio.res.lba_low << 8) +
-			    done_ccb->ataio.res.sector_count;
-			((uint32_t *)ident_buf)[0] = softc->pm_pid;
-			printf("PM Product ID: %08x\n", softc->pm_pid);
-			snprintf(ident_buf->model, sizeof(ident_buf->model),
-			    "Port Multiplier %08x", softc->pm_pid);
-			PROBE_SET_ACTION(softc, PROBE_PM_PRV);
+		/*
+		 * We conservatively request only
+		 * SHORT_INQUIRY_LEN bytes of inquiry
+		 * information during our first try
+		 * at sending an INQUIRY. If the device
+		 * has more information to give,
+		 * perform a second request specifying
+		 * the amount of information the device
+		 * is willing to give.
+		 */
+		len = inq_buf->additional_length
+		    + offsetof(struct scsi_inquiry_data, additional_length) + 1;
+		if (softc->action == PROBE_INQUIRY
+		    && len > SHORT_INQUIRY_LENGTH) {
+			PROBE_SET_ACTION(softc, PROBE_FULL_INQUIRY);
 			xpt_release_ccb(done_ccb);
 			xpt_schedule(periph, priority);
 			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
 		}
-		goto device_fail;
+
+		scsi_find_quirk(path->device);
+		ata_device_transport(path);
+		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
+			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
+			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
+			xpt_action(done_ccb);
+			xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path, done_ccb);
+		}
+		break;
+	}
+	case PROBE_PM_PID:
+		if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) == 0)
+			bzero(ident_buf, sizeof(*ident_buf));
+		softc->pm_pid = (done_ccb->ataio.res.lba_high << 24) +
+		    (done_ccb->ataio.res.lba_mid << 16) +
+		    (done_ccb->ataio.res.lba_low << 8) +
+		    done_ccb->ataio.res.sector_count;
+		((uint32_t *)ident_buf)[0] = softc->pm_pid;
+		printf("PM Product ID: %08x\n", softc->pm_pid);
+		snprintf(ident_buf->model, sizeof(ident_buf->model),
+		    "Port Multiplier %08x", softc->pm_pid);
+		PROBE_SET_ACTION(softc, PROBE_PM_PRV);
+		xpt_release_ccb(done_ccb);
+		xpt_schedule(periph, priority);
+		return;
 	case PROBE_PM_PRV:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_prv = (done_ccb->ataio.res.lba_high << 24) +
-			    (done_ccb->ataio.res.lba_mid << 16) +
-			    (done_ccb->ataio.res.lba_low << 8) +
-			    done_ccb->ataio.res.sector_count;
-			((uint32_t *)ident_buf)[1] = softc->pm_prv;
-			printf("PM Revision: %08x\n", softc->pm_prv);
-			snprintf(ident_buf->revision, sizeof(ident_buf->revision),
-			    "%04x", softc->pm_prv);
-			path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
-			if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
-				path->device->flags &= ~CAM_DEV_UNCONFIGURED;
-				done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
-				xpt_action(done_ccb);
-				xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
-				    done_ccb);
-			} else {
-				done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
-				xpt_action(done_ccb);
-				xpt_async(AC_SCSI_AEN, done_ccb->ccb_h.path,
-				    done_ccb);
-			}
-			xpt_release_ccb(done_ccb);
-			break;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
+		softc->pm_prv = (done_ccb->ataio.res.lba_high << 24) +
+		    (done_ccb->ataio.res.lba_mid << 16) +
+		    (done_ccb->ataio.res.lba_low << 8) +
+		    done_ccb->ataio.res.sector_count;
+		((uint32_t *)ident_buf)[1] = softc->pm_prv;
+		printf("PM Revision: %08x\n", softc->pm_prv);
+		snprintf(ident_buf->revision, sizeof(ident_buf->revision),
+		    "%04x", softc->pm_prv);
+		path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
+		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
+			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
+			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
+			xpt_action(done_ccb);
+			xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
+			    done_ccb);
+		} else {
+			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
+			xpt_action(done_ccb);
+			xpt_async(AC_SCSI_AEN, done_ccb->ccb_h.path, done_ccb);
 		}
-		goto device_fail;
+		break;
 	case PROBE_INVALID:
 		CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_INFO,
 		    ("probedone: invalid action state\n"));
 	default:
 		break;
 	}
+done:
+	xpt_release_ccb(done_ccb);
 	done_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs);
 	TAILQ_REMOVE(&softc->request_ccbs, &done_ccb->ccb_h, periph_links.tqe);
 	done_ccb->ccb_h.status = CAM_REQ_CMP;
diff --git a/sys/cam/cam.h b/sys/cam/cam.h
index 46a5e07..c26afb4 100644
--- a/sys/cam/cam.h
+++ b/sys/cam/cam.h
@@ -66,6 +66,7 @@ struct cam_periph;
  */
 typedef struct {
 	u_int32_t priority;
+#define CAM_PRIORITY_BUS	0
 #define CAM_PRIORITY_DEV	0
 #define CAM_PRIORITY_NORMAL	1
 #define CAM_PRIORITY_NONE	(u_int32_t)-1
diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h
index a750d93..483f22b 100644
--- a/sys/cam/cam_ccb.h
+++ b/sys/cam/cam_ccb.h
@@ -816,12 +816,22 @@ struct ccb_trans_settings_sas {
 	u_int32_t 	bitrate;	/* Mbps */
 };
 
+struct ccb_trans_settings_ata {
+	u_int     	valid;		/* Which fields to honor */
+#define	CTS_ATA_VALID_MODE		0x01
+#define	CTS_ATA_VALID_BYTECOUNT		0x04
+	u_int32_t 	mode;
+	u_int 		bytecount;	/* Length of PIO transaction */
+};
+
 struct ccb_trans_settings_sata {
 	u_int     	valid;		/* Which fields to honor */
 #define	CTS_SATA_VALID_SPEED		0x01
 #define	CTS_SATA_VALID_PM		0x02
+#define	CTS_SATA_VALID_BYTECOUNT	0x04
 	u_int32_t 	bitrate;	/* Mbps */
 	u_int 		pm_present;	/* PM is present (XPT->SIM) */
+	u_int 		bytecount;	/* Length of PIO transaction */
 };
 
 /* Get/Set transfer rate/width/disconnection/tag queueing settings */
@@ -841,6 +851,7 @@ struct ccb_trans_settings {
 		struct ccb_trans_settings_spi spi;
 		struct ccb_trans_settings_fc fc;
 		struct ccb_trans_settings_sas sas;
+		struct ccb_trans_settings_ata ata;
 		struct ccb_trans_settings_sata sata;
 	} xport_specific;
 };
diff --git a/sys/cam/cam_periph.h b/sys/cam/cam_periph.h
index f95a94f..258055f 100644
--- a/sys/cam/cam_periph.h
+++ b/sys/cam/cam_periph.h
@@ -79,6 +79,8 @@ struct periph_driver {
 	char			 *driver_name;
 	TAILQ_HEAD(,cam_periph)	 units;
 	u_int			 generation;
+	u_int			 flags;
+#define CAM_PERIPH_DRV_EARLY		0x01
 };
 
 typedef enum {
diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index d3ed9aa..187683e 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -161,7 +161,8 @@ static periph_init_t xpt_periph_init;
 static struct periph_driver xpt_driver =
 {
 	xpt_periph_init, "xpt",
-	TAILQ_HEAD_INITIALIZER(xpt_driver.units)
+	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
+	CAM_PERIPH_DRV_EARLY
 };
 
 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
@@ -1102,30 +1103,35 @@ xpt_announce_periph(struct cam_periph *periph, char *announce_string)
 	speed = cpi.base_transfer_speed;
 	freq = 0;
 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SPI) {
-		struct	ccb_trans_settings_spi *spi;
+		struct	ccb_trans_settings_spi *spi =
+		    &cts.xport_specific.spi;
 
-		spi = &cts.xport_specific.spi;
 		if ((spi->valid & CTS_SPI_VALID_SYNC_OFFSET) != 0
 		  && spi->sync_offset != 0) {
 			freq = scsi_calc_syncsrate(spi->sync_period);
 			speed = freq;
 		}
-
 		if ((spi->valid & CTS_SPI_VALID_BUS_WIDTH) != 0)
 			speed *= (0x01 << spi->bus_width);
 	}
 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_FC) {
-		struct	ccb_trans_settings_fc *fc = &cts.xport_specific.fc;
+		struct	ccb_trans_settings_fc *fc =
+		    &cts.xport_specific.fc;
+
 		if (fc->valid & CTS_FC_VALID_SPEED)
 			speed = fc->bitrate;
 	}
 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SAS) {
-		struct	ccb_trans_settings_sas *sas = &cts.xport_specific.sas;
+		struct	ccb_trans_settings_sas *sas =
+		    &cts.xport_specific.sas;
+
 		if (sas->valid & CTS_SAS_VALID_SPEED)
 			speed = sas->bitrate;
 	}
 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) {
-		struct	ccb_trans_settings_sata *sata = &cts.xport_specific.sata;
+		struct	ccb_trans_settings_sata *sata =
+		    &cts.xport_specific.sata;
+
 		if (sata->valid & CTS_SATA_VALID_SPEED)
 			speed = sata->bitrate;
 	}
@@ -1173,7 +1179,20 @@ xpt_announce_periph(struct cam_periph *periph, char *announce_string)
 		if (fc->valid & CTS_FC_VALID_PORT)
 			printf(" PortID 0x%x", fc->port);
 	}
+	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_ATA) {
+		struct ccb_trans_settings_ata *ata =
+		    &cts.xport_specific.ata;
+
+		if (ata->valid & CTS_ATA_VALID_BYTECOUNT)
+			printf(" (PIO size %dbytes)", ata->bytecount);
+	}
+	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) {
+		struct ccb_trans_settings_sata *sata =
+		    &cts.xport_specific.sata;
 
+		if (sata->valid & CTS_SATA_VALID_BYTECOUNT)
+			printf(" (PIO size %dbytes)", sata->bytecount);
+	}
 	if (path->device->inq_flags & SID_CmdQue
 	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
 		printf("\n%s%d: Command Queueing enabled",
@@ -4676,6 +4695,9 @@ xptconfigfunc(struct cam_eb *bus, void *arg)
 static void
 xpt_config(void *arg)
 {
+	struct	periph_driver **p_drv;
+	int	i;
+
 	/*
 	 * Now that interrupts are enabled, go find our devices
 	 */
@@ -4709,6 +4731,13 @@ xpt_config(void *arg)
 #endif /* CAM_DEBUG_BUS */
 #endif /* CAMDEBUG */
 
+	/* Register early peripheral drivers */
+	/* XXX This will have to change when we have loadable modules */
+	p_drv = periph_drivers;
+	for (i = 0; p_drv[i] != NULL; i++) {
+		if ((p_drv[i]->flags & CAM_PERIPH_DRV_EARLY) != 0)
+			(*p_drv[i]->init)();
+	}
 	/*
 	 * Scan all installed busses.
 	 */
@@ -4759,7 +4788,8 @@ xpt_finishconfig_task(void *context, int pending)
 		/* XXX This will have to change when we have loadable modules */
 		p_drv = periph_drivers;
 		for (i = 0; p_drv[i] != NULL; i++) {
-			(*p_drv[i]->init)();
+			if ((p_drv[i]->flags & CAM_PERIPH_DRV_EARLY) == 0)
+				(*p_drv[i]->init)();
 		}
 
 		/*
diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index bb9299a..8f740dc 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -1491,8 +1491,10 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 {
 	struct da_softc *softc;
 	struct ccb_scsiio *csio;
+	u_int32_t  priority;
 
 	softc = (struct da_softc *)periph->softc;
+	priority = done_ccb->ccb_h.pinfo.priority;
 	csio = &done_ccb->csio;
 	switch (csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK) {
 	case DA_CCB_BUFFER_IO:
@@ -1610,7 +1612,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 					softc->state = DA_STATE_PROBE2;
 					free(rdcap, M_SCSIDA);
 					xpt_release_ccb(done_ccb);
-					xpt_schedule(periph, /*priority*/5);
+					xpt_schedule(periph, priority);
 					return;
 				}
 			} else {
diff --git a/sys/cam/scsi/scsi_sg.c b/sys/cam/scsi/scsi_sg.c
index 4ab038b..d47e6e9 100644
--- a/sys/cam/scsi/scsi_sg.c
+++ b/sys/cam/scsi/scsi_sg.c
@@ -510,7 +510,7 @@ sgioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 			break;
 		}
 
-		ccb = cam_periph_getccb(periph, /*priority*/5);
+		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 		csio = &ccb->csio;
 
 		error = copyin(req.cmdp, &csio->cdb_io.cdb_bytes,
@@ -729,7 +729,7 @@ sgwrite(struct cdev *dev, struct uio *uio, int ioflag)
 
 	cam_periph_lock(periph);
 	sc = periph->softc;
-	xpt_setup_ccb(&ccb->ccb_h, periph->path, /*priority*/5);
+	xpt_setup_ccb(&ccb->ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 	cam_fill_csio(csio,
 		      /*retries*/1,
 		      sgdone,
diff --git a/sys/cam/scsi/scsi_xpt.c b/sys/cam/scsi/scsi_xpt.c
index 35a5ac2..a240db8 100644
--- a/sys/cam/scsi/scsi_xpt.c
+++ b/sys/cam/scsi/scsi_xpt.c
@@ -110,7 +110,8 @@ static periph_init_t probe_periph_init;
 static struct periph_driver probe_driver =
 {
 	probe_periph_init, "probe",
-	TAILQ_HEAD_INITIALIZER(probe_driver.units)
+	TAILQ_HEAD_INITIALIZER(probe_driver.units), /* generation */ 0,
+	CAM_PERIPH_DRV_EARLY
 };
 
 PERIPHDRIVER_DECLARE(probe, probe_driver);