10 files changed, 450 insertions, 515 deletions

diff --git a/sys/cam/ata/ata_da.c b/sys/cam/ata/ata_da.c
index 259d49d42522..76ed3d2b57fa 100644
--- a/sys/cam/ata/ata_da.c
+++ b/sys/cam/ata/ata_da.c
@@ -63,8 +63,7 @@ __FBSDID("$FreeBSD$");
 #define ATA_MAX_28BIT_LBA               268435455UL
 
 typedef enum {
-	ADA_STATE_NORMAL,
-	ADA_STATE_SET_MULTI
+	ADA_STATE_NORMAL
 } ada_state;
 
 typedef enum {
@@ -84,7 +83,6 @@ typedef enum {
 } ada_quirks;
 
 typedef enum {
-	ADA_CCB_SET_MULTI	= 0x01,
 	ADA_CCB_BUFFER_IO	= 0x03,
 	ADA_CCB_WAITING		= 0x04,
 	ADA_CCB_DUMP		= 0x05,
@@ -112,7 +110,6 @@ struct ada_softc {
 	ada_quirks quirks;
 	int	 ordered_tag_count;
 	int	 outstanding_cmds;
-	int	 secsperint;
 	struct	 disk_params params;
 	struct	 disk *disk;
 	union	 ccb saved_ccb;
@@ -550,22 +547,6 @@ adaasync(void *callback_arg, u_int32_t code,
 				"due to status 0x%x\n", status);
 		break;
 	}
-	case AC_SENT_BDR:
-	case AC_BUS_RESET:
-	{
-		struct ada_softc *softc = (struct ada_softc *)periph->softc;
-
-		cam_periph_async(periph, code, path, arg);
-		if (softc->state != ADA_STATE_NORMAL)
-			break;
-		/*
-		 * Restore device configuration.
-		 */
-		softc->state = ADA_STATE_SET_MULTI;
-		cam_periph_acquire(periph);
-		xpt_schedule(periph, CAM_PRIORITY_DEV);
-		break;
-	}
 	default:
 		cam_periph_async(periph, code, path, arg);
 		break;
@@ -644,8 +625,7 @@ adaregister(struct cam_periph *periph, void *arg)
 	if (cgd->ident_data.satacapabilities & ATA_SUPPORT_NCQ &&
 	    cgd->ident_data.queue >= 31)
 		softc->flags |= ADA_FLAG_CAN_NCQ;
-	softc->secsperint = max(1, min(cgd->ident_data.sectors_intr, 16));
-	softc->state = ADA_STATE_SET_MULTI;
+	softc->state = ADA_STATE_NORMAL;
 
 	periph->softc = softc;
 
@@ -734,18 +714,10 @@ adaregister(struct cam_periph *periph, void *arg)
 	 * them and the only alternative would be to
 	 * not attach the device on failure.
 	 */
-	xpt_register_async(AC_SENT_BDR | AC_BUS_RESET | AC_LOST_DEVICE,
+	xpt_register_async(AC_LOST_DEVICE,
 			   adaasync, periph, periph->path);
 
 	/*
-	 * Take an exclusive refcount on the periph while adastart is called
-	 * to finish the probe.  The reference will be dropped in adadone at
-	 * the end of probe.
-	 */
-	cam_periph_acquire(periph);
-	xpt_schedule(periph, /*priority*/5);
-
-	/*
 	 * Schedule a periodic event to occasionally send an
 	 * ordered tag to a device.
 	 */
@@ -901,21 +873,6 @@ adastart(struct cam_periph *periph, union ccb *start_ccb)
 		}
 		break;
 	}
-	case ADA_STATE_SET_MULTI:
-	{
-		cam_fill_ataio(ataio,
-		    ada_retry_count,
-		    adadone,
-		    CAM_DIR_NONE,
-		    0,
-		    NULL,
-		    0,
-		    ada_default_timeout*1000);
-
-		ata_28bit_cmd(ataio, ATA_SET_MULTI, 0, 0, softc->secsperint);
-		start_ccb->ccb_h.ccb_state = ADA_CCB_SET_MULTI;
-		xpt_action(start_ccb);
-	}
 	}
 }
 
@@ -1003,35 +960,6 @@ adadone(struct cam_periph *periph, union ccb *done_ccb)
 		wakeup(&done_ccb->ccb_h.cbfcnp);
 		return;
 	}
-	case ADA_CCB_SET_MULTI:
-	{
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-		} else {
-			int	error;
-
-			error = adaerror(done_ccb, 0, 0);
-			if (error == ERESTART) {
-				/* A retry was scheduled, so just return. */
-				return;
-			}
-		}
-		softc->state = ADA_STATE_NORMAL;
-		/*
-		 * Since our peripheral may be invalidated by an error
-		 * above or an external event, we must release our CCB
-		 * before releasing the probe lock on the peripheral.
-		 * The peripheral will only go away once the last lock
-		 * is removed, and we need it around for the CCB release
-		 * operation.
-		 */
-		xpt_release_ccb(done_ccb);
-		if (bioq_first(&softc->bio_queue) != NULL) {
-			/* Have more work to do, so ensure we stay scheduled */
-			xpt_schedule(periph, CAM_PRIORITY_NORMAL);
-		}
-		cam_periph_release_locked(periph);
-		return;
-	}
 	case ADA_CCB_DUMP:
 		/* No-op.  We're polling */
 		return;
diff --git a/sys/cam/ata/ata_pmp.c b/sys/cam/ata/ata_pmp.c
index 76b3c0ad8b68..ea387909b113 100644
--- a/sys/cam/ata/ata_pmp.c
+++ b/sys/cam/ata/ata_pmp.c
@@ -93,7 +93,9 @@ struct pmp_softc {
 	int			pm_step;
 	int			pm_try;
 	int			found;
+	int			reset;
 	int			frozen;
+	int			restart;
 	union			ccb saved_ccb;
 	struct task		sysctl_task;
 	struct sysctl_ctx_list	sysctl_ctx;
@@ -134,7 +136,8 @@ TUNABLE_INT("kern.cam.pmp.default_timeout", &pmp_default_timeout);
 static struct periph_driver pmpdriver =
 {
 	pmpinit, "pmp",
-	TAILQ_HEAD_INITIALIZER(pmpdriver.units), /* generation */ 0
+	TAILQ_HEAD_INITIALIZER(pmpdriver.units), /* generation */ 0,
+	CAM_PERIPH_DRV_EARLY
 };
 
 PERIPHDRIVER_DECLARE(pmp, pmpdriver);
@@ -292,14 +295,21 @@ pmpasync(void *callback_arg, u_int32_t code,
 	case AC_BUS_RESET:
 		softc = (struct pmp_softc *)periph->softc;
 		cam_periph_async(periph, code, path, arg);
-		if (softc->state != PMP_STATE_NORMAL)
+		if (code == AC_SCSI_AEN && softc->state != PMP_STATE_NORMAL &&
+		    softc->state != PMP_STATE_SCAN)
 			break;
-		pmpfreeze(periph, softc->found);
+		if (softc->state != PMP_STATE_SCAN)
+			pmpfreeze(periph, softc->found);
+		else
+			pmpfreeze(periph, softc->found & ~(1 << softc->pm_step));
 		if (code == AC_SENT_BDR || code == AC_BUS_RESET)
 			softc->found = 0; /* We have to reset everything. */
-		softc->state = PMP_STATE_PORTS;
-		cam_periph_acquire(periph);
-		xpt_schedule(periph, CAM_PRIORITY_DEV);
+		if (softc->state == PMP_STATE_NORMAL) {
+			softc->state = PMP_STATE_PORTS;
+			cam_periph_acquire(periph);
+			xpt_schedule(periph, CAM_PRIORITY_BUS);
+		} else
+			softc->restart = 1;
 		break;
 	default:
 		cam_periph_async(periph, code, path, arg);
@@ -395,7 +405,7 @@ pmpregister(struct cam_periph *periph, void *arg)
 	 * the end of probe.
 	 */
 	(void)cam_periph_acquire(periph);
-	xpt_schedule(periph, CAM_PRIORITY_DEV);
+	xpt_schedule(periph, CAM_PRIORITY_BUS);
 
 	return(CAM_REQ_CMP);
 }
@@ -408,6 +418,11 @@ pmpstart(struct cam_periph *periph, union ccb *start_ccb)
 
 	softc = (struct pmp_softc *)periph->softc;
 	ataio = &start_ccb->ataio;
+	
+	if (softc->restart) {
+		softc->restart = 0;
+		softc->state = PMP_STATE_PORTS;
+	}
 
 	switch (softc->state) {
 	case PMP_STATE_PORTS:
@@ -469,6 +484,7 @@ printf("PM RESET %d%s\n", softc->pm_step,
 		ata_pm_read_cmd(ataio, 0, softc->pm_step);
 		break;
 	case PMP_STATE_CLEAR:
+		softc->reset = 0;
 		cam_fill_ataio(ataio,
 		      pmp_retry_count,
 		      pmpdone,
@@ -492,7 +508,7 @@ pmpdone(struct cam_periph *periph, union ccb *done_ccb)
 	struct ccb_ataio *ataio;
 	union ccb *work_ccb;
 	struct cam_path *path, *dpath;
-	u_int32_t  priority;
+	u_int32_t  priority, res;
 
 	softc = (struct pmp_softc *)periph->softc;
 	ataio = &done_ccb->ataio;
@@ -502,193 +518,158 @@ pmpdone(struct cam_periph *periph, union ccb *done_ccb)
 	path = done_ccb->ccb_h.path;
 	priority = done_ccb->ccb_h.pinfo.priority;
 
-	switch (softc->state) {
-	case PMP_STATE_PORTS:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_ports = (done_ccb->ataio.res.lba_high << 24) +
-			    (done_ccb->ataio.res.lba_mid << 16) +
-			    (done_ccb->ataio.res.lba_low << 8) +
-			    done_ccb->ataio.res.sector_count;
-			/* This PM declares 6 ports, while only 5 of them are real.
-			 * Port 5 is enclosure management bridge port, which has implementation
-			 * problems, causing probe faults. Hide it for now. */
-			if (softc->pm_pid == 0x37261095 && softc->pm_ports == 6)
-				softc->pm_ports = 5;
-			/* This PM declares 7 ports, while only 5 of them are real.
-			 * Port 5 is some fake "Config  Disk" with 640 sectors size,
-			 * port 6 is enclosure management bridge port.
-			 * Both fake ports has implementation problems, causing
-			 * probe faults. Hide them for now. */
-			if (softc->pm_pid == 0x47261095 && softc->pm_ports == 7)
-				softc->pm_ports = 5;
-			printf("PM ports: %d\n", softc->pm_ports);
-			softc->state = PMP_STATE_CONFIG;
-			xpt_release_ccb(done_ccb);
-			xpt_schedule(periph, priority);
-			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
+	if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
+		if (cam_periph_error(done_ccb, 0, 0,
+		    &softc->saved_ccb) == ERESTART) {
 			return;
 		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+			cam_release_devq(done_ccb->ccb_h.path,
+			    /*relsim_flags*/0,
+			    /*reduction*/0,
+			    /*timeout*/0,
+			    /*getcount_only*/0);
+		}
+		goto done;
+	}
+
+	if (softc->restart) {
+		softc->restart = 0;
+		if (softc->state == PMP_STATE_SCAN) {
+			pmpfreeze(periph, 1 << softc->pm_step);
+			work_ccb = done_ccb;
+			done_ccb = (union ccb*)work_ccb->ccb_h.ppriv_ptr0;
+			/* Free the current request path- we're done with it. */
+		    	xpt_free_path(work_ccb->ccb_h.path);
+			xpt_free_ccb(work_ccb);
 		}
 		xpt_release_ccb(done_ccb);
-		break;
+		softc->state = PMP_STATE_PORTS;
+		xpt_schedule(periph, priority);
+		return;
+	}
+
+	switch (softc->state) {
+	case PMP_STATE_PORTS:
+		softc->pm_ports = (done_ccb->ataio.res.lba_high << 24) +
+		    (done_ccb->ataio.res.lba_mid << 16) +
+		    (done_ccb->ataio.res.lba_low << 8) +
+		    done_ccb->ataio.res.sector_count;
+		/* This PM declares 6 ports, while only 5 of them are real.
+		 * Port 5 is enclosure management bridge port, which has implementation
+		 * problems, causing probe faults. Hide it for now. */
+		if (softc->pm_pid == 0x37261095 && softc->pm_ports == 6)
+			softc->pm_ports = 5;
+		/* This PM declares 7 ports, while only 5 of them are real.
+		 * Port 5 is some fake "Config  Disk" with 640 sectors size,
+		 * port 6 is enclosure management bridge port.
+		 * Both fake ports has implementation problems, causing
+		 * probe faults. Hide them for now. */
+		if (softc->pm_pid == 0x47261095 && softc->pm_ports == 7)
+			softc->pm_ports = 5;
+		printf("PM ports: %d\n", softc->pm_ports);
+		softc->state = PMP_STATE_CONFIG;
+		xpt_release_ccb(done_ccb);
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_CONFIG:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_step = 0;
-			softc->state = PMP_STATE_RESET;
-			xpt_release_ccb(done_ccb);
-			xpt_schedule(periph, priority);
-			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
-		}
+		softc->pm_step = 0;
+		softc->state = PMP_STATE_RESET;
+		softc->reset |= ~softc->found;
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_RESET:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_step++;
-			if (softc->pm_step < softc->pm_ports) {
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			} else {
-				softc->pm_step = 0;
-				DELAY(5000);
-				printf("PM reset done\n");
-				softc->state = PMP_STATE_CONNECT;
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+		softc->pm_step++;
+		if (softc->pm_step >= softc->pm_ports) {
+			softc->pm_step = 0;
+			cam_freeze_devq(periph->path);
+			cam_release_devq(periph->path,
+			    RELSIM_RELEASE_AFTER_TIMEOUT,
+			    /*reduction*/0,
+			    /*timeout*/5,
+			    /*getcount_only*/0);
+			printf("PM reset done\n");
+			softc->state = PMP_STATE_CONNECT;
 		}
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_CONNECT:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_step++;
-			if (softc->pm_step < softc->pm_ports) {
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			} else {
-				softc->pm_step = 0;
-				softc->pm_try = 0;
-				printf("PM connect done\n");
-				softc->state = PMP_STATE_CHECK;
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+		softc->pm_step++;
+		if (softc->pm_step >= softc->pm_ports) {
+			softc->pm_step = 0;
+			softc->pm_try = 0;
+			cam_freeze_devq(periph->path);
+			cam_release_devq(periph->path,
+			    RELSIM_RELEASE_AFTER_TIMEOUT,
+			    /*reduction*/0,
+			    /*timeout*/10,
+			    /*getcount_only*/0);
+			printf("PM connect done\n");
+			softc->state = PMP_STATE_CHECK;
 		}
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_CHECK:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			int res = (done_ccb->ataio.res.lba_high << 24) +
-			    (done_ccb->ataio.res.lba_mid << 16) +
-			    (done_ccb->ataio.res.lba_low << 8) +
-			    done_ccb->ataio.res.sector_count;
-			if ((res & 0xf0f) == 0x103 && (res & 0x0f0) != 0) {
-				printf("PM status: %d - %08x\n", softc->pm_step, res);
-				softc->found |= (1 << softc->pm_step);
-				softc->pm_step++;
+		res = (done_ccb->ataio.res.lba_high << 24) +
+		    (done_ccb->ataio.res.lba_mid << 16) +
+		    (done_ccb->ataio.res.lba_low << 8) +
+		    done_ccb->ataio.res.sector_count;
+		if ((res & 0xf0f) == 0x103 && (res & 0x0f0) != 0) {
+			printf("PM status: %d - %08x\n", softc->pm_step, res);
+			softc->found |= (1 << softc->pm_step);
+			softc->pm_step++;
+		} else {
+			if (softc->pm_try < 10) {
+				cam_freeze_devq(periph->path);
+				cam_release_devq(periph->path,
+				    RELSIM_RELEASE_AFTER_TIMEOUT,
+				    /*reduction*/0,
+				    /*timeout*/10,
+				    /*getcount_only*/0);
+				softc->pm_try++;
 			} else {
-				if (softc->pm_try < 100) {
-					DELAY(10000);
-					softc->pm_try++;
-				} else {
-					printf("PM status: %d - %08x\n", softc->pm_step, res);
-					softc->found &= ~(1 << softc->pm_step);
-					if (xpt_create_path(&dpath, periph,
-					    done_ccb->ccb_h.path_id,
-					    softc->pm_step, 0) == CAM_REQ_CMP) {
-						xpt_async(AC_LOST_DEVICE, dpath, NULL);
-						xpt_free_path(dpath);
-					}
-					softc->pm_step++;
+				printf("PM status: %d - %08x\n", softc->pm_step, res);
+				softc->found &= ~(1 << softc->pm_step);
+				if (xpt_create_path(&dpath, periph,
+				    done_ccb->ccb_h.path_id,
+				    softc->pm_step, 0) == CAM_REQ_CMP) {
+					xpt_async(AC_LOST_DEVICE, dpath, NULL);
+					xpt_free_path(dpath);
 				}
+				softc->pm_step++;
 			}
-			if (softc->pm_step < softc->pm_ports) {
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			} else {
-				softc->pm_step = 0;
-				softc->state = PMP_STATE_CLEAR;
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
+		}
+		if (softc->pm_step >= softc->pm_ports) {
+			if (softc->reset & softc->found) {
+				cam_freeze_devq(periph->path);
+				cam_release_devq(periph->path,
+				    RELSIM_RELEASE_AFTER_TIMEOUT,
+				    /*reduction*/0,
+				    /*timeout*/1000,
+				    /*getcount_only*/0);
 			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+			softc->state = PMP_STATE_CLEAR;
+			softc->pm_step = 0;
 		}
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	case PMP_STATE_CLEAR:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_step++;
-			if (softc->pm_step < softc->pm_ports) {
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			} else if (softc->found) {
-				softc->pm_step = 0;
-				softc->state = PMP_STATE_SCAN;
-				work_ccb = xpt_alloc_ccb_nowait();
-				if (work_ccb != NULL)
-					goto do_scan;
-				xpt_release_ccb(done_ccb);
-			}
-			break;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
+		softc->pm_step++;
+		if (softc->pm_step < softc->pm_ports) {
+			xpt_release_ccb(done_ccb);
+			xpt_schedule(periph, priority);
 			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-				cam_release_devq(done_ccb->ccb_h.path,
-						 /*relsim_flags*/0,
-						 /*reduction*/0,
-						 /*timeout*/0,
-						 /*getcount_only*/0);
+		} else if (softc->found) {
+			softc->pm_step = 0;
+			softc->state = PMP_STATE_SCAN;
+			work_ccb = xpt_alloc_ccb_nowait();
+			if (work_ccb != NULL)
+				goto do_scan;
+			xpt_release_ccb(done_ccb);
 		}
-		xpt_release_ccb(done_ccb);
 		break;
 	case PMP_STATE_SCAN:
 		work_ccb = done_ccb;
@@ -703,7 +684,6 @@ do_scan:
 		}
 		if (softc->pm_step >= softc->pm_ports) {
 			xpt_free_ccb(work_ccb);
-			xpt_release_ccb(done_ccb);
 			break;
 		}
 		if (xpt_create_path(&dpath, periph,
@@ -712,7 +692,6 @@ do_scan:
 			printf("pmpdone: xpt_create_path failed"
 			    ", bus scan halted\n");
 			xpt_free_ccb(work_ccb);
-			xpt_release_ccb(done_ccb);
 			break;
 		}
 		xpt_setup_ccb(&work_ccb->ccb_h, dpath,
@@ -727,6 +706,8 @@ do_scan:
 	default:
 		break;
 	}
+done:
+	xpt_release_ccb(done_ccb);
 	softc->state = PMP_STATE_NORMAL;
 	pmprelease(periph, -1);
 	cam_periph_release_locked(periph);
diff --git a/sys/cam/ata/ata_xpt.c b/sys/cam/ata/ata_xpt.c
index a693e2cdc638..9e781cd72d6d 100644
--- a/sys/cam/ata/ata_xpt.c
+++ b/sys/cam/ata/ata_xpt.c
@@ -83,7 +83,8 @@ static periph_init_t probe_periph_init;
 static struct periph_driver probe_driver =
 {
 	probe_periph_init, "aprobe",
-	TAILQ_HEAD_INITIALIZER(probe_driver.units)
+	TAILQ_HEAD_INITIALIZER(probe_driver.units), /* generation */ 0,
+	CAM_PERIPH_DRV_EARLY
 };
 
 PERIPHDRIVER_DECLARE(aprobe, probe_driver);
@@ -92,6 +93,7 @@ typedef enum {
 	PROBE_RESET,
 	PROBE_IDENTIFY,
 	PROBE_SETMODE,
+	PROBE_SET_MULTI,
 	PROBE_INQUIRY,
 	PROBE_FULL_INQUIRY,
 	PROBE_PM_PID,
@@ -103,6 +105,7 @@ static char *probe_action_text[] = {
 	"PROBE_RESET",
 	"PROBE_IDENTIFY",
 	"PROBE_SETMODE",
+	"PROBE_SET_MULTI",
 	"PROBE_INQUIRY",
 	"PROBE_FULL_INQUIRY",
 	"PROBE_PM_PID",
@@ -282,12 +285,16 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 	struct ccb_ataio *ataio;
 	struct ccb_scsiio *csio;
 	probe_softc *softc;
+	struct cam_path *path;
+	struct ata_params *ident_buf;
 
 	CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_TRACE, ("probestart\n"));
 
 	softc = (probe_softc *)periph->softc;
+	path = start_ccb->ccb_h.path;
 	ataio = &start_ccb->ataio;
 	csio = &start_ccb->csio;
+	ident_buf = &periph->path->device->ident_data;
 
 	switch (softc->action) {
 	case PROBE_RESET:
@@ -302,10 +309,6 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 		ata_reset_cmd(ataio);
 		break;
 	case PROBE_IDENTIFY:
-	{
-		struct ata_params *ident_buf =
-		    &periph->path->device->ident_data;
-
 		if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
 			/* Prepare check that it is the same device. */
 			MD5_CTX context;
@@ -335,12 +338,7 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 		else
 			ata_28bit_cmd(ataio, ATA_ATAPI_IDENTIFY, 0, 0, 0);
 		break;
-	}
 	case PROBE_SETMODE:
-	{
-		struct ata_params *ident_buf =
-		    &periph->path->device->ident_data;
-
 		cam_fill_ataio(ataio,
 		      1,
 		      probedone,
@@ -352,6 +350,37 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 		ata_28bit_cmd(ataio, ATA_SETFEATURES, ATA_SF_SETXFER, 0,
 		    ata_max_mode(ident_buf, ATA_UDMA6, ATA_UDMA6));
 		break;
+	case PROBE_SET_MULTI:
+	{
+		struct ccb_trans_settings cts;
+		u_int sectors;
+
+		sectors = max(1, min(ident_buf->sectors_intr & 0xff, 16));
+
+		/* Report bytecount to SIM. */
+		bzero(&cts, sizeof(cts));
+		xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
+		cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
+		cts.type = CTS_TYPE_CURRENT_SETTINGS;
+		if (path->device->transport == XPORT_ATA) {
+			cts.xport_specific.ata.bytecount = sectors * 512;
+			cts.xport_specific.ata.valid = CTS_ATA_VALID_BYTECOUNT;
+		} else {
+			cts.xport_specific.sata.bytecount = sectors * 512;
+			cts.xport_specific.sata.valid = CTS_SATA_VALID_BYTECOUNT;
+		}
+		xpt_action((union ccb *)&cts);
+
+		cam_fill_ataio(ataio,
+		    1,
+		    probedone,
+		    CAM_DIR_NONE,
+		    0,
+		    NULL,
+		    0,
+		    30*1000);
+		ata_28bit_cmd(ataio, ATA_SET_MULTI, 0, 0, sectors);
+		break;
 	}
 	case PROBE_INQUIRY:
 	case PROBE_FULL_INQUIRY:
@@ -406,7 +435,7 @@ probestart(struct cam_periph *periph, union ccb *start_ccb)
 		ata_pm_read_cmd(ataio, 1, 15);
 		break;
 	case PROBE_INVALID:
-		CAM_DEBUG(start_ccb->ccb_h.path, CAM_DEBUG_INFO,
+		CAM_DEBUG(path, CAM_DEBUG_INFO,
 		    ("probestart: invalid action state\n"));
 	default:
 		break;
@@ -552,101 +581,119 @@ probedone(struct cam_periph *periph, union ccb *done_ccb)
 	priority = done_ccb->ccb_h.pinfo.priority;
 	ident_buf = &path->device->ident_data;
 
-	switch (softc->action) {
-	case PROBE_RESET:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			int sign = (done_ccb->ataio.res.lba_high << 8) +
-			    done_ccb->ataio.res.lba_mid;
-			xpt_print(path, "SIGNATURE: %04x\n", sign);
-			if (sign == 0x0000 &&
-			    done_ccb->ccb_h.target_id != 15) {
-				path->device->protocol = PROTO_ATA;
-				PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
-			} else if (sign == 0x9669 &&
-			    done_ccb->ccb_h.target_id == 15) {
-				struct ccb_trans_settings cts;
-
-				/* Report SIM that PM is present. */
-				bzero(&cts, sizeof(cts));
-				xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
-				cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
-				cts.type = CTS_TYPE_CURRENT_SETTINGS;
-				cts.xport_specific.sata.pm_present = 1;
-				cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
-				xpt_action((union ccb *)&cts);
-				path->device->protocol = PROTO_SATAPM;
-				PROBE_SET_ACTION(softc, PROBE_PM_PID);
-			} else if (sign == 0xeb14 &&
-			    done_ccb->ccb_h.target_id != 15) {
-				path->device->protocol = PROTO_SCSI;
-				PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
-			} else {
-				if (done_ccb->ccb_h.target_id != 15) {
-					xpt_print(path,
-					    "Unexpected signature 0x%04x\n", sign);
-				}
-				goto device_fail;
-			}
-			xpt_release_ccb(done_ccb);
-			xpt_schedule(periph, priority);
-			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
+	if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP) {
+device_fail:	if (cam_periph_error(done_ccb, 0, 0,
+		    &softc->saved_ccb) == ERESTART) {
 			return;
 		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
 			/* Don't wedge the queue */
 			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
 					 /*run_queue*/TRUE);
 		}
-		goto device_fail;
-	case PROBE_IDENTIFY:
+		/* Old PIO2 devices may not support mode setting. */
+		if (softc->action == PROBE_SETMODE &&
+		    ata_max_pmode(ident_buf) <= ATA_PIO2 &&
+		    (ident_buf->capabilities1 & ATA_SUPPORT_IORDY) == 0)
+			goto noerror;
+		/*
+		 * If we get to this point, we got an error status back
+		 * from the inquiry and the error status doesn't require
+		 * automatically retrying the command.  Therefore, the
+		 * inquiry failed.  If we had inquiry information before
+		 * for this device, but this latest inquiry command failed,
+		 * the device has probably gone away.  If this device isn't
+		 * already marked unconfigured, notify the peripheral
+		 * drivers that this device is no more.
+		 */
+		if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
+			xpt_async(AC_LOST_DEVICE, path, NULL);
+		found = 0;
+		goto done;
+	}
+noerror:
+	switch (softc->action) {
+	case PROBE_RESET:
 	{
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			int16_t *ptr;
+		int sign = (done_ccb->ataio.res.lba_high << 8) +
+		    done_ccb->ataio.res.lba_mid;
+		xpt_print(path, "SIGNATURE: %04x\n", sign);
+		if (sign == 0x0000 &&
+		    done_ccb->ccb_h.target_id != 15) {
+			path->device->protocol = PROTO_ATA;
+			PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
+		} else if (sign == 0x9669 &&
+		    done_ccb->ccb_h.target_id == 15) {
+			struct ccb_trans_settings cts;
 
-			for (ptr = (int16_t *)ident_buf;
-			     ptr < (int16_t *)ident_buf + sizeof(struct ata_params)/2; ptr++) {
-				*ptr = le16toh(*ptr);
-			}
-			if (strncmp(ident_buf->model, "FX", 2) &&
-			    strncmp(ident_buf->model, "NEC", 3) &&
-			    strncmp(ident_buf->model, "Pioneer", 7) &&
-			    strncmp(ident_buf->model, "SHARP", 5)) {
-				ata_bswap(ident_buf->model, sizeof(ident_buf->model));
-				ata_bswap(ident_buf->revision, sizeof(ident_buf->revision));
-				ata_bswap(ident_buf->serial, sizeof(ident_buf->serial));
+				/* Report SIM that PM is present. */
+			bzero(&cts, sizeof(cts));
+			xpt_setup_ccb(&cts.ccb_h, path, CAM_PRIORITY_NORMAL);
+			cts.ccb_h.func_code = XPT_SET_TRAN_SETTINGS;
+			cts.type = CTS_TYPE_CURRENT_SETTINGS;
+			cts.xport_specific.sata.pm_present = 1;
+			cts.xport_specific.sata.valid = CTS_SATA_VALID_PM;
+			xpt_action((union ccb *)&cts);
+			path->device->protocol = PROTO_SATAPM;
+			PROBE_SET_ACTION(softc, PROBE_PM_PID);
+		} else if (sign == 0xeb14 &&
+		    done_ccb->ccb_h.target_id != 15) {
+			path->device->protocol = PROTO_SCSI;
+			PROBE_SET_ACTION(softc, PROBE_IDENTIFY);
+		} else {
+			if (done_ccb->ccb_h.target_id != 15) {
+				xpt_print(path,
+				    "Unexpected signature 0x%04x\n", sign);
 			}
-			ata_btrim(ident_buf->model, sizeof(ident_buf->model));
-			ata_bpack(ident_buf->model, ident_buf->model, sizeof(ident_buf->model));
-			ata_btrim(ident_buf->revision, sizeof(ident_buf->revision));
-			ata_bpack(ident_buf->revision, ident_buf->revision, sizeof(ident_buf->revision));
-			ata_btrim(ident_buf->serial, sizeof(ident_buf->serial));
-			ata_bpack(ident_buf->serial, ident_buf->serial, sizeof(ident_buf->serial));
+			goto device_fail;
+		}
+		xpt_release_ccb(done_ccb);
+		xpt_schedule(periph, priority);
+		return;
+	}
+	case PROBE_IDENTIFY:
+	{
+		int16_t *ptr;
 
-			if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
-				/* Check that it is the same device. */
-				MD5_CTX context;
-				u_int8_t digest[16];
+		for (ptr = (int16_t *)ident_buf;
+		     ptr < (int16_t *)ident_buf + sizeof(struct ata_params)/2; ptr++) {
+			*ptr = le16toh(*ptr);
+		}
+		if (strncmp(ident_buf->model, "FX", 2) &&
+		    strncmp(ident_buf->model, "NEC", 3) &&
+		    strncmp(ident_buf->model, "Pioneer", 7) &&
+		    strncmp(ident_buf->model, "SHARP", 5)) {
+			ata_bswap(ident_buf->model, sizeof(ident_buf->model));
+			ata_bswap(ident_buf->revision, sizeof(ident_buf->revision));
+			ata_bswap(ident_buf->serial, sizeof(ident_buf->serial));
+		}
+		ata_btrim(ident_buf->model, sizeof(ident_buf->model));
+		ata_bpack(ident_buf->model, ident_buf->model, sizeof(ident_buf->model));
+		ata_btrim(ident_buf->revision, sizeof(ident_buf->revision));
+		ata_bpack(ident_buf->revision, ident_buf->revision, sizeof(ident_buf->revision));
+		ata_btrim(ident_buf->serial, sizeof(ident_buf->serial));
+		ata_bpack(ident_buf->serial, ident_buf->serial, sizeof(ident_buf->serial));
 
-				MD5Init(&context);
-				MD5Update(&context,
-				    (unsigned char *)ident_buf->model,
-				    sizeof(ident_buf->model));
-				MD5Update(&context,
-				    (unsigned char *)ident_buf->revision,
-				    sizeof(ident_buf->revision));
-				MD5Update(&context,
-				    (unsigned char *)ident_buf->serial,
-				    sizeof(ident_buf->serial));
-				MD5Final(digest, &context);
-				if (bcmp(digest, softc->digest, sizeof(digest))) {
-					/* Device changed. */
-					xpt_async(AC_LOST_DEVICE, path, NULL);
-				}
-				xpt_release_ccb(done_ccb);
-				break;
-			}
+		if ((periph->path->device->flags & CAM_DEV_UNCONFIGURED) == 0) {
+			/* Check that it is the same device. */
+			MD5_CTX context;
+			u_int8_t digest[16];
 
+			MD5Init(&context);
+			MD5Update(&context,
+			    (unsigned char *)ident_buf->model,
+			    sizeof(ident_buf->model));
+			MD5Update(&context,
+			    (unsigned char *)ident_buf->revision,
+			    sizeof(ident_buf->revision));
+			MD5Update(&context,
+			    (unsigned char *)ident_buf->serial,
+			    sizeof(ident_buf->serial));
+			MD5Final(digest, &context);
+			if (bcmp(digest, softc->digest, sizeof(digest))) {
+				/* Device changed. */
+				xpt_async(AC_LOST_DEVICE, path, NULL);
+			}
+		} else {
 			/* Clean up from previous instance of this device */
 			if (path->device->serial_num != NULL) {
 				free(path->device->serial_num, M_CAMXPT);
@@ -655,7 +702,7 @@ probedone(struct cam_periph *periph, union ccb *done_ccb)
 			}
 			path->device->serial_num =
 				(u_int8_t *)malloc((sizeof(ident_buf->serial) + 1),
-						   M_CAMXPT, M_NOWAIT);
+					   M_CAMXPT, M_NOWAIT);
 			if (path->device->serial_num != NULL) {
 				bcopy(ident_buf->serial,
 				      path->device->serial_num,
@@ -667,188 +714,120 @@ probedone(struct cam_periph *periph, union ccb *done_ccb)
 			}
 
 			path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
-			ata_device_transport(path);
-			PROBE_SET_ACTION(softc, PROBE_SETMODE);
-			xpt_release_ccb(done_ccb);
-			xpt_schedule(periph, priority);
-			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
 		}
-device_fail:
-		/*
-		 * If we get to this point, we got an error status back
-		 * from the inquiry and the error status doesn't require
-		 * automatically retrying the command.  Therefore, the
-		 * inquiry failed.  If we had inquiry information before
-		 * for this device, but this latest inquiry command failed,
-		 * the device has probably gone away.  If this device isn't
-		 * already marked unconfigured, notify the peripheral
-		 * drivers that this device is no more.
-		 */
-		if ((path->device->flags & CAM_DEV_UNCONFIGURED) == 0)
-			xpt_async(AC_LOST_DEVICE, path, NULL);
-		found = 0;
+		ata_device_transport(path);
+		PROBE_SET_ACTION(softc, PROBE_SETMODE);
 		xpt_release_ccb(done_ccb);
-		break;
+		xpt_schedule(periph, priority);
+		return;
 	}
 	case PROBE_SETMODE:
-	{
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-modedone:		if (path->device->protocol == PROTO_ATA) {
-				path->device->flags &= ~CAM_DEV_UNCONFIGURED;
-				done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
-				xpt_action(done_ccb);
-				xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
-				    done_ccb);
-				xpt_release_ccb(done_ccb);
-				break;
-			} else {
-				PROBE_SET_ACTION(softc, PROBE_INQUIRY);
-				xpt_release_ccb(done_ccb);
-				xpt_schedule(periph, priority);
-				return;
-			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
+		if (path->device->protocol == PROTO_ATA) {
+			PROBE_SET_ACTION(softc, PROBE_SET_MULTI);
+		} else {
+			PROBE_SET_ACTION(softc, PROBE_INQUIRY);
 		}
-		/* Old PIO2 devices may not support mode setting. */
-		if (ata_max_pmode(ident_buf) <= ATA_PIO2 &&
-		    (ident_buf->capabilities1 & ATA_SUPPORT_IORDY) == 0)
-			goto modedone;
-		goto device_fail;
-	}
+		xpt_release_ccb(done_ccb);
+		xpt_schedule(periph, priority);
+		return;
+	case PROBE_SET_MULTI:
+		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
+			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
+			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
+			xpt_action(done_ccb);
+			xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
+			    done_ccb);
+		}
+		break;
 	case PROBE_INQUIRY:
 	case PROBE_FULL_INQUIRY:
 	{
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			struct scsi_inquiry_data *inq_buf;
-			u_int8_t periph_qual;
-
-			path->device->flags |= CAM_DEV_INQUIRY_DATA_VALID;
-			inq_buf = &path->device->inq_data;
+		struct scsi_inquiry_data *inq_buf;
+		u_int8_t periph_qual, len;
 
-			periph_qual = SID_QUAL(inq_buf);
+		path->device->flags |= CAM_DEV_INQUIRY_DATA_VALID;
+		inq_buf = &path->device->inq_data;
 
-			if (periph_qual == SID_QUAL_LU_CONNECTED) {
-				u_int8_t len;
+		periph_qual = SID_QUAL(inq_buf);
 
-				/*
-				 * We conservatively request only
-				 * SHORT_INQUIRY_LEN bytes of inquiry
-				 * information during our first try
-				 * at sending an INQUIRY. If the device
-				 * has more information to give,
-				 * perform a second request specifying
-				 * the amount of information the device
-				 * is willing to give.
-				 */
-				len = inq_buf->additional_length
-				    + offsetof(struct scsi_inquiry_data,
-                                               additional_length) + 1;
-				if (softc->action == PROBE_INQUIRY
-				    && len > SHORT_INQUIRY_LENGTH) {
-					PROBE_SET_ACTION(softc, PROBE_FULL_INQUIRY);
-					xpt_release_ccb(done_ccb);
-					xpt_schedule(periph, priority);
-					return;
-				}
+		if (periph_qual != SID_QUAL_LU_CONNECTED)
+			break;
 
-				scsi_find_quirk(path->device);
-				ata_device_transport(path);
-				path->device->flags &= ~CAM_DEV_UNCONFIGURED;
-				done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
-				xpt_action(done_ccb);
-				xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
-				    done_ccb);
-				xpt_release_ccb(done_ccb);
-				break;
-			}
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
-		}
-		goto device_fail;
-	}
-	case PROBE_PM_PID:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) == 0)
-				bzero(ident_buf, sizeof(*ident_buf));
-			softc->pm_pid = (done_ccb->ataio.res.lba_high << 24) +
-			    (done_ccb->ataio.res.lba_mid << 16) +
-			    (done_ccb->ataio.res.lba_low << 8) +
-			    done_ccb->ataio.res.sector_count;
-			((uint32_t *)ident_buf)[0] = softc->pm_pid;
-			printf("PM Product ID: %08x\n", softc->pm_pid);
-			snprintf(ident_buf->model, sizeof(ident_buf->model),
-			    "Port Multiplier %08x", softc->pm_pid);
-			PROBE_SET_ACTION(softc, PROBE_PM_PRV);
+		/*
+		 * We conservatively request only
+		 * SHORT_INQUIRY_LEN bytes of inquiry
+		 * information during our first try
+		 * at sending an INQUIRY. If the device
+		 * has more information to give,
+		 * perform a second request specifying
+		 * the amount of information the device
+		 * is willing to give.
+		 */
+		len = inq_buf->additional_length
+		    + offsetof(struct scsi_inquiry_data, additional_length) + 1;
+		if (softc->action == PROBE_INQUIRY
+		    && len > SHORT_INQUIRY_LENGTH) {
+			PROBE_SET_ACTION(softc, PROBE_FULL_INQUIRY);
 			xpt_release_ccb(done_ccb);
 			xpt_schedule(periph, priority);
 			return;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
 		}
-		goto device_fail;
+
+		scsi_find_quirk(path->device);
+		ata_device_transport(path);
+		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
+			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
+			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
+			xpt_action(done_ccb);
+			xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path, done_ccb);
+		}
+		break;
+	}
+	case PROBE_PM_PID:
+		if ((path->device->flags & CAM_DEV_IDENTIFY_DATA_VALID) == 0)
+			bzero(ident_buf, sizeof(*ident_buf));
+		softc->pm_pid = (done_ccb->ataio.res.lba_high << 24) +
+		    (done_ccb->ataio.res.lba_mid << 16) +
+		    (done_ccb->ataio.res.lba_low << 8) +
+		    done_ccb->ataio.res.sector_count;
+		((uint32_t *)ident_buf)[0] = softc->pm_pid;
+		printf("PM Product ID: %08x\n", softc->pm_pid);
+		snprintf(ident_buf->model, sizeof(ident_buf->model),
+		    "Port Multiplier %08x", softc->pm_pid);
+		PROBE_SET_ACTION(softc, PROBE_PM_PRV);
+		xpt_release_ccb(done_ccb);
+		xpt_schedule(periph, priority);
+		return;
 	case PROBE_PM_PRV:
-		if ((done_ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP) {
-			softc->pm_prv = (done_ccb->ataio.res.lba_high << 24) +
-			    (done_ccb->ataio.res.lba_mid << 16) +
-			    (done_ccb->ataio.res.lba_low << 8) +
-			    done_ccb->ataio.res.sector_count;
-			((uint32_t *)ident_buf)[1] = softc->pm_prv;
-			printf("PM Revision: %08x\n", softc->pm_prv);
-			snprintf(ident_buf->revision, sizeof(ident_buf->revision),
-			    "%04x", softc->pm_prv);
-			path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
-			if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
-				path->device->flags &= ~CAM_DEV_UNCONFIGURED;
-				done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
-				xpt_action(done_ccb);
-				xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
-				    done_ccb);
-			} else {
-				done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
-				xpt_action(done_ccb);
-				xpt_async(AC_SCSI_AEN, done_ccb->ccb_h.path,
-				    done_ccb);
-			}
-			xpt_release_ccb(done_ccb);
-			break;
-		} else if (cam_periph_error(done_ccb, 0, 0,
-					    &softc->saved_ccb) == ERESTART) {
-			return;
-		} else if ((done_ccb->ccb_h.status & CAM_DEV_QFRZN) != 0) {
-			/* Don't wedge the queue */
-			xpt_release_devq(done_ccb->ccb_h.path, /*count*/1,
-					 /*run_queue*/TRUE);
+		softc->pm_prv = (done_ccb->ataio.res.lba_high << 24) +
+		    (done_ccb->ataio.res.lba_mid << 16) +
+		    (done_ccb->ataio.res.lba_low << 8) +
+		    done_ccb->ataio.res.sector_count;
+		((uint32_t *)ident_buf)[1] = softc->pm_prv;
+		printf("PM Revision: %08x\n", softc->pm_prv);
+		snprintf(ident_buf->revision, sizeof(ident_buf->revision),
+		    "%04x", softc->pm_prv);
+		path->device->flags |= CAM_DEV_IDENTIFY_DATA_VALID;
+		if (periph->path->device->flags & CAM_DEV_UNCONFIGURED) {
+			path->device->flags &= ~CAM_DEV_UNCONFIGURED;
+			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
+			xpt_action(done_ccb);
+			xpt_async(AC_FOUND_DEVICE, done_ccb->ccb_h.path,
+			    done_ccb);
+		} else {
+			done_ccb->ccb_h.func_code = XPT_GDEV_TYPE;
+			xpt_action(done_ccb);
+			xpt_async(AC_SCSI_AEN, done_ccb->ccb_h.path, done_ccb);
 		}
-		goto device_fail;
+		break;
 	case PROBE_INVALID:
 		CAM_DEBUG(done_ccb->ccb_h.path, CAM_DEBUG_INFO,
 		    ("probedone: invalid action state\n"));
 	default:
 		break;
 	}
+done:
+	xpt_release_ccb(done_ccb);
 	done_ccb = (union ccb *)TAILQ_FIRST(&softc->request_ccbs);
 	TAILQ_REMOVE(&softc->request_ccbs, &done_ccb->ccb_h, periph_links.tqe);
 	done_ccb->ccb_h.status = CAM_REQ_CMP;
diff --git a/sys/cam/cam.h b/sys/cam/cam.h
index 46a5e07eeeba..c26afb4600b7 100644
--- a/sys/cam/cam.h
+++ b/sys/cam/cam.h
@@ -66,6 +66,7 @@ struct cam_periph;
  */
 typedef struct {
 	u_int32_t priority;
+#define CAM_PRIORITY_BUS	0
 #define CAM_PRIORITY_DEV	0
 #define CAM_PRIORITY_NORMAL	1
 #define CAM_PRIORITY_NONE	(u_int32_t)-1
diff --git a/sys/cam/cam_ccb.h b/sys/cam/cam_ccb.h
index a750d935f43f..483f22b5de15 100644
--- a/sys/cam/cam_ccb.h
+++ b/sys/cam/cam_ccb.h
@@ -816,12 +816,22 @@ struct ccb_trans_settings_sas {
 	u_int32_t 	bitrate;	/* Mbps */
 };
 
+struct ccb_trans_settings_ata {
+	u_int     	valid;		/* Which fields to honor */
+#define	CTS_ATA_VALID_MODE		0x01
+#define	CTS_ATA_VALID_BYTECOUNT		0x04
+	u_int32_t 	mode;
+	u_int 		bytecount;	/* Length of PIO transaction */
+};
+
 struct ccb_trans_settings_sata {
 	u_int     	valid;		/* Which fields to honor */
 #define	CTS_SATA_VALID_SPEED		0x01
 #define	CTS_SATA_VALID_PM		0x02
+#define	CTS_SATA_VALID_BYTECOUNT	0x04
 	u_int32_t 	bitrate;	/* Mbps */
 	u_int 		pm_present;	/* PM is present (XPT->SIM) */
+	u_int 		bytecount;	/* Length of PIO transaction */
 };
 
 /* Get/Set transfer rate/width/disconnection/tag queueing settings */
@@ -841,6 +851,7 @@ struct ccb_trans_settings {
 		struct ccb_trans_settings_spi spi;
 		struct ccb_trans_settings_fc fc;
 		struct ccb_trans_settings_sas sas;
+		struct ccb_trans_settings_ata ata;
 		struct ccb_trans_settings_sata sata;
 	} xport_specific;
 };
diff --git a/sys/cam/cam_periph.h b/sys/cam/cam_periph.h
index f95a94f81b83..258055f5bfc7 100644
--- a/sys/cam/cam_periph.h
+++ b/sys/cam/cam_periph.h
@@ -79,6 +79,8 @@ struct periph_driver {
 	char			 *driver_name;
 	TAILQ_HEAD(,cam_periph)	 units;
 	u_int			 generation;
+	u_int			 flags;
+#define CAM_PERIPH_DRV_EARLY		0x01
 };
 
 typedef enum {
diff --git a/sys/cam/cam_xpt.c b/sys/cam/cam_xpt.c
index d3ed9aa66063..187683ed8814 100644
--- a/sys/cam/cam_xpt.c
+++ b/sys/cam/cam_xpt.c
@@ -161,7 +161,8 @@ static periph_init_t xpt_periph_init;
 static struct periph_driver xpt_driver =
 {
 	xpt_periph_init, "xpt",
-	TAILQ_HEAD_INITIALIZER(xpt_driver.units)
+	TAILQ_HEAD_INITIALIZER(xpt_driver.units), /* generation */ 0,
+	CAM_PERIPH_DRV_EARLY
 };
 
 PERIPHDRIVER_DECLARE(xpt, xpt_driver);
@@ -1102,30 +1103,35 @@ xpt_announce_periph(struct cam_periph *periph, char *announce_string)
 	speed = cpi.base_transfer_speed;
 	freq = 0;
 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SPI) {
-		struct	ccb_trans_settings_spi *spi;
+		struct	ccb_trans_settings_spi *spi =
+		    &cts.xport_specific.spi;
 
-		spi = &cts.xport_specific.spi;
 		if ((spi->valid & CTS_SPI_VALID_SYNC_OFFSET) != 0
 		  && spi->sync_offset != 0) {
 			freq = scsi_calc_syncsrate(spi->sync_period);
 			speed = freq;
 		}
-
 		if ((spi->valid & CTS_SPI_VALID_BUS_WIDTH) != 0)
 			speed *= (0x01 << spi->bus_width);
 	}
 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_FC) {
-		struct	ccb_trans_settings_fc *fc = &cts.xport_specific.fc;
+		struct	ccb_trans_settings_fc *fc =
+		    &cts.xport_specific.fc;
+
 		if (fc->valid & CTS_FC_VALID_SPEED)
 			speed = fc->bitrate;
 	}
 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SAS) {
-		struct	ccb_trans_settings_sas *sas = &cts.xport_specific.sas;
+		struct	ccb_trans_settings_sas *sas =
+		    &cts.xport_specific.sas;
+
 		if (sas->valid & CTS_SAS_VALID_SPEED)
 			speed = sas->bitrate;
 	}
 	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) {
-		struct	ccb_trans_settings_sata *sata = &cts.xport_specific.sata;
+		struct	ccb_trans_settings_sata *sata =
+		    &cts.xport_specific.sata;
+
 		if (sata->valid & CTS_SATA_VALID_SPEED)
 			speed = sata->bitrate;
 	}
@@ -1173,7 +1179,20 @@ xpt_announce_periph(struct cam_periph *periph, char *announce_string)
 		if (fc->valid & CTS_FC_VALID_PORT)
 			printf(" PortID 0x%x", fc->port);
 	}
+	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_ATA) {
+		struct ccb_trans_settings_ata *ata =
+		    &cts.xport_specific.ata;
+
+		if (ata->valid & CTS_ATA_VALID_BYTECOUNT)
+			printf(" (PIO size %dbytes)", ata->bytecount);
+	}
+	if (cts.ccb_h.status == CAM_REQ_CMP && cts.transport == XPORT_SATA) {
+		struct ccb_trans_settings_sata *sata =
+		    &cts.xport_specific.sata;
 
+		if (sata->valid & CTS_SATA_VALID_BYTECOUNT)
+			printf(" (PIO size %dbytes)", sata->bytecount);
+	}
 	if (path->device->inq_flags & SID_CmdQue
 	 || path->device->flags & CAM_DEV_TAG_AFTER_COUNT) {
 		printf("\n%s%d: Command Queueing enabled",
@@ -4676,6 +4695,9 @@ xptconfigfunc(struct cam_eb *bus, void *arg)
 static void
 xpt_config(void *arg)
 {
+	struct	periph_driver **p_drv;
+	int	i;
+
 	/*
 	 * Now that interrupts are enabled, go find our devices
 	 */
@@ -4709,6 +4731,13 @@ xpt_config(void *arg)
 #endif /* CAM_DEBUG_BUS */
 #endif /* CAMDEBUG */
 
+	/* Register early peripheral drivers */
+	/* XXX This will have to change when we have loadable modules */
+	p_drv = periph_drivers;
+	for (i = 0; p_drv[i] != NULL; i++) {
+		if ((p_drv[i]->flags & CAM_PERIPH_DRV_EARLY) != 0)
+			(*p_drv[i]->init)();
+	}
 	/*
 	 * Scan all installed busses.
 	 */
@@ -4759,7 +4788,8 @@ xpt_finishconfig_task(void *context, int pending)
 		/* XXX This will have to change when we have loadable modules */
 		p_drv = periph_drivers;
 		for (i = 0; p_drv[i] != NULL; i++) {
-			(*p_drv[i]->init)();
+			if ((p_drv[i]->flags & CAM_PERIPH_DRV_EARLY) == 0)
+				(*p_drv[i]->init)();
 		}
 
 		/*
diff --git a/sys/cam/scsi/scsi_da.c b/sys/cam/scsi/scsi_da.c
index bb9299a4515c..8f740dcdfad2 100644
--- a/sys/cam/scsi/scsi_da.c
+++ b/sys/cam/scsi/scsi_da.c
@@ -1491,8 +1491,10 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 {
 	struct da_softc *softc;
 	struct ccb_scsiio *csio;
+	u_int32_t  priority;
 
 	softc = (struct da_softc *)periph->softc;
+	priority = done_ccb->ccb_h.pinfo.priority;
 	csio = &done_ccb->csio;
 	switch (csio->ccb_h.ccb_state & DA_CCB_TYPE_MASK) {
 	case DA_CCB_BUFFER_IO:
@@ -1610,7 +1612,7 @@ dadone(struct cam_periph *periph, union ccb *done_ccb)
 					softc->state = DA_STATE_PROBE2;
 					free(rdcap, M_SCSIDA);
 					xpt_release_ccb(done_ccb);
-					xpt_schedule(periph, /*priority*/5);
+					xpt_schedule(periph, priority);
 					return;
 				}
 			} else {
diff --git a/sys/cam/scsi/scsi_sg.c b/sys/cam/scsi/scsi_sg.c
index 4ab038bc2ee9..d47e6e9adb3a 100644
--- a/sys/cam/scsi/scsi_sg.c
+++ b/sys/cam/scsi/scsi_sg.c
@@ -510,7 +510,7 @@ sgioctl(struct cdev *dev, u_long cmd, caddr_t arg, int flag, struct thread *td)
 			break;
 		}
 
-		ccb = cam_periph_getccb(periph, /*priority*/5);
+		ccb = cam_periph_getccb(periph, CAM_PRIORITY_NORMAL);
 		csio = &ccb->csio;
 
 		error = copyin(req.cmdp, &csio->cdb_io.cdb_bytes,
@@ -729,7 +729,7 @@ sgwrite(struct cdev *dev, struct uio *uio, int ioflag)
 
 	cam_periph_lock(periph);
 	sc = periph->softc;
-	xpt_setup_ccb(&ccb->ccb_h, periph->path, /*priority*/5);
+	xpt_setup_ccb(&ccb->ccb_h, periph->path, CAM_PRIORITY_NORMAL);
 	cam_fill_csio(csio,
 		      /*retries*/1,
 		      sgdone,
diff --git a/sys/cam/scsi/scsi_xpt.c b/sys/cam/scsi/scsi_xpt.c
index 35a5ac2a7064..a240db845cee 100644
--- a/sys/cam/scsi/scsi_xpt.c
+++ b/sys/cam/scsi/scsi_xpt.c
@@ -110,7 +110,8 @@ static periph_init_t probe_periph_init;
 static struct periph_driver probe_driver =
 {
 	probe_periph_init, "probe",
-	TAILQ_HEAD_INITIALIZER(probe_driver.units)
+	TAILQ_HEAD_INITIALIZER(probe_driver.units), /* generation */ 0,
+	CAM_PERIPH_DRV_EARLY
 };
 
 PERIPHDRIVER_DECLARE(probe, probe_driver);