Bunch of microoptimizations to reduce dereferences and cache collisions.

3 files changed, 137 insertions, 163 deletions

diff --git a/sys/dev/ahci/ahci.c b/sys/dev/ahci/ahci.c
index b15a3b4..6d1e28d 100644
--- a/sys/dev/ahci/ahci.c
+++ b/sys/dev/ahci/ahci.c
@@ -61,31 +61,31 @@ static void ahci_ch_pm(void *arg);
 static void ahci_ch_intr(void *arg);
 static void ahci_ch_intr_direct(void *arg);
 static void ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus);
-static void ahci_begin_transaction(device_t dev, union ccb *ccb);
+static void ahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb);
 static void ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
 static void ahci_execute_transaction(struct ahci_slot *slot);
 static void ahci_timeout(struct ahci_slot *slot);
 static void ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et);
-static int ahci_setup_fis(device_t dev, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag);
+static int ahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag);
 static void ahci_dmainit(device_t dev);
 static void ahci_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
 static void ahci_dmafini(device_t dev);
 static void ahci_slotsalloc(device_t dev);
 static void ahci_slotsfree(device_t dev);
-static void ahci_reset(device_t dev);
-static void ahci_start(device_t dev, int fbs);
-static void ahci_stop(device_t dev);
-static void ahci_clo(device_t dev);
-static void ahci_start_fr(device_t dev);
-static void ahci_stop_fr(device_t dev);
+static void ahci_reset(struct ahci_channel *ch);
+static void ahci_start(struct ahci_channel *ch, int fbs);
+static void ahci_stop(struct ahci_channel *ch);
+static void ahci_clo(struct ahci_channel *ch);
+static void ahci_start_fr(struct ahci_channel *ch);
+static void ahci_stop_fr(struct ahci_channel *ch);
 
 static int ahci_sata_connect(struct ahci_channel *ch);
-static int ahci_sata_phy_reset(device_t dev);
-static int ahci_wait_ready(device_t dev, int t, int t0);
+static int ahci_sata_phy_reset(struct ahci_channel *ch);
+static int ahci_wait_ready(struct ahci_channel *ch, int t, int t0);
 
-static void ahci_issue_recovery(device_t dev);
-static void ahci_process_read_log(device_t dev, union ccb *ccb);
-static void ahci_process_request_sense(device_t dev, union ccb *ccb);
+static void ahci_issue_recovery(struct ahci_channel *ch);
+static void ahci_process_read_log(struct ahci_channel *ch, union ccb *ccb);
+static void ahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb);
 
 static void ahciaction(struct cam_sim *sim, union ccb *ccb);
 static void ahcipoll(struct cam_sim *sim);
@@ -669,7 +669,7 @@ ahci_ch_attach(device_t dev)
 	}
 	if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
 	    ctlr->direct ? ahci_ch_intr_direct : ahci_ch_intr,
-	    dev, &ch->ih))) {
+	    ch, &ch->ih))) {
 		device_printf(dev, "Unable to setup interrupt\n");
 		error = ENXIO;
 		goto err1;
@@ -698,7 +698,7 @@ ahci_ch_attach(device_t dev)
 	}
 	/* Construct SIM entry */
 	ch->sim = cam_sim_alloc(ahciaction, ahcipoll, "ahcich", ch,
-	    device_get_unit(dev), &ch->mtx,
+	    device_get_unit(dev), (struct mtx *)&ch->mtx,
 	    min(2, ch->numslots),
 	    (ch->caps & AHCI_CAP_SNCQ) ? ch->numslots : 0,
 	    devq);
@@ -722,7 +722,7 @@ ahci_ch_attach(device_t dev)
 	if (ch->pm_level > 3) {
 		callout_reset(&ch->pm_timer,
 		    (ch->pm_level == 4) ? hz / 1000 : hz / 8,
-		    ahci_ch_pm, dev);
+		    ahci_ch_pm, ch);
 	}
 	mtx_unlock(&ch->mtx);
 	return (0);
@@ -792,8 +792,8 @@ ahci_ch_init(device_t dev)
 	     (AHCI_P_CMD_ACTIVE | AHCI_P_CMD_POD | AHCI_P_CMD_SUD |
 	     ((ch->pm_level == 2 || ch->pm_level == 3) ? AHCI_P_CMD_ALPE : 0) |
 	     ((ch->pm_level > 2) ? AHCI_P_CMD_ASP : 0 )));
-	ahci_start_fr(dev);
-	ahci_start(dev, 1);
+	ahci_start_fr(ch);
+	ahci_start(ch, 1);
 	return (0);
 }
 
@@ -805,8 +805,8 @@ ahci_ch_deinit(device_t dev)
 	/* Disable port interrupts. */
 	ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
 	/* Reset command register. */
-	ahci_stop(dev);
-	ahci_stop_fr(dev);
+	ahci_stop(ch);
+	ahci_stop_fr(ch);
 	ATA_OUTL(ch->r_mem, AHCI_P_CMD, 0);
 	/* Allow everything, including partial and slumber modes. */
 	ATA_OUTL(ch->r_mem, AHCI_P_SCTL, 0);
@@ -845,7 +845,7 @@ ahci_ch_resume(device_t dev)
 
 	mtx_lock(&ch->mtx);
 	ahci_ch_init(dev);
-	ahci_reset(dev);
+	ahci_reset(ch);
 	xpt_release_simq(ch->sim, TRUE);
 	mtx_unlock(&ch->mtx);
 	return (0);
@@ -976,7 +976,7 @@ ahci_slotsalloc(device_t dev)
 	for (i = 0; i < ch->numslots; i++) {
 		struct ahci_slot *slot = &ch->slot[i];
 
-		slot->dev = dev;
+		slot->ch = ch;
 		slot->slot = i;
 		slot->state = AHCI_SLOT_EMPTY;
 		slot->ccb = NULL;
@@ -1006,9 +1006,8 @@ ahci_slotsfree(device_t dev)
 }
 
 static int
-ahci_phy_check_events(device_t dev, u_int32_t serr)
+ahci_phy_check_events(struct ahci_channel *ch, u_int32_t serr)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 
 	if (((ch->pm_level == 0) && (serr & ATA_SE_PHY_CHANGED)) ||
 	    ((ch->pm_level != 0 || ch->listening) && (serr & ATA_SE_EXCHANGED))) {
@@ -1017,11 +1016,11 @@ ahci_phy_check_events(device_t dev, u_int32_t serr)
 
 		if (bootverbose) {
 			if ((status & ATA_SS_DET_MASK) != ATA_SS_DET_NO_DEVICE)
-				device_printf(dev, "CONNECT requested\n");
+				device_printf(ch->dev, "CONNECT requested\n");
 			else
-				device_printf(dev, "DISCONNECT requested\n");
+				device_printf(ch->dev, "DISCONNECT requested\n");
 		}
-		ahci_reset(dev);
+		ahci_reset(ch);
 		if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
 			return (0);
 		if (xpt_create_path(&ccb->ccb_h.path, NULL,
@@ -1037,11 +1036,11 @@ ahci_phy_check_events(device_t dev, u_int32_t serr)
 }
 
 static void
-ahci_cpd_check_events(device_t dev)
+ahci_cpd_check_events(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	u_int32_t status;
 	union ccb *ccb;
+	device_t dev;
 
 	if (ch->pm_level == 0)
 		return;
@@ -1051,12 +1050,13 @@ ahci_cpd_check_events(device_t dev)
 		return;
 
 	if (bootverbose) {
+		dev = ch->dev;
 		if (status & AHCI_P_CMD_CPS) {
 			device_printf(dev, "COLD CONNECT requested\n");
 		} else
 			device_printf(dev, "COLD DISCONNECT requested\n");
 	}
-	ahci_reset(dev);
+	ahci_reset(ch);
 	if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
 		return;
 	if (xpt_create_path(&ccb->ccb_h.path, NULL, cam_sim_path(ch->sim),
@@ -1068,16 +1068,15 @@ ahci_cpd_check_events(device_t dev)
 }
 
 static void
-ahci_notify_events(device_t dev, u_int32_t status)
+ahci_notify_events(struct ahci_channel *ch, u_int32_t status)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	struct cam_path *dpath;
 	int i;
 
 	if (ch->caps & AHCI_CAP_SSNTF)
 		ATA_OUTL(ch->r_mem, AHCI_P_SNTF, status);
 	if (bootverbose)
-		device_printf(dev, "SNTF 0x%04x\n", status);
+		device_printf(ch->dev, "SNTF 0x%04x\n", status);
 	for (i = 0; i < 16; i++) {
 		if ((status & (1 << i)) == 0)
 			continue;
@@ -1106,8 +1105,7 @@ ahci_done(struct ahci_channel *ch, union ccb *ccb)
 static void
 ahci_ch_intr(void *arg)
 {
-	device_t dev = (device_t)arg;
-	struct ahci_channel *ch = device_get_softc(dev);
+	struct ahci_channel *ch = (struct ahci_channel *)arg;
 	uint32_t istatus;
 
 	/* Read interrupt statuses. */
@@ -1123,8 +1121,7 @@ ahci_ch_intr(void *arg)
 static void
 ahci_ch_intr_direct(void *arg)
 {
-	device_t dev = (device_t)arg;
-	struct ahci_channel *ch = device_get_softc(dev);
+	struct ahci_channel *ch = (struct ahci_channel *)arg;
 	struct ccb_hdr *ccb_h;
 	uint32_t istatus;
 
@@ -1147,8 +1144,7 @@ ahci_ch_intr_direct(void *arg)
 static void
 ahci_ch_pm(void *arg)
 {
-	device_t dev = (device_t)arg;
-	struct ahci_channel *ch = device_get_softc(dev);
+	struct ahci_channel *ch = (struct ahci_channel *)arg;
 	uint32_t work;
 
 	if (ch->numrslots != 0)
@@ -1164,7 +1160,6 @@ ahci_ch_pm(void *arg)
 static void
 ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus)
 {
-	device_t dev = ch->dev;
 	uint32_t cstatus, serr = 0, sntf = 0, ok, err;
 	enum ahci_err_type et;
 	int i, ccs, port, reset = 0;
@@ -1206,12 +1201,12 @@ ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus)
 		serr = ATA_INL(ch->r_mem, AHCI_P_SERR);
 		if (serr) {
 			ATA_OUTL(ch->r_mem, AHCI_P_SERR, serr);
-			reset = ahci_phy_check_events(dev, serr);
+			reset = ahci_phy_check_events(ch, serr);
 		}
 	}
 	/* Process cold presence detection events */
 	if ((istatus & AHCI_P_IX_CPD) && !reset)
-		ahci_cpd_check_events(dev);
+		ahci_cpd_check_events(ch);
 	/* Process command errors */
 	if (istatus & (AHCI_P_IX_OF | AHCI_P_IX_IF |
 	    AHCI_P_IX_HBD | AHCI_P_IX_HBF | AHCI_P_IX_TFE)) {
@@ -1306,14 +1301,13 @@ ahci_ch_intr_main(struct ahci_channel *ch, uint32_t istatus)
 	}
 	/* Process NOTIFY events */
 	if (sntf)
-		ahci_notify_events(dev, sntf);
+		ahci_notify_events(ch, sntf);
 }
 
 /* Must be called with channel locked. */
 static int
-ahci_check_collision(device_t dev, union ccb *ccb)
+ahci_check_collision(struct ahci_channel *ch, union ccb *ccb)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	int t = ccb->ccb_h.target_id;
 
 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
@@ -1362,9 +1356,8 @@ ahci_check_collision(device_t dev, union ccb *ccb)
 
 /* Must be called with channel locked. */
 static void
-ahci_begin_transaction(device_t dev, union ccb *ccb)
+ahci_begin_transaction(struct ahci_channel *ch, union ccb *ccb)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	struct ahci_slot *slot;
 	int tag, tags;
 
@@ -1373,14 +1366,14 @@ ahci_begin_transaction(device_t dev, union ccb *ccb)
 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
 	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA))
 		tags = ch->curr[ccb->ccb_h.target_id].tags;
-	tag = ch->lastslot;
-	while (1) {
-		if (tag >= tags)
-			tag = 0;
-		if (ch->slot[tag].state == AHCI_SLOT_EMPTY)
-			break;
-		tag++;
-	};
+	if (ch->lastslot + 1 < tags)
+		tag = ffs(~(ch->oslots >> (ch->lastslot + 1)));
+	else
+		tag = 0;
+	if (tag == 0 || tag + ch->lastslot >= tags)
+		tag = ffs(~ch->oslots) - 1;
+	else
+		tag += ch->lastslot;
 	ch->lastslot = tag;
 	/* Occupy chosen slot. */
 	slot = &ch->slot[tag];
@@ -1389,7 +1382,7 @@ ahci_begin_transaction(device_t dev, union ccb *ccb)
 	if (ch->numrslots == 0 && ch->pm_level > 3)
 		callout_stop(&ch->pm_timer);
 	/* Update channel stats. */
-	ch->oslots |= (1 << slot->slot);
+	ch->oslots |= (1 << tag);
 	ch->numrslots++;
 	ch->numrslotspd[ccb->ccb_h.target_id]++;
 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
@@ -1400,14 +1393,15 @@ ahci_begin_transaction(device_t dev, union ccb *ccb)
 	}
 	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
 	    (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)))
-		ch->aslots |= (1 << slot->slot);
-	slot->dma.nsegs = 0;
+		ch->aslots |= (1 << tag);
 	if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
 		slot->state = AHCI_SLOT_LOADING;
 		bus_dmamap_load_ccb(ch->dma.data_tag, slot->dma.data_map, ccb,
 		    ahci_dmasetprd, slot, 0);
-	} else
+	} else {
+		slot->dma.nsegs = 0;
 		ahci_execute_transaction(slot);
+	}
 }
 
 /* Locked by busdma engine. */
@@ -1415,13 +1409,13 @@ static void
 ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 {    
 	struct ahci_slot *slot = arg;
-	struct ahci_channel *ch = device_get_softc(slot->dev);
+	struct ahci_channel *ch = slot->ch;
 	struct ahci_cmd_tab *ctp;
 	struct ahci_dma_prd *prd;
 	int i;
 
 	if (error) {
-		device_printf(slot->dev, "DMA load error\n");
+		device_printf(ch->dev, "DMA load error\n");
 		ahci_end_transaction(slot, AHCI_ERR_INVALID);
 		return;
 	}
@@ -1446,8 +1440,7 @@ ahci_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
 static void
 ahci_execute_transaction(struct ahci_slot *slot)
 {
-	device_t dev = slot->dev;
-	struct ahci_channel *ch = device_get_softc(dev);
+	struct ahci_channel *ch = slot->ch;
 	struct ahci_cmd_tab *ctp;
 	struct ahci_cmd_list *clp;
 	union ccb *ccb = slot->ccb;
@@ -1460,7 +1453,7 @@ ahci_execute_transaction(struct ahci_slot *slot)
 	ctp = (struct ahci_cmd_tab *)
 		(ch->dma.work + AHCI_CT_OFFSET + (AHCI_CT_SIZE * slot->slot));
 	/* Setup the FIS for this request */
-	if (!(fis_size = ahci_setup_fis(dev, ctp, ccb, slot->slot))) {
+	if (!(fis_size = ahci_setup_fis(ch, ctp, ccb, slot->slot))) {
 		device_printf(ch->dev, "Setting up SATA FIS failed\n");
 		ahci_end_transaction(slot, AHCI_ERR_INVALID);
 		return;
@@ -1481,9 +1474,9 @@ ahci_execute_transaction(struct ahci_slot *slot)
 		if (ccb->ataio.cmd.control & ATA_A_RESET) {
 			softreset = 1;
 			/* Kick controller into sane state */
-			ahci_stop(dev);
-			ahci_clo(dev);
-			ahci_start(dev, 0);
+			ahci_stop(ch);
+			ahci_clo(ch);
+			ahci_start(ch, 0);
 			clp->cmd_flags |= AHCI_CMD_RESET | AHCI_CMD_CLR_BUSY;
 		} else {
 			softreset = 2;
@@ -1561,9 +1554,9 @@ ahci_execute_transaction(struct ahci_slot *slot)
 		}
 
 		if (timeout && (count >= timeout)) {
-			device_printf(dev, "Poll timeout on slot %d port %d\n",
+			device_printf(ch->dev, "Poll timeout on slot %d port %d\n",
 			    slot->slot, port);
-			device_printf(dev, "is %08x cs %08x ss %08x "
+			device_printf(ch->dev, "is %08x cs %08x ss %08x "
 			    "rs %08x tfd %02x serr %08x cmd %08x\n",
 			    ATA_INL(ch->r_mem, AHCI_P_IS),
 			    ATA_INL(ch->r_mem, AHCI_P_CI),
@@ -1588,9 +1581,8 @@ ahci_execute_transaction(struct ahci_slot *slot)
 
 /* Must be called with channel locked. */
 static void
-ahci_process_timeout(device_t dev)
+ahci_process_timeout(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	int i;
 
 	mtx_assert(&ch->mtx, MA_OWNED);
@@ -1605,9 +1597,8 @@ ahci_process_timeout(device_t dev)
 
 /* Must be called with channel locked. */
 static void
-ahci_rearm_timeout(device_t dev)
+ahci_rearm_timeout(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	int i;
 
 	mtx_assert(&ch->mtx, MA_OWNED);
@@ -1629,8 +1620,8 @@ ahci_rearm_timeout(device_t dev)
 static void
 ahci_timeout(struct ahci_slot *slot)
 {
-	device_t dev = slot->dev;
-	struct ahci_channel *ch = device_get_softc(dev);
+	struct ahci_channel *ch = slot->ch;
+	device_t dev = ch->dev;
 	uint32_t sstatus;
 	int ccs;
 	int i;
@@ -1697,7 +1688,7 @@ ahci_timeout(struct ahci_slot *slot)
 			xpt_freeze_simq(ch->sim, 1);
 		ch->toslots |= (1 << slot->slot);
 		if ((ch->rslots & ~ch->toslots) == 0)
-			ahci_process_timeout(dev);
+			ahci_process_timeout(ch);
 		else
 			device_printf(dev, " ... waiting for slots %08x\n",
 			    ch->rslots & ~ch->toslots);
@@ -1708,8 +1699,7 @@ ahci_timeout(struct ahci_slot *slot)
 static void
 ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et)
 {
-	device_t dev = slot->dev;
-	struct ahci_channel *ch = device_get_softc(dev);
+	struct ahci_channel *ch = slot->ch;
 	union ccb *ccb = slot->ccb;
 	struct ahci_cmd_list *clp;
 	int lastto;
@@ -1867,15 +1857,15 @@ ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et)
 	    (ccb->ataio.cmd.control & ATA_A_RESET) &&
 	    et == AHCI_ERR_NONE) {
 		ccb->ataio.cmd.control &= ~ATA_A_RESET;
-		ahci_begin_transaction(dev, ccb);
+		ahci_begin_transaction(ch, ccb);
 		return;
 	}
 	/* If it was our READ LOG command - process it. */
 	if (ccb->ccb_h.recovery_type == RECOVERY_READ_LOG) {
-		ahci_process_read_log(dev, ccb);
+		ahci_process_read_log(ch, ccb);
 	/* If it was our REQUEST SENSE command - process it. */
 	} else if (ccb->ccb_h.recovery_type == RECOVERY_REQUEST_SENSE) {
-		ahci_process_request_sense(dev, ccb);
+		ahci_process_request_sense(ch, ccb);
 	/* If it was NCQ or ATAPI command error, put result on hold. */
 	} else if (et == AHCI_ERR_NCQ ||
 	    ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_SCSI_STATUS_ERROR &&
@@ -1888,27 +1878,27 @@ ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et)
 	if (ch->rslots == 0) {
 		/* if there was fatal error - reset port. */
 		if (ch->toslots != 0 || ch->fatalerr) {
-			ahci_reset(dev);
+			ahci_reset(ch);
 		} else {
 			/* if we have slots in error, we can reinit port. */
 			if (ch->eslots != 0) {
-				ahci_stop(dev);
-				ahci_clo(dev);
-				ahci_start(dev, 1);
+				ahci_stop(ch);
+				ahci_clo(ch);
+				ahci_start(ch, 1);
 			}
 			/* if there commands on hold, we can do READ LOG. */
 			if (!ch->recoverycmd && ch->numhslots)
-				ahci_issue_recovery(dev);
+				ahci_issue_recovery(ch);
 		}
 	/* If all the rest of commands are in timeout - give them chance. */
 	} else if ((ch->rslots & ~ch->toslots) == 0 &&
 	    et != AHCI_ERR_TIMEOUT)
-		ahci_rearm_timeout(dev);
+		ahci_rearm_timeout(ch);
 	/* Unfreeze frozen command. */
-	if (ch->frozen && !ahci_check_collision(dev, ch->frozen)) {
+	if (ch->frozen && !ahci_check_collision(ch, ch->frozen)) {
 		union ccb *fccb = ch->frozen;
 		ch->frozen = NULL;
-		ahci_begin_transaction(dev, fccb);
+		ahci_begin_transaction(ch, fccb);
 		xpt_release_simq(ch->sim, TRUE);
 	}
 	/* Start PM timer. */
@@ -1920,9 +1910,8 @@ ahci_end_transaction(struct ahci_slot *slot, enum ahci_err_type et)
 }
 
 static void
-ahci_issue_recovery(device_t dev)
+ahci_issue_recovery(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	union ccb *ccb;
 	struct ccb_ataio *ataio;
 	struct ccb_scsiio *csio;
@@ -1935,7 +1924,7 @@ ahci_issue_recovery(device_t dev)
 	}
 	ccb = xpt_alloc_ccb_nowait();
 	if (ccb == NULL) {
-		device_printf(dev, "Unable to allocate recovery command\n");
+		device_printf(ch->dev, "Unable to allocate recovery command\n");
 completeall:
 		/* We can't do anything -- complete held commands. */
 		for (i = 0; i < ch->numslots; i++) {
@@ -1947,7 +1936,7 @@ completeall:
 			ch->hold[i] = NULL;
 			ch->numhslots--;
 		}
-		ahci_reset(dev);
+		ahci_reset(ch);
 		return;
 	}
 	ccb->ccb_h = ch->hold[i]->ccb_h;	/* Reuse old header. */
@@ -1961,7 +1950,7 @@ completeall:
 		ataio->data_ptr = malloc(512, M_AHCI, M_NOWAIT);
 		if (ataio->data_ptr == NULL) {
 			xpt_free_ccb(ccb);
-			device_printf(dev,
+			device_printf(ch->dev,
 			    "Unable to allocate memory for READ LOG command\n");
 			goto completeall;
 		}
@@ -1993,13 +1982,12 @@ completeall:
 	/* Freeze SIM while doing recovery. */
 	ch->recoverycmd = 1;
 	xpt_freeze_simq(ch->sim, 1);
-	ahci_begin_transaction(dev, ccb);
+	ahci_begin_transaction(ch, ccb);
 }
 
 static void
-ahci_process_read_log(device_t dev, union ccb *ccb)
+ahci_process_read_log(struct ahci_channel *ch, union ccb *ccb)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	uint8_t *data;
 	struct ata_res *res;
 	int i;
@@ -2037,9 +2025,9 @@ ahci_process_read_log(device_t dev, union ccb *ccb)
 		}
 	} else {
 		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
-			device_printf(dev, "Error while READ LOG EXT\n");
+			device_printf(ch->dev, "Error while READ LOG EXT\n");
 		else if ((data[0] & 0x80) == 0) {
-			device_printf(dev, "Non-queued command error in READ LOG EXT\n");
+			device_printf(ch->dev, "Non-queued command error in READ LOG EXT\n");
 		}
 		for (i = 0; i < ch->numslots; i++) {
 			if (!ch->hold[i])
@@ -2057,9 +2045,8 @@ ahci_process_read_log(device_t dev, union ccb *ccb)
 }
 
 static void
-ahci_process_request_sense(device_t dev, union ccb *ccb)
+ahci_process_request_sense(struct ahci_channel *ch, union ccb *ccb)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	int i;
 
 	ch->recoverycmd = 0;
@@ -2079,9 +2066,8 @@ ahci_process_request_sense(device_t dev, union ccb *ccb)
 }
 
 static void
-ahci_start(device_t dev, int fbs)
+ahci_start(struct ahci_channel *ch, int fbs)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	u_int32_t cmd;
 
 	/* Clear SATA error register */
@@ -2102,9 +2088,8 @@ ahci_start(device_t dev, int fbs)
 }
 
 static void
-ahci_stop(device_t dev)
+ahci_stop(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	u_int32_t cmd;
 	int timeout;
 
@@ -2116,7 +2101,7 @@ ahci_stop(device_t dev)
 	do {
 		DELAY(10);
 		if (timeout++ > 50000) {
-			device_printf(dev, "stopping AHCI engine failed\n");
+			device_printf(ch->dev, "stopping AHCI engine failed\n");
 			break;
 		}
 	} while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CR);
@@ -2124,9 +2109,8 @@ ahci_stop(device_t dev)
 }
 
 static void
-ahci_clo(device_t dev)
+ahci_clo(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	u_int32_t cmd;
 	int timeout;
 
@@ -2139,7 +2123,7 @@ ahci_clo(device_t dev)
 		do {
 			DELAY(10);
 			if (timeout++ > 50000) {
-			    device_printf(dev, "executing CLO failed\n");
+			    device_printf(ch->dev, "executing CLO failed\n");
 			    break;
 			}
 		} while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_CLO);
@@ -2147,9 +2131,8 @@ ahci_clo(device_t dev)
 }
 
 static void
-ahci_stop_fr(device_t dev)
+ahci_stop_fr(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	u_int32_t cmd;
 	int timeout;
 
@@ -2161,16 +2144,15 @@ ahci_stop_fr(device_t dev)
 	do {
 		DELAY(10);
 		if (timeout++ > 50000) {
-			device_printf(dev, "stopping AHCI FR engine failed\n");
+			device_printf(ch->dev, "stopping AHCI FR engine failed\n");
 			break;
 		}
 	} while (ATA_INL(ch->r_mem, AHCI_P_CMD) & AHCI_P_CMD_FR);
 }
 
 static void
-ahci_start_fr(device_t dev)
+ahci_start_fr(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	u_int32_t cmd;
 
 	/* Start FIS reception on this channel */
@@ -2179,9 +2161,8 @@ ahci_start_fr(device_t dev)
 }
 
 static int
-ahci_wait_ready(device_t dev, int t, int t0)
+ahci_wait_ready(struct ahci_channel *ch, int t, int t0)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	int timeout = 0;
 	uint32_t val;
 
@@ -2189,7 +2170,7 @@ ahci_wait_ready(device_t dev, int t, int t0)
 	    (ATA_S_BUSY | ATA_S_DRQ)) {
 		if (timeout > t) {
 			if (t != 0) {
-				device_printf(dev,
+				device_printf(ch->dev,
 				    "AHCI reset: device not ready after %dms "
 				    "(tfd = %08x)\n",
 				    MAX(t, 0) + t0, val);
@@ -2200,7 +2181,7 @@ ahci_wait_ready(device_t dev, int t, int t0)
 		timeout++;
 	}
 	if (bootverbose)
-		device_printf(dev, "AHCI reset: device ready after %dms\n",
+		device_printf(ch->dev, "AHCI reset: device ready after %dms\n",
 		    timeout + t0);
 	return (0);
 }
@@ -2208,22 +2189,21 @@ ahci_wait_ready(device_t dev, int t, int t0)
 static void
 ahci_reset_to(void *arg)
 {
-	device_t dev = arg;
-	struct ahci_channel *ch = device_get_softc(dev);
+	struct ahci_channel *ch = arg;
 
 	if (ch->resetting == 0)
 		return;
 	ch->resetting--;
-	if (ahci_wait_ready(dev, ch->resetting == 0 ? -1 : 0,
+	if (ahci_wait_ready(ch, ch->resetting == 0 ? -1 : 0,
 	    (310 - ch->resetting) * 100) == 0) {
 		ch->resetting = 0;
-		ahci_start(dev, 1);
+		ahci_start(ch, 1);
 		xpt_release_simq(ch->sim, TRUE);
 		return;
 	}
 	if (ch->resetting == 0) {
-		ahci_clo(dev);
-		ahci_start(dev, 1);
+		ahci_clo(ch);
+		ahci_start(ch, 1);
 		xpt_release_simq(ch->sim, TRUE);
 		return;
 	}
@@ -2231,15 +2211,14 @@ ahci_reset_to(void *arg)
 }
 
 static void
-ahci_reset(device_t dev)
+ahci_reset(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
-	struct ahci_controller *ctlr = device_get_softc(device_get_parent(dev));
+	struct ahci_controller *ctlr = device_get_softc(device_get_parent(ch->dev));
 	int i;
 
 	xpt_freeze_simq(ch->sim, 1);
 	if (bootverbose)
-		device_printf(dev, "AHCI reset...\n");
+		device_printf(ch->dev, "AHCI reset...\n");
 	/* Forget about previous reset. */
 	if (ch->resetting) {
 		ch->resetting = 0;
@@ -2258,7 +2237,7 @@ ahci_reset(device_t dev)
 		ahci_done(ch, fccb);
 	}
 	/* Kill the engine and requeue all running commands. */
-	ahci_stop(dev);
+	ahci_stop(ch);
 	for (i = 0; i < ch->numslots; i++) {
 		/* Do we have a running request on slot? */
 		if (ch->slot[i].state < AHCI_SLOT_RUNNING)
@@ -2284,9 +2263,9 @@ ahci_reset(device_t dev)
 	/* Disable port interrupts */
 	ATA_OUTL(ch->r_mem, AHCI_P_IE, 0);
 	/* Reset and reconnect PHY, */
-	if (!ahci_sata_phy_reset(dev)) {
+	if (!ahci_sata_phy_reset(ch)) {
 		if (bootverbose)
-			device_printf(dev,
+			device_printf(ch->dev,
 			    "AHCI reset: device not found\n");
 		ch->devices = 0;
 		/* Enable wanted port interrupts */
@@ -2297,11 +2276,11 @@ ahci_reset(device_t dev)
 		return;
 	}
 	if (bootverbose)
-		device_printf(dev, "AHCI reset: device found\n");
+		device_printf(ch->dev, "AHCI reset: device found\n");
 	/* Wait for clearing busy status. */
-	if (ahci_wait_ready(dev, dumping ? 31000 : 0, 0)) {
+	if (ahci_wait_ready(ch, dumping ? 31000 : 0, 0)) {
 		if (dumping)
-			ahci_clo(dev);
+			ahci_clo(ch);
 		else
 			ch->resetting = 310;
 	}
@@ -2315,17 +2294,16 @@ ahci_reset(device_t dev)
 	      AHCI_P_IX_DP | AHCI_P_IX_UF | (ctlr->ccc ? 0 : AHCI_P_IX_SDB) |
 	      AHCI_P_IX_DS | AHCI_P_IX_PS | (ctlr->ccc ? 0 : AHCI_P_IX_DHR)));
 	if (ch->resetting)
-		callout_reset(&ch->reset_timer, hz / 10, ahci_reset_to, dev);
+		callout_reset(&ch->reset_timer, hz / 10, ahci_reset_to, ch);
 	else {
-		ahci_start(dev, 1);
+		ahci_start(ch, 1);
 		xpt_release_simq(ch->sim, TRUE);
 	}
 }
 
 static int
-ahci_setup_fis(device_t dev, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag)
+ahci_setup_fis(struct ahci_channel *ch, struct ahci_cmd_tab *ctp, union ccb *ccb, int tag)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	u_int8_t *fis = &ctp->cfis[0];
 
 	bzero(fis, 20);
@@ -2417,9 +2395,8 @@ ahci_sata_connect(struct ahci_channel *ch)
 }
 
 static int
-ahci_sata_phy_reset(device_t dev)
+ahci_sata_phy_reset(struct ahci_channel *ch)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 	int sata_rev;
 	uint32_t val;
 
@@ -2459,9 +2436,8 @@ ahci_sata_phy_reset(device_t dev)
 }
 
 static int
-ahci_check_ids(device_t dev, union ccb *ccb)
+ahci_check_ids(struct ahci_channel *ch, union ccb *ccb)
 {
-	struct ahci_channel *ch = device_get_softc(dev);
 
 	if (ccb->ccb_h.target_id > ((ch->caps & AHCI_CAP_SPM) ? 15 : 0)) {
 		ccb->ccb_h.status = CAM_TID_INVALID;
@@ -2479,19 +2455,17 @@ ahci_check_ids(device_t dev, union ccb *ccb)
 static void
 ahciaction(struct cam_sim *sim, union ccb *ccb)
 {
-	device_t dev, parent;
 	struct ahci_channel *ch;
 
 	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("ahciaction func_code=%x\n",
 	    ccb->ccb_h.func_code));
 
 	ch = (struct ahci_channel *)cam_sim_softc(sim);
-	dev = ch->dev;
 	switch (ccb->ccb_h.func_code) {
 	/* Common cases first */
 	case XPT_ATA_IO:	/* Execute the requested I/O operation */
 	case XPT_SCSI_IO:
-		if (ahci_check_ids(dev, ccb))
+		if (ahci_check_ids(ch, ccb))
 			return;
 		if (ch->devices == 0 ||
 		    (ch->pm_present == 0 &&
@@ -2501,14 +2475,14 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
 		}
 		ccb->ccb_h.recovery_type = RECOVERY_NONE;
 		/* Check for command collision. */
-		if (ahci_check_collision(dev, ccb)) {
+		if (ahci_check_collision(ch, ccb)) {
 			/* Freeze command. */
 			ch->frozen = ccb;
 			/* We have only one frozen slot, so freeze simq also. */
 			xpt_freeze_simq(ch->sim, 1);
 			return;
 		}
-		ahci_begin_transaction(dev, ccb);
+		ahci_begin_transaction(ch, ccb);
 		return;
 	case XPT_EN_LUN:		/* Enable LUN as a target */
 	case XPT_TARGET_IO:		/* Execute target I/O request */
@@ -2523,7 +2497,7 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
 		struct	ccb_trans_settings *cts = &ccb->cts;
 		struct	ahci_device *d; 
 
-		if (ahci_check_ids(dev, ccb))
+		if (ahci_check_ids(ch, ccb))
 			return;
 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
 			d = &ch->curr[ccb->ccb_h.target_id];
@@ -2553,7 +2527,7 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
 		struct  ahci_device *d;
 		uint32_t status;
 
-		if (ahci_check_ids(dev, ccb))
+		if (ahci_check_ids(ch, ccb))
 			return;
 		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
 			d = &ch->curr[ccb->ccb_h.target_id];
@@ -2610,7 +2584,7 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
 	}
 	case XPT_RESET_BUS:		/* Reset the specified SCSI bus */
 	case XPT_RESET_DEV:	/* Bus Device Reset the specified SCSI device */
-		ahci_reset(dev);
+		ahci_reset(ch);
 		ccb->ccb_h.status = CAM_REQ_CMP;
 		break;
 	case XPT_TERM_IO:		/* Terminate the I/O process */
@@ -2621,7 +2595,6 @@ ahciaction(struct cam_sim *sim, union ccb *ccb)
 	{
 		struct ccb_pathinq *cpi = &ccb->cpi;
 
-		parent = device_get_parent(dev);
 		cpi->version_num = 1; /* XXX??? */
 		cpi->hba_inquiry = PI_SDTR_ABLE;
 		if (ch->caps & AHCI_CAP_SNCQ)
@@ -2678,7 +2651,7 @@ ahcipoll(struct cam_sim *sim)
 	if (ch->resetting != 0 &&
 	    (--ch->resetpolldiv <= 0 || !callout_pending(&ch->reset_timer))) {
 		ch->resetpolldiv = 1000;
-		ahci_reset_to(ch->dev);
+		ahci_reset_to(ch);
 	}
 }
 MODULE_VERSION(ahci, 1);
diff --git a/sys/dev/ahci/ahci.h b/sys/dev/ahci/ahci.h
index 15d87a2..f5a3ca8 100644
--- a/sys/dev/ahci/ahci.h
+++ b/sys/dev/ahci/ahci.h
@@ -375,7 +375,7 @@ enum ahci_slot_states {
 };
 
 struct ahci_slot {
-    device_t                    dev;            /* Device handle */
+    struct ahci_channel		*ch;		/* Channel */
     u_int8_t			slot;           /* Number of this slot */
     enum ahci_slot_states	state;          /* Slot state */
     union ccb			*ccb;		/* CCB occupying slot */
@@ -422,20 +422,19 @@ struct ahci_channel {
 	int			quirks;
 	int			numslots;	/* Number of present slots */
 	int			pm_level;	/* power management level */
-
-	struct ahci_slot	slot[AHCI_MAX_SLOTS];
-	union ccb		*hold[AHCI_MAX_SLOTS];
-	struct mtx		mtx;		/* state lock */
-	STAILQ_HEAD(, ccb_hdr)	doneq;		/* queue of completed CCBs */
-	int			batch;		/* doneq is in use */
 	int			devices;        /* What is present */
 	int			pm_present;	/* PM presence reported */
 	int			fbs_enabled;	/* FIS-based switching enabled */
+
+	union ccb		*hold[AHCI_MAX_SLOTS];
+	struct ahci_slot	slot[AHCI_MAX_SLOTS];
 	uint32_t		oslots;		/* Occupied slots */
 	uint32_t		rslots;		/* Running slots */
 	uint32_t		aslots;		/* Slots with atomic commands  */
 	uint32_t		eslots;		/* Slots in error */
 	uint32_t		toslots;	/* Slots in timeout */
+	int			lastslot;	/* Last used slot */
+	int			taggedtarget;	/* Last tagged target */
 	int			numrslots;	/* Number of running slots */
 	int			numrslotspd[16];/* Number of running slots per dev */
 	int			numtslots;	/* Number of tagged slots */
@@ -443,8 +442,6 @@ struct ahci_channel {
 	int			numhslots;	/* Number of held slots */
 	int			recoverycmd;	/* Our READ LOG active */
 	int			fatalerr;	/* Fatal error happend */
-	int			lastslot;	/* Last used slot */
-	int			taggedtarget;	/* Last tagged target */
 	int			resetting;	/* Hard-reset in progress. */
 	int			resetpolldiv;	/* Hard-reset poll divider. */
 	int			listening;	/* SUD bit is cleared. */
@@ -455,6 +452,10 @@ struct ahci_channel {
 
 	struct ahci_device	user[16];	/* User-specified settings */
 	struct ahci_device	curr[16];	/* Current settings */
+
+	struct mtx_padalign	mtx;		/* state lock */
+	STAILQ_HEAD(, ccb_hdr)	doneq;		/* queue of completed CCBs */
+	int			batch;		/* doneq is in use */
 };
 
 struct ahci_enclosure {
diff --git a/sys/dev/ahci/ahciem.c b/sys/dev/ahci/ahciem.c
index 8785a78..94185ee 100644
--- a/sys/dev/ahci/ahciem.c
+++ b/sys/dev/ahci/ahciem.c
@@ -344,7 +344,7 @@ ahci_em_led(void *priv, int onoff)
 }
 
 static int
-ahci_check_ids(device_t dev, union ccb *ccb)
+ahci_check_ids(union ccb *ccb)
 {
 
 	if (ccb->ccb_h.target_id != 0) {
@@ -554,7 +554,7 @@ ahciemaction(struct cam_sim *sim, union ccb *ccb)
 	dev = enc->dev;
 	switch (ccb->ccb_h.func_code) {
 	case XPT_ATA_IO:	/* Execute the requested I/O operation */
-		if (ahci_check_ids(dev, ccb))
+		if (ahci_check_ids(ccb))
 			return;
 		ahci_em_begin_transaction(dev, ccb);
 		return;