Import mvs(4) - Marvell 88SX50XX/88SX60XX/88SX70XX/SoC SATA controllers

driver for CAM ATA subsystem. This driver supports same hardware as
atamarvell, ataadaptec and atamvsata drivers from ata(4), but provides
many additional features, such as NCQ, PMP, etc.

5 files changed, 3801 insertions, 0 deletions

diff --git a/sys/dev/mvs/mvs.c b/sys/dev/mvs/mvs.c
new file mode 100644
index 0000000..038e9f21
--- /dev/null
+++ b/sys/dev/mvs/mvs.c
@@ -0,0 +1,2173 @@
+/*-
+ * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/module.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/ata.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/malloc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <vm/uma.h>
+#include <machine/stdarg.h>
+#include <machine/resource.h>
+#include <machine/bus.h>
+#include <sys/rman.h>
+#include "mvs.h"
+
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+#include <cam/cam_sim.h>
+#include <cam/cam_xpt_sim.h>
+#include <cam/cam_debug.h>
+
+/* local prototypes */
+static int mvs_ch_suspend(device_t dev);
+static int mvs_ch_resume(device_t dev);
+static void mvs_dmainit(device_t dev);
+static void mvs_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error);
+static void mvs_dmafini(device_t dev);
+static void mvs_slotsalloc(device_t dev);
+static void mvs_slotsfree(device_t dev);
+static void mvs_setup_edma_queues(device_t dev);
+static void mvs_set_edma_mode(device_t dev, enum mvs_edma_mode mode);
+static void mvs_ch_pm(void *arg);
+static void mvs_ch_intr_locked(void *data);
+static void mvs_ch_intr(void *data);
+static void mvs_reset(device_t dev);
+static void mvs_softreset(device_t dev, union ccb *ccb);
+
+static int mvs_sata_connect(struct mvs_channel *ch);
+static int mvs_sata_phy_reset(device_t dev);
+static int mvs_wait(device_t dev, u_int s, u_int c, int t);
+static void mvs_tfd_read(device_t dev, union ccb *ccb);
+static void mvs_tfd_write(device_t dev, union ccb *ccb);
+static void mvs_legacy_intr(device_t dev);
+static void mvs_crbq_intr(device_t dev);
+static void mvs_begin_transaction(device_t dev, union ccb *ccb);
+static void mvs_legacy_execute_transaction(struct mvs_slot *slot);
+static void mvs_timeout(struct mvs_slot *slot);
+static void mvs_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error);
+static void mvs_requeue_frozen(device_t dev);
+static void mvs_execute_transaction(struct mvs_slot *slot);
+static void mvs_end_transaction(struct mvs_slot *slot, enum mvs_err_type et);
+
+static void mvs_issue_read_log(device_t dev);
+static void mvs_process_read_log(device_t dev, union ccb *ccb);
+
+static void mvsaction(struct cam_sim *sim, union ccb *ccb);
+static void mvspoll(struct cam_sim *sim);
+
+MALLOC_DEFINE(M_MVS, "MVS driver", "MVS driver data buffers");
+
+static int
+mvs_ch_probe(device_t dev)
+{
+
+	device_set_desc_copy(dev, "Marvell SATA channel");
+	return (0);
+}
+
+static int
+mvs_ch_attach(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(device_get_parent(dev));
+	struct mvs_channel *ch = device_get_softc(dev);
+	struct cam_devq *devq;
+	int rid, error, i, sata_rev = 0;
+
+	ch->dev = dev;
+	ch->unit = (intptr_t)device_get_ivars(dev);
+	ch->quirks = ctlr->quirks;
+	mtx_init(&ch->mtx, "MVS channel lock", NULL, MTX_DEF);
+	resource_int_value(device_get_name(dev),
+	    device_get_unit(dev), "pm_level", &ch->pm_level);
+	if (ch->pm_level > 3)
+		callout_init_mtx(&ch->pm_timer, &ch->mtx, 0);
+	resource_int_value(device_get_name(dev),
+	    device_get_unit(dev), "sata_rev", &sata_rev);
+	for (i = 0; i < 16; i++) {
+		ch->user[i].revision = sata_rev;
+		ch->user[i].mode = 0;
+		ch->user[i].bytecount = (ch->quirks & MVS_Q_GENIIE) ? 8192 : 2048;
+		ch->user[i].tags = MVS_MAX_SLOTS;
+		ch->curr[i] = ch->user[i];
+		if (ch->pm_level) {
+			ch->user[i].caps = CTS_SATA_CAPS_H_PMREQ |
+			    CTS_SATA_CAPS_H_APST |
+			    CTS_SATA_CAPS_D_PMREQ | CTS_SATA_CAPS_D_APST;
+		}
+	}
+	rid = ch->unit;
+	if (!(ch->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &rid, RF_ACTIVE)))
+		return (ENXIO);
+	mvs_dmainit(dev);
+	mvs_slotsalloc(dev);
+	mvs_ch_resume(dev);
+	mtx_lock(&ch->mtx);
+	rid = ATA_IRQ_RID;
+	if (!(ch->r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+	    &rid, RF_SHAREABLE | RF_ACTIVE))) {
+		device_printf(dev, "Unable to map interrupt\n");
+		error = ENXIO;
+		goto err0;
+	}
+	if ((bus_setup_intr(dev, ch->r_irq, ATA_INTR_FLAGS, NULL,
+	    mvs_ch_intr_locked, dev, &ch->ih))) {
+		device_printf(dev, "Unable to setup interrupt\n");
+		error = ENXIO;
+		goto err1;
+	}
+	/* Create the device queue for our SIM. */
+	devq = cam_simq_alloc(MVS_MAX_SLOTS - 1);
+	if (devq == NULL) {
+		device_printf(dev, "Unable to allocate simq\n");
+		error = ENOMEM;
+		goto err1;
+	}
+	/* Construct SIM entry */
+	ch->sim = cam_sim_alloc(mvsaction, mvspoll, "mvsch", ch,
+	    device_get_unit(dev), &ch->mtx,
+	    2, (ch->quirks & MVS_Q_GENI) ? 0 : MVS_MAX_SLOTS - 1,
+	    devq);
+	if (ch->sim == NULL) {
+		cam_simq_free(devq);
+		device_printf(dev, "unable to allocate sim\n");
+		error = ENOMEM;
+		goto err1;
+	}
+	if (xpt_bus_register(ch->sim, dev, 0) != CAM_SUCCESS) {
+		device_printf(dev, "unable to register xpt bus\n");
+		error = ENXIO;
+		goto err2;
+	}
+	if (xpt_create_path(&ch->path, /*periph*/NULL, cam_sim_path(ch->sim),
+	    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
+		device_printf(dev, "unable to create path\n");
+		error = ENXIO;
+		goto err3;
+	}
+	if (ch->pm_level > 3) {
+		callout_reset(&ch->pm_timer,
+		    (ch->pm_level == 4) ? hz / 1000 : hz / 8,
+		    mvs_ch_pm, dev);
+	}
+	mtx_unlock(&ch->mtx);
+	return (0);
+
+err3:
+	xpt_bus_deregister(cam_sim_path(ch->sim));
+err2:
+	cam_sim_free(ch->sim, /*free_devq*/TRUE);
+err1:
+	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
+err0:
+	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
+	mtx_unlock(&ch->mtx);
+	mtx_destroy(&ch->mtx);
+	return (error);
+}
+
+static int
+mvs_ch_detach(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+
+	mtx_lock(&ch->mtx);
+	xpt_async(AC_LOST_DEVICE, ch->path, NULL);
+	xpt_free_path(ch->path);
+	xpt_bus_deregister(cam_sim_path(ch->sim));
+	cam_sim_free(ch->sim, /*free_devq*/TRUE);
+	mtx_unlock(&ch->mtx);
+
+	if (ch->pm_level > 3)
+		callout_drain(&ch->pm_timer);
+	bus_teardown_intr(dev, ch->r_irq, ch->ih);
+	bus_release_resource(dev, SYS_RES_IRQ, ATA_IRQ_RID, ch->r_irq);
+
+	mvs_ch_suspend(dev);
+	mvs_slotsfree(dev);
+	mvs_dmafini(dev);
+
+	bus_release_resource(dev, SYS_RES_MEMORY, ch->unit, ch->r_mem);
+	mtx_destroy(&ch->mtx);
+	return (0);
+}
+
+static int
+mvs_ch_suspend(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+
+	/* Stop EDMA */
+	mvs_set_edma_mode(dev, MVS_EDMA_OFF);
+	/* Disable port interrupts. */
+	ATA_OUTL(ch->r_mem, EDMA_IEM, 0);
+	return (0);
+}
+
+static int
+mvs_ch_resume(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	uint32_t reg;
+
+	/* Disable port interrupts */
+	ATA_OUTL(ch->r_mem, EDMA_IEM, 0);
+	/* Stop EDMA */
+	ch->curr_mode = MVS_EDMA_UNKNOWN;
+	mvs_set_edma_mode(dev, MVS_EDMA_OFF);
+	/* Clear and configure FIS interrupts. */
+	ATA_OUTL(ch->r_mem, SATA_FISIC, 0);
+	reg = ATA_INL(ch->r_mem, SATA_FISC);
+	reg |= SATA_FISC_FISWAIT4HOSTRDYEN_B1;
+	ATA_OUTL(ch->r_mem, SATA_FISC, reg);
+	reg = ATA_INL(ch->r_mem, SATA_FISIM);
+	reg |= SATA_FISC_FISWAIT4HOSTRDYEN_B1;
+	ATA_OUTL(ch->r_mem, SATA_FISC, reg);
+	/* Clear SATA error register. */
+	ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
+	/* Clear any outstanding error interrupts. */
+	ATA_OUTL(ch->r_mem, EDMA_IEC, 0);
+	/* Unmask all error interrupts */
+	ATA_OUTL(ch->r_mem, EDMA_IEM, ~EDMA_IE_TRANSIENT);
+	return (0);
+}
+
+struct mvs_dc_cb_args {
+	bus_addr_t maddr;
+	int error;
+};
+
+static void
+mvs_dmainit(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	struct mvs_dc_cb_args dcba;
+
+	/* EDMA command request area. */
+	if (bus_dma_tag_create(bus_get_dma_tag(dev), 1024, 0,
+	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
+	    NULL, NULL, MVS_WORKRQ_SIZE, 1, MVS_WORKRQ_SIZE,
+	    0, NULL, NULL, &ch->dma.workrq_tag))
+		goto error;
+	if (bus_dmamem_alloc(ch->dma.workrq_tag, (void **)&ch->dma.workrq, 0,
+	    &ch->dma.workrq_map))
+		goto error;
+	if (bus_dmamap_load(ch->dma.workrq_tag, ch->dma.workrq_map, ch->dma.workrq,
+	    MVS_WORKRQ_SIZE, mvs_dmasetupc_cb, &dcba, 0) || dcba.error) {
+		bus_dmamem_free(ch->dma.workrq_tag, ch->dma.workrq, ch->dma.workrq_map);
+		goto error;
+	}
+	ch->dma.workrq_bus = dcba.maddr;
+	/* EDMA command response area. */
+	if (bus_dma_tag_create(bus_get_dma_tag(dev), 256, 0,
+	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
+	    NULL, NULL, MVS_WORKRP_SIZE, 1, MVS_WORKRP_SIZE,
+	    0, NULL, NULL, &ch->dma.workrp_tag))
+		goto error;
+	if (bus_dmamem_alloc(ch->dma.workrp_tag, (void **)&ch->dma.workrp, 0,
+	    &ch->dma.workrp_map))
+		goto error;
+	if (bus_dmamap_load(ch->dma.workrp_tag, ch->dma.workrp_map, ch->dma.workrp,
+	    MVS_WORKRP_SIZE, mvs_dmasetupc_cb, &dcba, 0) || dcba.error) {
+		bus_dmamem_free(ch->dma.workrp_tag, ch->dma.workrp, ch->dma.workrp_map);
+		goto error;
+	}
+	ch->dma.workrp_bus = dcba.maddr;
+	/* Data area. */
+	if (bus_dma_tag_create(bus_get_dma_tag(dev), 2, MVS_EPRD_MAX,
+	    BUS_SPACE_MAXADDR, BUS_SPACE_MAXADDR,
+	    NULL, NULL,
+	    MVS_SG_ENTRIES * PAGE_SIZE * MVS_MAX_SLOTS,
+	    MVS_SG_ENTRIES, MVS_EPRD_MAX,
+	    0, busdma_lock_mutex, &ch->mtx, &ch->dma.data_tag)) {
+		goto error;
+	}
+	return;
+
+error:
+	device_printf(dev, "WARNING - DMA initialization failed\n");
+	mvs_dmafini(dev);
+}
+
+static void
+mvs_dmasetupc_cb(void *xsc, bus_dma_segment_t *segs, int nsegs, int error)
+{
+	struct mvs_dc_cb_args *dcba = (struct mvs_dc_cb_args *)xsc;
+
+	if (!(dcba->error = error))
+		dcba->maddr = segs[0].ds_addr;
+}
+
+static void
+mvs_dmafini(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+
+	if (ch->dma.data_tag) {
+		bus_dma_tag_destroy(ch->dma.data_tag);
+		ch->dma.data_tag = NULL;
+	}
+	if (ch->dma.workrp_bus) {
+		bus_dmamap_unload(ch->dma.workrp_tag, ch->dma.workrp_map);
+		bus_dmamem_free(ch->dma.workrp_tag, ch->dma.workrp, ch->dma.workrp_map);
+		ch->dma.workrp_bus = 0;
+		ch->dma.workrp_map = NULL;
+		ch->dma.workrp = NULL;
+	}
+	if (ch->dma.workrp_tag) {
+		bus_dma_tag_destroy(ch->dma.workrp_tag);
+		ch->dma.workrp_tag = NULL;
+	}
+	if (ch->dma.workrq_bus) {
+		bus_dmamap_unload(ch->dma.workrq_tag, ch->dma.workrq_map);
+		bus_dmamem_free(ch->dma.workrq_tag, ch->dma.workrq, ch->dma.workrq_map);
+		ch->dma.workrq_bus = 0;
+		ch->dma.workrq_map = NULL;
+		ch->dma.workrq = NULL;
+	}
+	if (ch->dma.workrq_tag) {
+		bus_dma_tag_destroy(ch->dma.workrq_tag);
+		ch->dma.workrq_tag = NULL;
+	}
+}
+
+static void
+mvs_slotsalloc(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	int i;
+
+	/* Alloc and setup command/dma slots */
+	bzero(ch->slot, sizeof(ch->slot));
+	for (i = 0; i < MVS_MAX_SLOTS; i++) {
+		struct mvs_slot *slot = &ch->slot[i];
+
+		slot->dev = dev;
+		slot->slot = i;
+		slot->state = MVS_SLOT_EMPTY;
+		slot->ccb = NULL;
+		callout_init_mtx(&slot->timeout, &ch->mtx, 0);
+
+		if (bus_dmamap_create(ch->dma.data_tag, 0, &slot->dma.data_map))
+			device_printf(ch->dev, "FAILURE - create data_map\n");
+	}
+}
+
+static void
+mvs_slotsfree(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	int i;
+
+	/* Free all dma slots */
+	for (i = 0; i < MVS_MAX_SLOTS; i++) {
+		struct mvs_slot *slot = &ch->slot[i];
+
+		callout_drain(&slot->timeout);
+		if (slot->dma.data_map) {
+			bus_dmamap_destroy(ch->dma.data_tag, slot->dma.data_map);
+			slot->dma.data_map = NULL;
+		}
+	}
+}
+
+static void
+mvs_setup_edma_queues(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	uint64_t work;
+
+	/* Requests queue. */
+	work = ch->dma.workrq_bus;
+	ATA_OUTL(ch->r_mem, EDMA_REQQBAH, work >> 32);
+	ATA_OUTL(ch->r_mem, EDMA_REQQIP, work & 0xffffffff);
+	ATA_OUTL(ch->r_mem, EDMA_REQQOP, work & 0xffffffff);
+	bus_dmamap_sync(ch->dma.workrq_tag, ch->dma.workrq_map, BUS_DMASYNC_PREWRITE);
+	/* Reponses queue. */
+	bzero(ch->dma.workrp, 256);
+	work = ch->dma.workrp_bus;
+	ATA_OUTL(ch->r_mem, EDMA_RESQBAH, work >> 32);
+	ATA_OUTL(ch->r_mem, EDMA_RESQIP, work & 0xffffffff);
+	ATA_OUTL(ch->r_mem, EDMA_RESQOP, work & 0xffffffff);
+	bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map, BUS_DMASYNC_PREREAD);
+	ch->out_idx = 0;
+	ch->in_idx = 0;
+}
+
+static void
+mvs_set_edma_mode(device_t dev, enum mvs_edma_mode mode)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	int timeout;
+	uint32_t ecfg, fcfg, hc, ltm, unkn;
+
+	if (mode == ch->curr_mode)
+		return;
+	/* If we are running, we should stop first. */
+	if (ch->curr_mode != MVS_EDMA_OFF) {
+		ATA_OUTL(ch->r_mem, EDMA_CMD, EDMA_CMD_EDSEDMA);
+		timeout = 0;
+		while (ATA_INL(ch->r_mem, EDMA_CMD) & EDMA_CMD_EENEDMA) {
+			DELAY(1000);
+			if (timeout++ > 1000) {
+				device_printf(dev, "stopping EDMA engine failed\n");
+				break;
+			}
+		};
+	}
+	ch->curr_mode = mode;
+	ch->fbs_enabled = 0;
+	ch->fake_busy = 0;
+	/* Report mode to controller. Needed for correct CCC operation. */
+	MVS_EDMA(device_get_parent(dev), dev, mode);
+	/* Configure new mode. */
+	ecfg = EDMA_CFG_RESERVED | EDMA_CFG_RESERVED2 | EDMA_CFG_EHOSTQUEUECACHEEN;
+	if (ch->pm_present) {
+		ecfg |= EDMA_CFG_EMASKRXPM;
+		if (ch->quirks & MVS_Q_GENIIE) {
+			ecfg |= EDMA_CFG_EEDMAFBS;
+			ch->fbs_enabled = 1;
+		}
+	}
+	if (ch->quirks & MVS_Q_GENI)
+		ecfg |= EDMA_CFG_ERDBSZ;
+	else if (ch->quirks & MVS_Q_GENII)
+		ecfg |= EDMA_CFG_ERDBSZEXT | EDMA_CFG_EWRBUFFERLEN;
+	if (ch->quirks & MVS_Q_CT)
+		ecfg |= EDMA_CFG_ECUTTHROUGHEN;
+	if (mode != MVS_EDMA_OFF)
+		ecfg |= EDMA_CFG_EEARLYCOMPLETIONEN;
+	if (mode == MVS_EDMA_QUEUED)
+		ecfg |= EDMA_CFG_EQUE;
+	else if (mode == MVS_EDMA_NCQ)
+		ecfg |= EDMA_CFG_ESATANATVCMDQUE;
+	ATA_OUTL(ch->r_mem, EDMA_CFG, ecfg);
+	mvs_setup_edma_queues(dev);
+	if (ch->quirks & MVS_Q_GENIIE) {
+		/* Configure FBS-related registers */
+		fcfg = ATA_INL(ch->r_mem, SATA_FISC);
+		ltm = ATA_INL(ch->r_mem, SATA_LTM);
+		hc = ATA_INL(ch->r_mem, EDMA_HC);
+		if (ch->fbs_enabled) {
+			fcfg |= SATA_FISC_FISDMAACTIVATESYNCRESP;
+			if (mode == MVS_EDMA_NCQ) {
+				fcfg &= ~SATA_FISC_FISWAIT4HOSTRDYEN_B0;
+				hc &= ~EDMA_IE_EDEVERR;
+			} else {
+				fcfg |= SATA_FISC_FISWAIT4HOSTRDYEN_B0;
+				hc |= EDMA_IE_EDEVERR;
+			}
+			ltm |= (1 << 8);
+		} else {
+			fcfg &= ~SATA_FISC_FISDMAACTIVATESYNCRESP;
+			fcfg &= ~SATA_FISC_FISWAIT4HOSTRDYEN_B0;
+			hc |= EDMA_IE_EDEVERR;
+			ltm &= ~(1 << 8);
+		}
+		ATA_OUTL(ch->r_mem, SATA_FISC, fcfg);
+		ATA_OUTL(ch->r_mem, SATA_LTM, ltm);
+		ATA_OUTL(ch->r_mem, EDMA_HC, hc);
+		/* This is some magic, required to handle several DRQs
+		 * with basic DMA. */
+		unkn = ATA_INL(ch->r_mem, EDMA_UNKN_RESD);
+		if (mode == MVS_EDMA_OFF)
+			unkn |= 1;
+		else
+			unkn &= ~1;
+		ATA_OUTL(ch->r_mem, EDMA_UNKN_RESD, unkn);
+	}
+	/* Run EDMA. */
+	if (mode != MVS_EDMA_OFF)
+		ATA_OUTL(ch->r_mem, EDMA_CMD, EDMA_CMD_EENEDMA);
+}
+
+devclass_t mvs_devclass;
+devclass_t mvsch_devclass;
+static device_method_t mvsch_methods[] = {
+	DEVMETHOD(device_probe,     mvs_ch_probe),
+	DEVMETHOD(device_attach,    mvs_ch_attach),
+	DEVMETHOD(device_detach,    mvs_ch_detach),
+	DEVMETHOD(device_suspend,   mvs_ch_suspend),
+	DEVMETHOD(device_resume,    mvs_ch_resume),
+	{ 0, 0 }
+};
+static driver_t mvsch_driver = {
+        "mvsch",
+        mvsch_methods,
+        sizeof(struct mvs_channel)
+};
+DRIVER_MODULE(mvsch, mvs, mvsch_driver, mvsch_devclass, 0, 0);
+DRIVER_MODULE(mvsch, sata, mvsch_driver, mvsch_devclass, 0, 0);
+
+static void
+mvs_phy_check_events(device_t dev, u_int32_t serr)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+
+	if (ch->pm_level == 0) {
+		u_int32_t status = ATA_INL(ch->r_mem, SATA_SS);
+		union ccb *ccb;
+
+		if (bootverbose) {
+			if (((status & SATA_SS_DET_MASK) == SATA_SS_DET_PHY_ONLINE) &&
+			    ((status & SATA_SS_SPD_MASK) != SATA_SS_SPD_NO_SPEED) &&
+			    ((status & SATA_SS_IPM_MASK) == SATA_SS_IPM_ACTIVE)) {
+				device_printf(dev, "CONNECT requested\n");
+			} else
+				device_printf(dev, "DISCONNECT requested\n");
+		}
+		mvs_reset(dev);
+		if ((ccb = xpt_alloc_ccb_nowait()) == NULL)
+			return;
+		if (xpt_create_path(&ccb->ccb_h.path, NULL,
+		    cam_sim_path(ch->sim),
+		    CAM_TARGET_WILDCARD, CAM_LUN_WILDCARD) != CAM_REQ_CMP) {
+			xpt_free_ccb(ccb);
+			return;
+		}
+		xpt_rescan(ccb);
+	}
+}
+
+static void
+mvs_notify_events(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	struct cam_path *dpath;
+	uint32_t fis;
+	int d;
+
+	/* Try to read PMP field from SDB FIS. Present only for Gen-IIe. */
+	fis = ATA_INL(ch->r_mem, SATA_FISDW0);
+	if ((fis & 0x80ff) == 0x80a1)
+		d = (fis & 0x0f00) >> 8;
+	else
+		d = ch->pm_present ? 15 : 0;
+	if (bootverbose)
+		device_printf(dev, "SNTF %d\n", d);
+	if (xpt_create_path(&dpath, NULL,
+	    xpt_path_path_id(ch->path), d, 0) == CAM_REQ_CMP) {
+		xpt_async(AC_SCSI_AEN, dpath, NULL);
+		xpt_free_path(dpath);
+	}
+}
+
+static void
+mvs_ch_intr_locked(void *data)
+{
+	struct mvs_intr_arg *arg = (struct mvs_intr_arg *)data;
+	device_t dev = (device_t)arg->arg;
+	struct mvs_channel *ch = device_get_softc(dev);
+
+	mtx_lock(&ch->mtx);
+	mvs_ch_intr(data);
+	mtx_unlock(&ch->mtx);
+}
+
+static void
+mvs_ch_pm(void *arg)
+{
+	device_t dev = (device_t)arg;
+	struct mvs_channel *ch = device_get_softc(dev);
+	uint32_t work;
+
+	if (ch->numrslots != 0)
+		return;
+	/* If we are idle - request power state transition. */
+	work = ATA_INL(ch->r_mem, SATA_SC);
+	work &= ~SATA_SC_SPM_MASK;
+	if (ch->pm_level == 4)
+		work |= SATA_SC_SPM_PARTIAL;
+	else
+		work |= SATA_SC_SPM_SLUMBER;
+	ATA_OUTL(ch->r_mem, SATA_SC, work);
+}
+
+static void
+mvs_ch_pm_wake(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	uint32_t work;
+	int timeout = 0;
+
+	work = ATA_INL(ch->r_mem, SATA_SS);
+	if (work & SATA_SS_IPM_ACTIVE)
+		return;
+	/* If we are not in active state - request power state transition. */
+	work = ATA_INL(ch->r_mem, SATA_SC);
+	work &= ~SATA_SC_SPM_MASK;
+	work |= SATA_SC_SPM_ACTIVE;
+	ATA_OUTL(ch->r_mem, SATA_SC, work);
+	/* Wait for transition to happen. */
+	while ((ATA_INL(ch->r_mem, SATA_SS) & SATA_SS_IPM_ACTIVE) == 0 &&
+	    timeout++ < 100) {
+		DELAY(100);
+	}
+}
+
+static void
+mvs_ch_intr(void *data)
+{
+	struct mvs_intr_arg *arg = (struct mvs_intr_arg *)data;
+	device_t dev = (device_t)arg->arg;
+	struct mvs_channel *ch = device_get_softc(dev);
+	uint32_t iec, serr = 0, fisic = 0;
+	enum mvs_err_type et;
+	int i, ccs, port = -1, selfdis = 0;
+	int edma = (ch->numtslots != 0 || ch->numdslots != 0);
+
+//device_printf(dev, "irq cause %02x EDMA %d IEC %08x\n",
+//    arg->cause, edma, ATA_INL(ch->r_mem, EDMA_IEC));
+	/* New item in response queue. */
+	if ((arg->cause & 2) && edma)
+		mvs_crbq_intr(dev);
+	/* Some error or special event. */
+	if (arg->cause & 1) {
+		iec = ATA_INL(ch->r_mem, EDMA_IEC);
+//device_printf(dev, "irq cause %02x EDMA %d IEC %08x\n",
+//    arg->cause, edma, iec);
+		if (iec & EDMA_IE_SERRINT) {
+			serr = ATA_INL(ch->r_mem, SATA_SE);
+			ATA_OUTL(ch->r_mem, SATA_SE, serr);
+//device_printf(dev, "SERR %08x\n", serr);
+		}
+		/* EDMA self-disabled due to error. */
+		if (iec & EDMA_IE_ESELFDIS)
+			selfdis = 1;
+		/* Transport interrupt. */
+		if (iec & EDMA_IE_ETRANSINT) {
+			/* For Gen-I this bit means self-disable. */
+			if (ch->quirks & MVS_Q_GENI)
+				selfdis = 1;
+			/* For Gen-II this bit means SDB-N. */
+			else if (ch->quirks & MVS_Q_GENII)
+				fisic = SATA_FISC_FISWAIT4HOSTRDYEN_B1;
+			else	/* For Gen-IIe - read FIS interrupt cause. */
+				fisic = ATA_INL(ch->r_mem, SATA_FISIC);
+//device_printf(dev, "FISIC %08x\n", fisic);
+		}
+		if (selfdis)
+			ch->curr_mode = MVS_EDMA_UNKNOWN;
+		ATA_OUTL(ch->r_mem, EDMA_IEC, ~iec);
+		/* Interface errors or Device error. */
+		if (iec & (0xfc1e9000 | EDMA_IE_EDEVERR)) {
+			port = -1;
+			if (ch->numpslots != 0) {
+				ccs = 0;
+			} else {
+				if (ch->quirks & MVS_Q_GENIIE)
+					ccs = EDMA_S_EIOID(ATA_INL(ch->r_mem, EDMA_S));
+				else
+					ccs = EDMA_S_EDEVQUETAG(ATA_INL(ch->r_mem, EDMA_S));
+				/* Check if error is one-PMP-port-specific, */
+				if (ch->fbs_enabled) {
+					/* Which ports were active. */
+					for (i = 0; i < 16; i++) {
+						if (ch->numrslotspd[i] == 0)
+							continue;
+						if (port == -1)
+							port = i;
+						else if (port != i) {
+							port = -2;
+							break;
+						}
+					}
+					/* If several ports were active and EDMA still enabled - 
+					 * other ports are probably unaffected and may continue.
+					 */
+					if (port == -2 && !selfdis) {
+						uint16_t p = ATA_INL(ch->r_mem, SATA_SATAITC) >> 16;
+						port = ffs(p) - 1;
+						if (port != (fls(p) - 1))
+							port = -2;
+					}
+				}
+			}
+//device_printf(dev, "err slot %d port %d\n", ccs, port);
+			mvs_requeue_frozen(dev);
+			for (i = 0; i < MVS_MAX_SLOTS; i++) {
+				/* XXX: reqests in loading state. */
+				if (((ch->rslots >> i) & 1) == 0)
+					continue;
+				if (port >= 0 &&
+				    ch->slot[i].ccb->ccb_h.target_id != port)
+					continue;
+				if (iec & EDMA_IE_EDEVERR) { /* Device error. */
+				    if (port != -2) {
+					if (ch->numtslots == 0) {
+						/* Untagged operation. */
+						if (i == ccs)
+							et = MVS_ERR_TFE;
+						else
+							et = MVS_ERR_INNOCENT;
+					} else {
+						/* Tagged operation. */
+						et = MVS_ERR_NCQ;
+					}
+				    } else {
+					et = MVS_ERR_TFE;
+					ch->fatalerr = 1;
+				    }
+				} else if (iec & 0xfc1e9000) {
+					if (ch->numtslots == 0 && i != ccs && port != -2)
+						et = MVS_ERR_INNOCENT;
+					else
+						et = MVS_ERR_SATA;
+				} else
+					et = MVS_ERR_INVALID;
+				mvs_end_transaction(&ch->slot[i], et);
+			}
+		}
+		/* Process SDB-N. */
+		if (fisic & SATA_FISC_FISWAIT4HOSTRDYEN_B1)
+			mvs_notify_events(dev);
+		if (fisic)
+			ATA_OUTL(ch->r_mem, SATA_FISIC, ~fisic);
+		/* Process hot-plug. */
+		if ((iec & (EDMA_IE_EDEVDIS | EDMA_IE_EDEVCON)) ||
+		    (serr & SATA_SE_PHY_CHANGED))
+			mvs_phy_check_events(dev, serr);
+	}
+	/* Legacy mode device interrupt. */
+	if ((arg->cause & 2) && !edma)
+		mvs_legacy_intr(dev);
+}
+
+static uint8_t
+mvs_getstatus(device_t dev, int clear)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	uint8_t status = ATA_INB(ch->r_mem, clear ? ATA_STATUS : ATA_ALTSTAT);
+
+	if (ch->fake_busy) {
+		if (status & (ATA_S_BUSY | ATA_S_DRQ | ATA_S_ERROR))
+			ch->fake_busy = 0;
+		else
+			status |= ATA_S_BUSY;
+	}
+	return (status);
+}
+
+static void
+mvs_legacy_intr(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	struct mvs_slot *slot = &ch->slot[0]; /* PIO is always in slot 0. */
+	union ccb *ccb = slot->ccb;
+	enum mvs_err_type et = MVS_ERR_NONE;
+	int port;
+	u_int length;
+	uint8_t status, ireason;
+
+	/* Clear interrupt and get status. */
+	status = mvs_getstatus(dev, 1);
+//	device_printf(dev, "Legacy intr status %02x\n",
+//	    status);
+	if (slot->state < MVS_SLOT_RUNNING)
+	    return;
+	port = ccb->ccb_h.target_id & 0x0f;
+	/* Wait a bit for late !BUSY status update. */
+	if (status & ATA_S_BUSY) {
+		DELAY(100);
+		if ((status = mvs_getstatus(dev, 1)) & ATA_S_BUSY) {
+			DELAY(1000);
+			if ((status = mvs_getstatus(dev, 1)) & ATA_S_BUSY)
+				return;
+		}
+	}
+	/* If we got an error, we are done. */
+	if (status & ATA_S_ERROR) {
+		et = MVS_ERR_TFE;
+		goto end_finished;
+	}
+	if (ccb->ccb_h.func_code == XPT_ATA_IO) { /* ATA PIO */
+		ccb->ataio.res.status = status;
+		/* Are we moving data? */
+		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
+		    /* If data read command - get them. */
+		    if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
+			if (mvs_wait(dev, ATA_S_DRQ, ATA_S_BUSY, 1000) < 0) {
+			    device_printf(dev, "timeout waiting for read DRQ\n");
+			    et = MVS_ERR_TIMEOUT;
+			    goto end_finished;
+			}
+			ATA_INSW_STRM(ch->r_mem, ATA_DATA,
+			   (uint16_t *)(ccb->ataio.data_ptr + ch->donecount),
+			   ch->transfersize / 2);
+		    }
+		    /* Update how far we've gotten. */
+		    ch->donecount += ch->transfersize;
+		    /* Do we need more? */
+		    if (ccb->ataio.dxfer_len > ch->donecount) {
+			/* Set this transfer size according to HW capabilities */
+			ch->transfersize = min(ccb->ataio.dxfer_len - ch->donecount,
+			    ch->curr[ccb->ccb_h.target_id].bytecount);
+			/* If data write command - put them */
+			if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
+				if (mvs_wait(dev, ATA_S_DRQ, ATA_S_BUSY, 1000) < 0) {
+				    device_printf(dev, "timeout waiting for write DRQ\n");
+				    et = MVS_ERR_TIMEOUT;
+				    goto end_finished;
+				}
+				ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
+				   (uint16_t *)(ccb->ataio.data_ptr + ch->donecount),
+				   ch->transfersize / 2);
+				return;
+			}
+			/* If data read command, return & wait for interrupt */
+			if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN)
+				return;
+		    }
+		}
+	} else if (ch->basic_dma) {	/* ATAPI DMA */
+		if (status & ATA_S_DWF)
+			et = MVS_ERR_TFE;
+		else if (ATA_INL(ch->r_mem, DMA_S) & DMA_S_ERR)
+			et = MVS_ERR_TFE;
+		/* Stop basic DMA. */
+		ATA_OUTL(ch->r_mem, DMA_C, 0);
+		goto end_finished;
+	} else {			/* ATAPI PIO */
+		length = ATA_INB(ch->r_mem,ATA_CYL_LSB) | (ATA_INB(ch->r_mem,ATA_CYL_MSB) << 8);
+		ireason = ATA_INB(ch->r_mem,ATA_IREASON);
+//device_printf(dev, "status %02x, ireason %02x, length %d\n", status, ireason, length);
+		switch ((ireason & (ATA_I_CMD | ATA_I_IN)) |
+			(status & ATA_S_DRQ)) {
+
+		case ATAPI_P_CMDOUT:
+device_printf(dev, "ATAPI CMDOUT\n");
+		    /* Return wait for interrupt */
+		    return;
+
+		case ATAPI_P_WRITE:
+//device_printf(dev, "ATAPI WRITE\n");
+		    if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
+			device_printf(dev, "trying to write on read buffer\n");
+			et = MVS_ERR_TFE;
+			goto end_finished;
+			break;
+		    }
+		    ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
+			(uint16_t *)(ccb->csio.data_ptr + ch->donecount),
+			length / 2);
+		    ch->donecount += length;
+		    /* Set next transfer size according to HW capabilities */
+		    ch->transfersize = min(ccb->csio.dxfer_len - ch->donecount,
+			    ch->curr[ccb->ccb_h.target_id].bytecount);
+		    /* Return wait for interrupt */
+		    return;
+
+		case ATAPI_P_READ:
+//device_printf(dev, "ATAPI READ\n");
+		    if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
+			device_printf(dev, "trying to read on write buffer\n");
+			et = MVS_ERR_TFE;
+			goto end_finished;
+		    }
+		    ATA_INSW_STRM(ch->r_mem, ATA_DATA,
+			(uint16_t *)(ccb->csio.data_ptr + ch->donecount),
+			length / 2);
+		    ch->donecount += length;
+		    /* Set next transfer size according to HW capabilities */
+		    ch->transfersize = min(ccb->csio.dxfer_len - ch->donecount,
+			    ch->curr[ccb->ccb_h.target_id].bytecount);
+		    /* Return wait for interrupt */
+		    return;
+
+		case ATAPI_P_DONEDRQ:
+device_printf(dev, "ATAPI DONEDRQ\n");
+		    device_printf(dev,
+			  "WARNING - DONEDRQ non conformant device\n");
+		    if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) {
+			ATA_INSW_STRM(ch->r_mem, ATA_DATA,
+			    (uint16_t *)(ccb->csio.data_ptr + ch->donecount),
+			    length / 2);
+			ch->donecount += length;
+		    }
+		    else if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
+			ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
+			    (uint16_t *)(ccb->csio.data_ptr + ch->donecount),
+			    length / 2);
+			ch->donecount += length;
+		    }
+		    else
+			et = MVS_ERR_TFE;
+		    /* FALLTHROUGH */
+
+		case ATAPI_P_ABORT:
+		case ATAPI_P_DONE:
+//device_printf(dev, "ATAPI ABORT/DONE\n");
+		    if (status & (ATA_S_ERROR | ATA_S_DWF))
+			et = MVS_ERR_TFE;
+		    goto end_finished;
+
+		default:
+		    device_printf(dev, "unknown transfer phase (status %02x, ireason %02x)\n",
+			status, ireason);
+		    et = MVS_ERR_TFE;
+		}
+	}
+
+end_finished:
+	mvs_end_transaction(slot, et);
+}
+
+static void
+mvs_crbq_intr(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	struct mvs_crpb *crpb;
+	union ccb *ccb;
+	int in_idx, cin_idx, slot;
+	uint16_t flags;
+
+	in_idx = (ATA_INL(ch->r_mem, EDMA_RESQIP) & EDMA_RESQP_ERPQP_MASK) >>
+	    EDMA_RESQP_ERPQP_SHIFT;
+	bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
+	    BUS_DMASYNC_POSTREAD);
+	cin_idx = ch->in_idx;
+	ch->in_idx = in_idx;
+	while (in_idx != cin_idx) {
+		crpb = (struct mvs_crpb *)
+		    (ch->dma.workrp + MVS_CRPB_OFFSET + (MVS_CRPB_SIZE * cin_idx));
+		slot = le16toh(crpb->id) & MVS_CRPB_TAG_MASK;
+		flags = le16toh(crpb->rspflg);
+//device_printf(dev, "CRPB %d %d %04x\n", cin_idx, slot, flags);
+		/*
+		 * Handle only successfull completions here.
+		 * Errors will be handled by main intr handler.
+		 */
+		if (ch->numtslots != 0 || (flags & EDMA_IE_EDEVERR) == 0) {
+if ((flags >> 8) & ATA_S_ERROR)
+device_printf(dev, "ERROR STATUS CRPB %d %d %04x\n", cin_idx, slot, flags);
+			if (ch->slot[slot].state >= MVS_SLOT_RUNNING) {
+				ccb = ch->slot[slot].ccb;
+				ccb->ataio.res.status = (flags & MVS_CRPB_ATASTS_MASK) >>
+				    MVS_CRPB_ATASTS_SHIFT;
+				mvs_end_transaction(&ch->slot[slot], MVS_ERR_NONE);
+			} else 
+device_printf(dev, "EMPTY CRPB %d (->%d) %d %04x\n", cin_idx, in_idx, slot, flags);
+		} else
+device_printf(dev, "ERROR FLAGS CRPB %d %d %04x\n", cin_idx, slot, flags);
+
+		cin_idx = (cin_idx + 1) & (MVS_MAX_SLOTS - 1);
+	}
+	bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
+	    BUS_DMASYNC_PREREAD);
+	if (cin_idx == ch->in_idx) {
+		ATA_OUTL(ch->r_mem, EDMA_RESQOP,
+		    ch->dma.workrp_bus | (cin_idx << EDMA_RESQP_ERPQP_SHIFT));
+	}
+}
+
+/* Must be called with channel locked. */
+static int
+mvs_check_collision(device_t dev, union ccb *ccb)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+
+	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
+		/* NCQ DMA */
+		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
+			/* Can't mix NCQ and non-NCQ DMA commands. */
+			if (ch->numdslots != 0)
+				return (1);
+			/* Can't mix NCQ and PIO commands. */
+			if (ch->numpslots != 0)
+				return (1);
+			/* If we have no FBS */
+			if (!ch->fbs_enabled) {
+				/* Tagged command while tagged to other target is active. */
+				if (ch->numtslots != 0 &&
+				    ch->taggedtarget != ccb->ccb_h.target_id)
+					return (1);
+			}
+		/* Non-NCQ DMA */
+		} else if (ccb->ataio.cmd.flags & CAM_ATAIO_DMA) {
+			/* Can't mix non-NCQ DMA and NCQ commands. */
+			if (ch->numtslots != 0)
+				return (1);
+			/* Can't mix non-NCQ DMA and PIO commands. */
+			if (ch->numpslots != 0)
+				return (1);
+		/* PIO */
+		} else {
+			/* Can't mix PIO with anything. */
+			if (ch->numrslots != 0)
+				return (1);
+		}
+		if (ccb->ataio.cmd.flags & (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)) {
+			/* Atomic command while anything active. */
+			if (ch->numrslots != 0)
+				return (1);
+		}
+	} else { /* ATAPI */
+		/* ATAPI goes without EDMA, so can't mix it with anything. */
+		if (ch->numrslots != 0)
+			return (1);
+	}
+	/* We have some atomic command running. */
+	if (ch->aslots != 0)
+		return (1);
+	return (0);
+}
+
+static void
+mvs_tfd_read(device_t dev, union ccb *ccb)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	struct ata_res *res = &ccb->ataio.res;
+
+	res->status = ATA_INB(ch->r_mem, ATA_ALTSTAT);
+	res->error =  ATA_INB(ch->r_mem, ATA_ERROR);
+	res->device = ATA_INB(ch->r_mem, ATA_DRIVE);
+	ATA_OUTB(ch->r_mem, ATA_CONTROL, ATA_A_HOB);
+	res->sector_count_exp = ATA_INB(ch->r_mem, ATA_COUNT);
+	res->lba_low_exp = ATA_INB(ch->r_mem, ATA_SECTOR);
+	res->lba_mid_exp = ATA_INB(ch->r_mem, ATA_CYL_LSB);
+	res->lba_high_exp = ATA_INB(ch->r_mem, ATA_CYL_MSB);
+	ATA_OUTB(ch->r_mem, ATA_CONTROL, 0);
+	res->sector_count = ATA_INB(ch->r_mem, ATA_COUNT);
+	res->lba_low = ATA_INB(ch->r_mem, ATA_SECTOR);
+	res->lba_mid = ATA_INB(ch->r_mem, ATA_CYL_LSB);
+	res->lba_high = ATA_INB(ch->r_mem, ATA_CYL_MSB);
+}
+
+static void
+mvs_tfd_write(device_t dev, union ccb *ccb)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	struct ata_cmd *cmd = &ccb->ataio.cmd;
+
+	ATA_OUTB(ch->r_mem, ATA_DRIVE, cmd->device);
+	ATA_OUTB(ch->r_mem, ATA_CONTROL, cmd->control);
+	ATA_OUTB(ch->r_mem, ATA_FEATURE, cmd->features_exp);
+	ATA_OUTB(ch->r_mem, ATA_FEATURE, cmd->features);
+	ATA_OUTB(ch->r_mem, ATA_COUNT, cmd->sector_count_exp);
+	ATA_OUTB(ch->r_mem, ATA_COUNT, cmd->sector_count);
+	ATA_OUTB(ch->r_mem, ATA_SECTOR, cmd->lba_low_exp);
+	ATA_OUTB(ch->r_mem, ATA_SECTOR, cmd->lba_low);
+	ATA_OUTB(ch->r_mem, ATA_CYL_LSB, cmd->lba_mid_exp);
+	ATA_OUTB(ch->r_mem, ATA_CYL_LSB, cmd->lba_mid);
+	ATA_OUTB(ch->r_mem, ATA_CYL_MSB, cmd->lba_high_exp);
+	ATA_OUTB(ch->r_mem, ATA_CYL_MSB, cmd->lba_high);
+	ATA_OUTB(ch->r_mem, ATA_COMMAND, cmd->command);
+}
+
+
+/* Must be called with channel locked. */
+static void
+mvs_begin_transaction(device_t dev, union ccb *ccb)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	struct mvs_slot *slot;
+	int slotn, tag;
+
+	if (ch->pm_level > 0)
+		mvs_ch_pm_wake(dev);
+	/* Softreset is a special case. */
+	if (ccb->ccb_h.func_code == XPT_ATA_IO &&
+	    (ccb->ataio.cmd.flags & CAM_ATAIO_CONTROL)) {
+		mvs_softreset(dev, ccb);
+		return;
+	}
+	/* Choose empty slot. */
+	slotn = ffs(~ch->oslots) - 1;
+	if ((ccb->ccb_h.func_code == XPT_ATA_IO) &&
+	    (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA)) {
+		if (ch->quirks & MVS_Q_GENIIE)
+			tag = ffs(~ch->otagspd[ccb->ccb_h.target_id]) - 1;
+		else
+			tag = slotn;
+	} else
+		tag = 0;
+	/* Occupy chosen slot. */
+	slot = &ch->slot[slotn];
+	slot->ccb = ccb;
+	slot->tag = tag;
+	/* Stop PM timer. */
+	if (ch->numrslots == 0 && ch->pm_level > 3)
+		callout_stop(&ch->pm_timer);
+	/* Update channel stats. */
+	ch->oslots |= (1 << slot->slot);
+	ch->numrslots++;
+	ch->numrslotspd[ccb->ccb_h.target_id]++;
+	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
+		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
+			ch->otagspd[ccb->ccb_h.target_id] |= (1 << slot->tag);
+			ch->numtslots++;
+			ch->numtslotspd[ccb->ccb_h.target_id]++;
+			ch->taggedtarget = ccb->ccb_h.target_id;
+			mvs_set_edma_mode(dev, MVS_EDMA_NCQ);
+		} else if (ccb->ataio.cmd.flags & CAM_ATAIO_DMA) {
+			ch->numdslots++;
+			mvs_set_edma_mode(dev, MVS_EDMA_ON);
+		} else {
+			ch->numpslots++;
+			mvs_set_edma_mode(dev, MVS_EDMA_OFF);
+		}
+		if (ccb->ataio.cmd.flags &
+		    (CAM_ATAIO_CONTROL | CAM_ATAIO_NEEDRESULT)) {
+			ch->aslots |= (1 << slot->slot);
+		}
+	} else {
+		uint8_t *cdb = (ccb->ccb_h.flags & CAM_CDB_POINTER) ?
+		    ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes;
+		ch->numpslots++;
+		/* Use ATAPI DMA only for commands without under-/overruns. */
+		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE &&
+		    ch->curr[ccb->ccb_h.target_id].mode >= ATA_DMA &&
+		    (ch->quirks & MVS_Q_SOC) == 0 &&
+		    (cdb[0] == 0x08 ||
+		     cdb[0] == 0x0a ||
+		     cdb[0] == 0x28 ||
+		     cdb[0] == 0x2a ||
+		     cdb[0] == 0x88 ||
+		     cdb[0] == 0x8a ||
+		     cdb[0] == 0xa8 ||
+		     cdb[0] == 0xaa ||
+		     cdb[0] == 0xbe)) {
+			ch->basic_dma = 1;
+		}
+		mvs_set_edma_mode(dev, MVS_EDMA_OFF);
+	}
+	if (ch->numpslots == 0 || ch->basic_dma) {
+		void *buf;
+		bus_size_t size;
+
+		slot->state = MVS_SLOT_LOADING;
+		if (ccb->ccb_h.func_code == XPT_ATA_IO) {
+			buf = ccb->ataio.data_ptr;
+			size = ccb->ataio.dxfer_len;
+		} else {
+			buf = ccb->csio.data_ptr;
+			size = ccb->csio.dxfer_len;
+		}
+		bus_dmamap_load(ch->dma.data_tag, slot->dma.data_map,
+		    buf, size, mvs_dmasetprd, slot, 0);
+	} else
+		mvs_legacy_execute_transaction(slot);
+}
+
+/* Locked by busdma engine. */
+static void
+mvs_dmasetprd(void *arg, bus_dma_segment_t *segs, int nsegs, int error)
+{    
+	struct mvs_slot *slot = arg;
+	struct mvs_channel *ch = device_get_softc(slot->dev);
+	struct mvs_eprd *eprd;
+	int i;
+
+	if (error) {
+		device_printf(slot->dev, "DMA load error\n");
+		mvs_end_transaction(slot, MVS_ERR_INVALID);
+		return;
+	}
+	KASSERT(nsegs <= MVS_SG_ENTRIES, ("too many DMA segment entries\n"));
+	/* If there is only one segment - no need to use S/G table on Gen-IIe. */
+	if (nsegs == 1 && ch->basic_dma == 0 && (ch->quirks & MVS_Q_GENIIE)) {
+		slot->dma.addr = segs[0].ds_addr;
+		slot->dma.len = segs[0].ds_len;
+	} else {
+		slot->dma.addr = 0;
+		/* Get a piece of the workspace for this EPRD */
+		eprd = (struct mvs_eprd *)
+		    (ch->dma.workrq + MVS_EPRD_OFFSET + (MVS_EPRD_SIZE * slot->slot));
+		/* Fill S/G table */
+		for (i = 0; i < nsegs; i++) {
+			eprd[i].prdbal = htole32(segs[i].ds_addr);
+			eprd[i].bytecount = htole32(segs[i].ds_len & MVS_EPRD_MASK);
+			eprd[i].prdbah = htole32((segs[i].ds_addr >> 16) >> 16);
+		}
+		eprd[i - 1].bytecount |= htole32(MVS_EPRD_EOF);
+	}
+	bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
+	    ((slot->ccb->ccb_h.flags & CAM_DIR_IN) ?
+	    BUS_DMASYNC_PREREAD : BUS_DMASYNC_PREWRITE));
+	if (ch->basic_dma)
+		mvs_legacy_execute_transaction(slot);
+	else
+		mvs_execute_transaction(slot);
+}
+
+static void
+mvs_legacy_execute_transaction(struct mvs_slot *slot)
+{
+	device_t dev = slot->dev;
+	struct mvs_channel *ch = device_get_softc(dev);
+	bus_addr_t eprd;
+	union ccb *ccb = slot->ccb;
+	int port = ccb->ccb_h.target_id & 0x0f;
+	int timeout;
+
+	slot->state = MVS_SLOT_RUNNING;
+	ch->rslots |= (1 << slot->slot);
+	ATA_OUTB(ch->r_mem, SATA_SATAICTL, port << SATA_SATAICTL_PMPTX_SHIFT);
+	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
+//		device_printf(dev, "%d Legacy command %02x size %d\n",
+//		    port, ccb->ataio.cmd.command, ccb->ataio.dxfer_len);
+		mvs_tfd_write(dev, ccb);
+		/* Device reset doesn't interrupt. */
+		if (ccb->ataio.cmd.command == ATA_DEVICE_RESET) {
+			int timeout = 1000000;
+			do {
+			    DELAY(10);
+			    ccb->ataio.res.status = ATA_INB(ch->r_mem, ATA_STATUS);
+			} while (ccb->ataio.res.status & ATA_S_BUSY && timeout--);
+			mvs_legacy_intr(dev);
+			return;
+		}
+		ch->donecount = 0;
+		ch->transfersize = min(ccb->ataio.dxfer_len,
+		    ch->curr[port].bytecount);
+		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE)
+			ch->fake_busy = 1;
+		/* If data write command - output the data */
+		if ((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_OUT) {
+			if (mvs_wait(dev, ATA_S_DRQ, ATA_S_BUSY, 1000) < 0) {
+				device_printf(dev, "timeout waiting for write DRQ\n");
+				mvs_end_transaction(slot, MVS_ERR_TIMEOUT);
+				return;
+			}
+			ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
+			   (uint16_t *)(ccb->ataio.data_ptr + ch->donecount),
+			   ch->transfersize / 2);
+		}
+	} else {
+//		device_printf(dev, "%d ATAPI command %02x size %d dma %d\n",
+//		    port, ccb->csio.cdb_io.cdb_bytes[0], ccb->csio.dxfer_len,
+//		    ch->basic_dma);
+		ch->donecount = 0;
+		ch->transfersize = min(ccb->csio.dxfer_len,
+		    ch->curr[port].bytecount);
+		/* Write ATA PACKET command. */
+		if (ch->basic_dma) {
+			ATA_OUTB(ch->r_mem, ATA_FEATURE, ATA_F_DMA);
+			ATA_OUTB(ch->r_mem, ATA_CYL_LSB, 0);
+		    	ATA_OUTB(ch->r_mem, ATA_CYL_MSB, 0);
+		} else {
+			ATA_OUTB(ch->r_mem, ATA_FEATURE, 0);
+			ATA_OUTB(ch->r_mem, ATA_CYL_LSB, ch->transfersize);
+		    	ATA_OUTB(ch->r_mem, ATA_CYL_MSB, ch->transfersize >> 8);
+		}
+		ATA_OUTB(ch->r_mem, ATA_COMMAND, ATA_PACKET_CMD);
+		ch->fake_busy = 1;
+		/* Wait for ready to write ATAPI command block */
+		if (mvs_wait(dev, 0, ATA_S_BUSY, 1000) < 0) {
+			device_printf(dev, "timeout waiting for ATAPI !BUSY\n");
+			mvs_end_transaction(slot, MVS_ERR_TIMEOUT);
+			return;
+		}
+		timeout = 5000;
+		while (timeout--) {
+		    int reason = ATA_INB(ch->r_mem, ATA_IREASON);
+		    int status = ATA_INB(ch->r_mem, ATA_STATUS);
+
+		    if (((reason & (ATA_I_CMD | ATA_I_IN)) |
+			 (status & (ATA_S_DRQ | ATA_S_BUSY))) == ATAPI_P_CMDOUT)
+			break;
+		    DELAY(20);
+		}
+		if (timeout <= 0) {
+			device_printf(dev, "timeout waiting for ATAPI command ready\n");
+			mvs_end_transaction(slot, MVS_ERR_TIMEOUT);
+			return;
+		}
+		/* Write ATAPI command. */
+		ATA_OUTSW_STRM(ch->r_mem, ATA_DATA,
+		   (uint16_t *)((ccb->ccb_h.flags & CAM_CDB_POINTER) ?
+		    ccb->csio.cdb_io.cdb_ptr : ccb->csio.cdb_io.cdb_bytes),
+		   ch->curr[port].atapi / 2);
+		DELAY(10);
+		if (ch->basic_dma) {
+			/* Start basic DMA. */
+			eprd = ch->dma.workrq_bus + MVS_EPRD_OFFSET +
+			    (MVS_EPRD_SIZE * slot->slot);
+			ATA_OUTL(ch->r_mem, DMA_DTLBA, eprd);
+			ATA_OUTL(ch->r_mem, DMA_DTHBA, (eprd >> 16) >> 16);
+			ATA_OUTL(ch->r_mem, DMA_C, DMA_C_START |
+			    (((ccb->ccb_h.flags & CAM_DIR_MASK) == CAM_DIR_IN) ?
+			    DMA_C_READ : 0));
+		} else if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE)
+			ch->fake_busy = 1;
+	}
+	/* Start command execution timeout */
+	callout_reset(&slot->timeout, (int)ccb->ccb_h.timeout * hz / 1000,
+	    (timeout_t*)mvs_timeout, slot);
+}
+
+/* Must be called with channel locked. */
+static void
+mvs_execute_transaction(struct mvs_slot *slot)
+{
+	device_t dev = slot->dev;
+	struct mvs_channel *ch = device_get_softc(dev);
+	bus_addr_t eprd;
+	struct mvs_crqb *crqb;
+	struct mvs_crqb_gen2e *crqb2e;
+	union ccb *ccb = slot->ccb;
+	int port = ccb->ccb_h.target_id & 0x0f;
+	int i;
+
+//	device_printf(dev, "%d EDMA command %02x size %d slot %d tag %d\n",
+//	    port, ccb->ataio.cmd.command, ccb->ataio.dxfer_len, slot->slot, slot->tag);
+	/* Get address of the prepared EPRD */
+	eprd = ch->dma.workrq_bus + MVS_EPRD_OFFSET + (MVS_EPRD_SIZE * slot->slot);
+	/* Prepare CRQB. Gen IIe uses different CRQB format. */
+	if (ch->quirks & MVS_Q_GENIIE) {
+		crqb2e = (struct mvs_crqb_gen2e *)
+		    (ch->dma.workrq + MVS_CRQB_OFFSET + (MVS_CRQB_SIZE * ch->out_idx));
+		crqb2e->ctrlflg = htole32(
+		    ((ccb->ccb_h.flags & CAM_DIR_IN) ? MVS_CRQB2E_READ : 0) |
+		    (slot->tag << MVS_CRQB2E_DTAG_SHIFT) |
+		    (port << MVS_CRQB2E_PMP_SHIFT) |
+		    (slot->slot << MVS_CRQB2E_HTAG_SHIFT));
+		/* If there is only one segment - no need to use S/G table. */
+		if (slot->dma.addr != 0) {
+			eprd = slot->dma.addr;
+			crqb2e->ctrlflg |= htole32(MVS_CRQB2E_CPRD);
+			crqb2e->drbc = slot->dma.len;
+		}
+		crqb2e->cprdbl = htole32(eprd);
+		crqb2e->cprdbh = htole32((eprd >> 16) >> 16);
+		crqb2e->cmd[0] = 0;
+		crqb2e->cmd[1] = 0;
+		crqb2e->cmd[2] = ccb->ataio.cmd.command;
+		crqb2e->cmd[3] = ccb->ataio.cmd.features;
+		crqb2e->cmd[4] = ccb->ataio.cmd.lba_low;
+		crqb2e->cmd[5] = ccb->ataio.cmd.lba_mid;
+		crqb2e->cmd[6] = ccb->ataio.cmd.lba_high;
+		crqb2e->cmd[7] = ccb->ataio.cmd.device;
+		crqb2e->cmd[8] = ccb->ataio.cmd.lba_low_exp;
+		crqb2e->cmd[9] = ccb->ataio.cmd.lba_mid_exp;
+		crqb2e->cmd[10] = ccb->ataio.cmd.lba_high_exp;
+		crqb2e->cmd[11] = ccb->ataio.cmd.features_exp;
+		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
+			crqb2e->cmd[12] = slot->tag << 3;
+			crqb2e->cmd[13] = 0;
+		} else {
+			crqb2e->cmd[12] = ccb->ataio.cmd.sector_count;
+			crqb2e->cmd[13] = ccb->ataio.cmd.sector_count_exp;
+		}
+		crqb2e->cmd[14] = 0;
+		crqb2e->cmd[15] = 0;
+	} else {
+		crqb = (struct mvs_crqb *)
+		    (ch->dma.workrq + MVS_CRQB_OFFSET + (MVS_CRQB_SIZE * ch->out_idx));
+		crqb->cprdbl = htole32(eprd);
+		crqb->cprdbh = htole32((eprd >> 16) >> 16);
+		crqb->ctrlflg = htole16(
+		    ((ccb->ccb_h.flags & CAM_DIR_IN) ? MVS_CRQB_READ : 0) |
+		    (slot->slot << MVS_CRQB_TAG_SHIFT) |
+		    (port << MVS_CRQB_PMP_SHIFT));
+		i = 0;
+		/*
+		 * Controller can handle only 11 of 12 ATA registers,
+		 * so we have to choose which one to skip.
+		 */
+		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
+			crqb->cmd[i++] = ccb->ataio.cmd.features_exp;
+			crqb->cmd[i++] = 0x11;
+		}
+		crqb->cmd[i++] = ccb->ataio.cmd.features;
+		crqb->cmd[i++] = 0x11;
+		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
+			crqb->cmd[i++] = slot->tag << 3;
+			crqb->cmd[i++] = 0x12;
+		} else {
+			crqb->cmd[i++] = ccb->ataio.cmd.sector_count_exp;
+			crqb->cmd[i++] = 0x12;
+			crqb->cmd[i++] = ccb->ataio.cmd.sector_count;
+			crqb->cmd[i++] = 0x12;
+		}
+		crqb->cmd[i++] = ccb->ataio.cmd.lba_low_exp;
+		crqb->cmd[i++] = 0x13;
+		crqb->cmd[i++] = ccb->ataio.cmd.lba_low;
+		crqb->cmd[i++] = 0x13;
+		crqb->cmd[i++] = ccb->ataio.cmd.lba_mid_exp;
+		crqb->cmd[i++] = 0x14;
+		crqb->cmd[i++] = ccb->ataio.cmd.lba_mid;
+		crqb->cmd[i++] = 0x14;
+		crqb->cmd[i++] = ccb->ataio.cmd.lba_high_exp;
+		crqb->cmd[i++] = 0x15;
+		crqb->cmd[i++] = ccb->ataio.cmd.lba_high;
+		crqb->cmd[i++] = 0x15;
+		crqb->cmd[i++] = ccb->ataio.cmd.device;
+		crqb->cmd[i++] = 0x16;
+		crqb->cmd[i++] = ccb->ataio.cmd.command;
+		crqb->cmd[i++] = 0x97;
+	}
+	bus_dmamap_sync(ch->dma.workrq_tag, ch->dma.workrq_map,
+	    BUS_DMASYNC_PREWRITE);
+	bus_dmamap_sync(ch->dma.workrp_tag, ch->dma.workrp_map,
+	    BUS_DMASYNC_PREREAD);
+	slot->state = MVS_SLOT_RUNNING;
+	ch->rslots |= (1 << slot->slot);
+	/* Issue command to the controller. */
+	ch->out_idx = (ch->out_idx + 1) & (MVS_MAX_SLOTS - 1);
+	ATA_OUTL(ch->r_mem, EDMA_REQQIP,
+	    ch->dma.workrq_bus + MVS_CRQB_OFFSET + (MVS_CRQB_SIZE * ch->out_idx));
+	/* Start command execution timeout */
+	callout_reset(&slot->timeout, (int)ccb->ccb_h.timeout * hz / 1000,
+	    (timeout_t*)mvs_timeout, slot);
+	return;
+}
+
+/* Must be called with channel locked. */
+static void
+mvs_process_timeout(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	int i;
+
+	mtx_assert(&ch->mtx, MA_OWNED);
+	/* Handle the rest of commands. */
+	for (i = 0; i < MVS_MAX_SLOTS; i++) {
+		/* Do we have a running request on slot? */
+		if (ch->slot[i].state < MVS_SLOT_RUNNING)
+			continue;
+		mvs_end_transaction(&ch->slot[i], MVS_ERR_TIMEOUT);
+	}
+}
+
+/* Must be called with channel locked. */
+static void
+mvs_rearm_timeout(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	int i;
+
+	mtx_assert(&ch->mtx, MA_OWNED);
+	for (i = 0; i < MVS_MAX_SLOTS; i++) {
+		struct mvs_slot *slot = &ch->slot[i];
+
+		/* Do we have a running request on slot? */
+		if (slot->state < MVS_SLOT_RUNNING)
+			continue;
+		if ((ch->toslots & (1 << i)) == 0)
+			continue;
+		callout_reset(&slot->timeout,
+		    (int)slot->ccb->ccb_h.timeout * hz / 2000,
+		    (timeout_t*)mvs_timeout, slot);
+	}
+}
+
+/* Locked by callout mechanism. */
+static void
+mvs_timeout(struct mvs_slot *slot)
+{
+	device_t dev = slot->dev;
+	struct mvs_channel *ch = device_get_softc(dev);
+
+	/* Check for stale timeout. */
+	if (slot->state < MVS_SLOT_RUNNING)
+		return;
+	device_printf(dev, "Timeout on slot %d\n", slot->slot);
+	device_printf(dev, "iec %08x sstat %08x serr %08x edma_s %08x "
+	    "dma_c %08x dma_s %08x rs %08x status %02x\n",
+	    ATA_INL(ch->r_mem, EDMA_IEC),
+	    ATA_INL(ch->r_mem, SATA_SS), ATA_INL(ch->r_mem, SATA_SE),
+	    ATA_INL(ch->r_mem, EDMA_S), ATA_INL(ch->r_mem, DMA_C),
+	    ATA_INL(ch->r_mem, DMA_S), ch->rslots,
+	    ATA_INB(ch->r_mem, ATA_ALTSTAT));
+	/* Handle frozen command. */
+	mvs_requeue_frozen(dev);
+	/* We wait for other commands timeout and pray. */
+	if (ch->toslots == 0)
+		xpt_freeze_simq(ch->sim, 1);
+	ch->toslots |= (1 << slot->slot);
+	if ((ch->rslots & ~ch->toslots) == 0)
+		mvs_process_timeout(dev);
+	else
+		device_printf(dev, " ... waiting for slots %08x\n",
+		    ch->rslots & ~ch->toslots);
+}
+
+/* Must be called with channel locked. */
+static void
+mvs_end_transaction(struct mvs_slot *slot, enum mvs_err_type et)
+{
+	device_t dev = slot->dev;
+	struct mvs_channel *ch = device_get_softc(dev);
+	union ccb *ccb = slot->ccb;
+
+//device_printf(dev, "cmd done status %d\n", et);
+	bus_dmamap_sync(ch->dma.workrq_tag, ch->dma.workrq_map,
+	    BUS_DMASYNC_POSTWRITE);
+	/* Read result registers to the result struct
+	 * May be incorrect if several commands finished same time,
+	 * so read only when sure or have to.
+	 */
+	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
+		struct ata_res *res = &ccb->ataio.res;
+
+		if ((et == MVS_ERR_TFE) ||
+		    (ccb->ataio.cmd.flags & CAM_ATAIO_NEEDRESULT)) {
+			mvs_tfd_read(dev, ccb);
+		} else
+			bzero(res, sizeof(*res));
+	}
+	if (ch->numpslots == 0 || ch->basic_dma) {
+		if ((ccb->ccb_h.flags & CAM_DIR_MASK) != CAM_DIR_NONE) {
+			bus_dmamap_sync(ch->dma.data_tag, slot->dma.data_map,
+			    (ccb->ccb_h.flags & CAM_DIR_IN) ?
+			    BUS_DMASYNC_POSTREAD : BUS_DMASYNC_POSTWRITE);
+			bus_dmamap_unload(ch->dma.data_tag, slot->dma.data_map);
+		}
+	}
+	if (et != MVS_ERR_NONE)
+		ch->eslots |= (1 << slot->slot);
+	/* In case of error, freeze device for proper recovery. */
+	if ((et != MVS_ERR_NONE) && (!ch->readlog) &&
+	    !(ccb->ccb_h.status & CAM_DEV_QFRZN)) {
+		xpt_freeze_devq(ccb->ccb_h.path, 1);
+		ccb->ccb_h.status |= CAM_DEV_QFRZN;
+	}
+	/* Set proper result status. */
+	ccb->ccb_h.status &= ~CAM_STATUS_MASK;
+	switch (et) {
+	case MVS_ERR_NONE:
+		ccb->ccb_h.status |= CAM_REQ_CMP;
+		if (ccb->ccb_h.func_code == XPT_SCSI_IO)
+			ccb->csio.scsi_status = SCSI_STATUS_OK;
+		break;
+	case MVS_ERR_INVALID:
+		ch->fatalerr = 1;
+		ccb->ccb_h.status |= CAM_REQ_INVALID;
+		break;
+	case MVS_ERR_INNOCENT:
+		ccb->ccb_h.status |= CAM_REQUEUE_REQ;
+		break;
+	case MVS_ERR_TFE:
+	case MVS_ERR_NCQ:
+		if (ccb->ccb_h.func_code == XPT_SCSI_IO) {
+			ccb->ccb_h.status |= CAM_SCSI_STATUS_ERROR;
+			ccb->csio.scsi_status = SCSI_STATUS_CHECK_COND;
+		} else {
+			ccb->ccb_h.status |= CAM_ATA_STATUS_ERROR;
+		}
+		break;
+	case MVS_ERR_SATA:
+		ch->fatalerr = 1;
+		if (!ch->readlog) {
+			xpt_freeze_simq(ch->sim, 1);
+			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
+			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
+		}
+		ccb->ccb_h.status |= CAM_UNCOR_PARITY;
+		break;
+	case MVS_ERR_TIMEOUT:
+		if (!ch->readlog) {
+			xpt_freeze_simq(ch->sim, 1);
+			ccb->ccb_h.status &= ~CAM_STATUS_MASK;
+			ccb->ccb_h.status |= CAM_RELEASE_SIMQ;
+		}
+		ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
+		break;
+	default:
+		ch->fatalerr = 1;
+		ccb->ccb_h.status |= CAM_REQ_CMP_ERR;
+	}
+	/* Free slot. */
+	ch->oslots &= ~(1 << slot->slot);
+	ch->rslots &= ~(1 << slot->slot);
+	ch->aslots &= ~(1 << slot->slot);
+	if (et != MVS_ERR_TIMEOUT) {
+		if (ch->toslots == (1 << slot->slot))
+			xpt_release_simq(ch->sim, TRUE);
+		ch->toslots &= ~(1 << slot->slot);
+	}
+	slot->state = MVS_SLOT_EMPTY;
+	slot->ccb = NULL;
+	/* Update channel stats. */
+	ch->numrslots--;
+	ch->numrslotspd[ccb->ccb_h.target_id]--;
+	if (ccb->ccb_h.func_code == XPT_ATA_IO) {
+		if (ccb->ataio.cmd.flags & CAM_ATAIO_FPDMA) {
+			ch->otagspd[ccb->ccb_h.target_id] &= ~(1 << slot->tag);
+			ch->numtslots--;
+			ch->numtslotspd[ccb->ccb_h.target_id]--;
+		} else if (ccb->ataio.cmd.flags & CAM_ATAIO_DMA) {
+			ch->numdslots--;
+		} else {
+			ch->numpslots--;
+		}
+	} else {
+		ch->numpslots--;
+		ch->basic_dma = 0;
+	}
+	/* If it was our READ LOG command - process it. */
+	if (ch->readlog) {
+		mvs_process_read_log(dev, ccb);
+	/* If it was NCQ command error, put result on hold. */
+	} else if (et == MVS_ERR_NCQ) {
+		ch->hold[slot->slot] = ccb;
+		ch->holdtag[slot->slot] = slot->tag;
+		ch->numhslots++;
+	} else
+		xpt_done(ccb);
+	/* Unfreeze frozen command. */
+	if (ch->frozen && !mvs_check_collision(dev, ch->frozen)) {
+		union ccb *fccb = ch->frozen;
+		ch->frozen = NULL;
+		mvs_begin_transaction(dev, fccb);
+		xpt_release_simq(ch->sim, TRUE);
+	}
+	/* If we have no other active commands, ... */
+	if (ch->rslots == 0) {
+		/* if there was fatal error - reset port. */
+		if (ch->toslots != 0 || ch->fatalerr) {
+			mvs_reset(dev);
+		} else {
+			/* if we have slots in error, we can reinit port. */
+			if (ch->eslots != 0) {
+				mvs_set_edma_mode(dev, MVS_EDMA_OFF);
+				ch->eslots = 0;
+			}
+			/* if there commands on hold, we can do READ LOG. */
+			if (!ch->readlog && ch->numhslots)
+				mvs_issue_read_log(dev);
+		}
+	/* If all the rest of commands are in timeout - give them chance. */
+	} else if ((ch->rslots & ~ch->toslots) == 0 &&
+	    et != MVS_ERR_TIMEOUT)
+		mvs_rearm_timeout(dev);
+	/* Start PM timer. */
+	if (ch->numrslots == 0 && ch->pm_level > 3 &&
+	    (ch->curr[ch->pm_present ? 15 : 0].caps & CTS_SATA_CAPS_D_PMREQ)) {
+		callout_schedule(&ch->pm_timer,
+		    (ch->pm_level == 4) ? hz / 1000 : hz / 8);
+	}
+}
+
+static void
+mvs_issue_read_log(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	union ccb *ccb;
+	struct ccb_ataio *ataio;
+	int i;
+
+	ch->readlog = 1;
+	/* Find some holden command. */
+	for (i = 0; i < MVS_MAX_SLOTS; i++) {
+		if (ch->hold[i])
+			break;
+	}
+	ccb = xpt_alloc_ccb_nowait();
+	if (ccb == NULL) {
+		device_printf(dev, "Unable allocate READ LOG command");
+		return; /* XXX */
+	}
+	ccb->ccb_h = ch->hold[i]->ccb_h;	/* Reuse old header. */
+	ccb->ccb_h.func_code = XPT_ATA_IO;
+	ccb->ccb_h.flags = CAM_DIR_IN;
+	ccb->ccb_h.timeout = 1000;	/* 1s should be enough. */
+	ataio = &ccb->ataio;
+	ataio->data_ptr = malloc(512, M_MVS, M_NOWAIT);
+	if (ataio->data_ptr == NULL) {
+		device_printf(dev, "Unable allocate memory for READ LOG command");
+		return; /* XXX */
+	}
+	ataio->dxfer_len = 512;
+	bzero(&ataio->cmd, sizeof(ataio->cmd));
+	ataio->cmd.flags = CAM_ATAIO_48BIT;
+	ataio->cmd.command = 0x2F;	/* READ LOG EXT */
+	ataio->cmd.sector_count = 1;
+	ataio->cmd.sector_count_exp = 0;
+	ataio->cmd.lba_low = 0x10;
+	ataio->cmd.lba_mid = 0;
+	ataio->cmd.lba_mid_exp = 0;
+	/* Freeze SIM while doing READ LOG EXT. */
+	xpt_freeze_simq(ch->sim, 1);
+	mvs_begin_transaction(dev, ccb);
+}
+
+static void
+mvs_process_read_log(device_t dev, union ccb *ccb)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	uint8_t *data;
+	struct ata_res *res;
+	int i;
+
+	ch->readlog = 0;
+
+	data = ccb->ataio.data_ptr;
+	if ((ccb->ccb_h.status & CAM_STATUS_MASK) == CAM_REQ_CMP &&
+	    (data[0] & 0x80) == 0) {
+		for (i = 0; i < MVS_MAX_SLOTS; i++) {
+			if (!ch->hold[i])
+				continue;
+			if (ch->hold[i]->ccb_h.target_id != ccb->ccb_h.target_id)
+				continue;
+			if ((data[0] & 0x1F) == ch->holdtag[i]) {
+				res = &ch->hold[i]->ataio.res;
+				res->status = data[2];
+				res->error = data[3];
+				res->lba_low = data[4];
+				res->lba_mid = data[5];
+				res->lba_high = data[6];
+				res->device = data[7];
+				res->lba_low_exp = data[8];
+				res->lba_mid_exp = data[9];
+				res->lba_high_exp = data[10];
+				res->sector_count = data[12];
+				res->sector_count_exp = data[13];
+			} else {
+				ch->hold[i]->ccb_h.status &= ~CAM_STATUS_MASK;
+				ch->hold[i]->ccb_h.status |= CAM_REQUEUE_REQ;
+			}
+			xpt_done(ch->hold[i]);
+			ch->hold[i] = NULL;
+			ch->numhslots--;
+		}
+	} else {
+		if ((ccb->ccb_h.status & CAM_STATUS_MASK) != CAM_REQ_CMP)
+			device_printf(dev, "Error while READ LOG EXT\n");
+		else if ((data[0] & 0x80) == 0) {
+			device_printf(dev, "Non-queued command error in READ LOG EXT\n");
+		}
+		for (i = 0; i < MVS_MAX_SLOTS; i++) {
+			if (!ch->hold[i])
+				continue;
+			if (ch->hold[i]->ccb_h.target_id != ccb->ccb_h.target_id)
+				continue;
+			xpt_done(ch->hold[i]);
+			ch->hold[i] = NULL;
+			ch->numhslots--;
+		}
+	}
+	free(ccb->ataio.data_ptr, M_MVS);
+	xpt_free_ccb(ccb);
+	xpt_release_simq(ch->sim, TRUE);
+}
+
+static int
+mvs_wait(device_t dev, u_int s, u_int c, int t)
+{
+	int timeout = 0;
+	uint8_t st;
+
+	while (((st =  mvs_getstatus(dev, 0)) & (s | c)) != s) {
+		DELAY(1000);
+		if (timeout++ > t) {
+			device_printf(dev, "Wait status %02x\n", st);
+			return (-1);
+		}
+	} 
+	return (timeout);
+}
+
+static void
+mvs_requeue_frozen(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	union ccb *fccb = ch->frozen;
+
+	if (fccb) {
+		ch->frozen = NULL;
+		fccb->ccb_h.status = CAM_REQUEUE_REQ | CAM_RELEASE_SIMQ;
+		if (!(fccb->ccb_h.status & CAM_DEV_QFRZN)) {
+			xpt_freeze_devq(fccb->ccb_h.path, 1);
+			fccb->ccb_h.status |= CAM_DEV_QFRZN;
+		}
+		xpt_done(fccb);
+	}
+}
+
+static void
+mvs_reset(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	int i;
+
+	xpt_freeze_simq(ch->sim, 1);
+	if (bootverbose)
+		device_printf(dev, "MVS reset...\n");
+	/* Requeue freezed command. */
+	mvs_requeue_frozen(dev);
+	/* Kill the engine and requeue all running commands. */
+	mvs_set_edma_mode(dev, MVS_EDMA_OFF);
+	ATA_OUTL(ch->r_mem, DMA_C, 0);
+	for (i = 0; i < MVS_MAX_SLOTS; i++) {
+		/* Do we have a running request on slot? */
+		if (ch->slot[i].state < MVS_SLOT_RUNNING)
+			continue;
+		/* XXX; Commands in loading state. */
+		mvs_end_transaction(&ch->slot[i], MVS_ERR_INNOCENT);
+	}
+	for (i = 0; i < MVS_MAX_SLOTS; i++) {
+		if (!ch->hold[i])
+			continue;
+		xpt_done(ch->hold[i]);
+		ch->hold[i] = NULL;
+		ch->numhslots--;
+	}
+	if (ch->toslots != 0)
+		xpt_release_simq(ch->sim, TRUE);
+	ch->eslots = 0;
+	ch->toslots = 0;
+	ch->fatalerr = 0;
+	/* Tell the XPT about the event */
+	xpt_async(AC_BUS_RESET, ch->path, NULL);
+	ATA_OUTL(ch->r_mem, EDMA_IEM, 0);
+	ATA_OUTL(ch->r_mem, EDMA_CMD, EDMA_CMD_EATARST);
+	DELAY(25);
+	ATA_OUTL(ch->r_mem, EDMA_CMD, 0);
+	/* Reset and reconnect PHY, */
+	if (!mvs_sata_phy_reset(dev)) {
+		if (bootverbose)
+			device_printf(dev,
+			    "MVS reset done: phy reset found no device\n");
+		ch->devices = 0;
+		ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
+		ATA_OUTL(ch->r_mem, EDMA_IEC, 0);
+		ATA_OUTL(ch->r_mem, EDMA_IEM, ~EDMA_IE_TRANSIENT);
+		xpt_release_simq(ch->sim, TRUE);
+		return;
+	}
+	/* Wait for clearing busy status. */
+	if ((i = mvs_wait(dev, 0, ATA_S_BUSY | ATA_S_DRQ, 15000)) < 0)
+		device_printf(dev, "device is not ready\n");
+	else if (bootverbose)                                                        
+		device_printf(dev, "ready wait time=%dms\n", i);
+	ch->devices = 1;
+	ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
+	ATA_OUTL(ch->r_mem, EDMA_IEC, 0);
+	ATA_OUTL(ch->r_mem, EDMA_IEM, ~EDMA_IE_TRANSIENT);
+	if (bootverbose)
+		device_printf(dev, "MVS reset done: device found\n");
+	xpt_release_simq(ch->sim, TRUE);
+}
+
+static void
+mvs_softreset(device_t dev, union ccb *ccb)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	int port = ccb->ccb_h.target_id & 0x0f;
+	int i;
+
+	mvs_set_edma_mode(dev, MVS_EDMA_OFF);
+	ATA_OUTB(ch->r_mem, SATA_SATAICTL, port << SATA_SATAICTL_PMPTX_SHIFT);
+	ATA_OUTB(ch->r_mem, ATA_CONTROL, ATA_A_RESET);
+	DELAY(10000);
+	ATA_OUTB(ch->r_mem, ATA_CONTROL, 0);
+	ccb->ccb_h.status &= ~CAM_STATUS_MASK;
+	/* Wait for clearing busy status. */
+	if ((i = mvs_wait(dev, 0, ATA_S_BUSY | ATA_S_DRQ, ccb->ccb_h.timeout)) < 0) {
+		ccb->ccb_h.status |= CAM_CMD_TIMEOUT;
+	} else {
+		ccb->ccb_h.status |= CAM_REQ_CMP;
+	}
+	mvs_tfd_read(dev, ccb);
+	xpt_done(ccb);
+}
+
+static int
+mvs_sata_connect(struct mvs_channel *ch)
+{
+	u_int32_t status;
+	int timeout;
+
+	/* Wait up to 100ms for "connect well" */
+	for (timeout = 0; timeout < 100 ; timeout++) {
+		status = ATA_INL(ch->r_mem, SATA_SS);
+		if (((status & SATA_SS_DET_MASK) == SATA_SS_DET_PHY_ONLINE) &&
+		    ((status & SATA_SS_SPD_MASK) != SATA_SS_SPD_NO_SPEED) &&
+		    ((status & SATA_SS_IPM_MASK) == SATA_SS_IPM_ACTIVE))
+			break;
+		if ((status & SATA_SS_DET_MASK) == SATA_SS_DET_PHY_OFFLINE) {
+			if (bootverbose) {
+				device_printf(ch->dev, "SATA offline status=%08x\n",
+				    status);
+			}
+			return (0);
+		}
+		DELAY(1000);
+	}
+	if (timeout >= 100) {
+		if (bootverbose) {
+			device_printf(ch->dev, "SATA connect timeout status=%08x\n",
+			    status);
+		}
+		return (0);
+	}
+	if (bootverbose) {
+		device_printf(ch->dev, "SATA connect time=%dms status=%08x\n",
+		    timeout, status);
+	}
+	/* Clear SATA error register */
+	ATA_OUTL(ch->r_mem, SATA_SE, 0xffffffff);
+	return (1);
+}
+
+static int
+mvs_sata_phy_reset(device_t dev)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+	int sata_rev;
+	uint32_t val;
+
+	sata_rev = ch->user[ch->pm_present ? 15 : 0].revision;
+	if (sata_rev == 1)
+		val = SATA_SC_SPD_SPEED_GEN1;
+	else if (sata_rev == 2)
+		val = SATA_SC_SPD_SPEED_GEN2;
+	else if (sata_rev == 3)
+		val = SATA_SC_SPD_SPEED_GEN3;
+	else
+		val = 0;
+	ATA_OUTL(ch->r_mem, SATA_SC,
+	    SATA_SC_DET_RESET | val |
+	    SATA_SC_IPM_DIS_PARTIAL | SATA_SC_IPM_DIS_SLUMBER);
+	DELAY(5000);
+	ATA_OUTL(ch->r_mem, SATA_SC,
+	    SATA_SC_DET_IDLE | val | ((ch->pm_level > 0) ? 0 :
+	    (SATA_SC_IPM_DIS_PARTIAL | SATA_SC_IPM_DIS_SLUMBER)));
+	DELAY(5000);
+	if (!mvs_sata_connect(ch)) {
+		if (ch->pm_level > 0)
+			ATA_OUTL(ch->r_mem, SATA_SC, SATA_SC_DET_DISABLE);
+		return (0);
+	}
+	return (1);
+}
+
+static int
+mvs_check_ids(device_t dev, union ccb *ccb)
+{
+	struct mvs_channel *ch = device_get_softc(dev);
+
+	if (ccb->ccb_h.target_id > ((ch->quirks & MVS_Q_GENI) ? 0 : 15)) {
+		ccb->ccb_h.status = CAM_TID_INVALID;
+		xpt_done(ccb);
+		return (-1);
+	}
+	if (ccb->ccb_h.target_lun != 0) {
+		ccb->ccb_h.status = CAM_LUN_INVALID;
+		xpt_done(ccb);
+		return (-1);
+	}
+	return (0);
+}
+
+static void
+mvsaction(struct cam_sim *sim, union ccb *ccb)
+{
+	device_t dev;
+	struct mvs_channel *ch;
+
+	CAM_DEBUG(ccb->ccb_h.path, CAM_DEBUG_TRACE, ("mvsaction func_code=%x\n",
+	    ccb->ccb_h.func_code));
+
+	ch = (struct mvs_channel *)cam_sim_softc(sim);
+	dev = ch->dev;
+	switch (ccb->ccb_h.func_code) {
+	/* Common cases first */
+	case XPT_ATA_IO:	/* Execute the requested I/O operation */
+	case XPT_SCSI_IO:
+		if (mvs_check_ids(dev, ccb))
+			return;
+		if (ch->devices == 0 ||
+		    (ch->pm_present == 0 &&
+		     ccb->ccb_h.target_id > 0 && ccb->ccb_h.target_id < 15)) {
+			ccb->ccb_h.status = CAM_SEL_TIMEOUT;
+			break;
+		}
+		/* Check for command collision. */
+		if (mvs_check_collision(dev, ccb)) {
+			/* Freeze command. */
+			ch->frozen = ccb;
+			/* We have only one frozen slot, so freeze simq also. */
+			xpt_freeze_simq(ch->sim, 1);
+			return;
+		}
+		mvs_begin_transaction(dev, ccb);
+		return;
+	case XPT_EN_LUN:		/* Enable LUN as a target */
+	case XPT_TARGET_IO:		/* Execute target I/O request */
+	case XPT_ACCEPT_TARGET_IO:	/* Accept Host Target Mode CDB */
+	case XPT_CONT_TARGET_IO:	/* Continue Host Target I/O Connection*/
+	case XPT_ABORT:			/* Abort the specified CCB */
+		/* XXX Implement */
+		ccb->ccb_h.status = CAM_REQ_INVALID;
+		break;
+	case XPT_SET_TRAN_SETTINGS:
+	{
+		struct	ccb_trans_settings *cts = &ccb->cts;
+		struct	mvs_device *d; 
+
+		if (mvs_check_ids(dev, ccb))
+			return;
+		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
+			d = &ch->curr[ccb->ccb_h.target_id];
+		else
+			d = &ch->user[ccb->ccb_h.target_id];
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_REVISION)
+			d->revision = cts->xport_specific.sata.revision;
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_MODE)
+			d->mode = cts->xport_specific.sata.mode;
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_BYTECOUNT) {
+			d->bytecount = min((ch->quirks & MVS_Q_GENIIE) ? 8192 : 2048,
+			    cts->xport_specific.sata.bytecount);
+		}
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_TAGS)
+			d->tags = min(MVS_MAX_SLOTS, cts->xport_specific.sata.tags);
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_PM)
+			ch->pm_present = cts->xport_specific.sata.pm_present;
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_ATAPI)
+			d->atapi = cts->xport_specific.sata.atapi;
+		if (cts->xport_specific.sata.valid & CTS_SATA_VALID_CAPS)
+			d->caps = cts->xport_specific.sata.caps;
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_GET_TRAN_SETTINGS:
+	/* Get default/user set transfer settings for the target */
+	{
+		struct	ccb_trans_settings *cts = &ccb->cts;
+		struct  mvs_device *d;
+		uint32_t status;
+
+		if (mvs_check_ids(dev, ccb))
+			return;
+		if (cts->type == CTS_TYPE_CURRENT_SETTINGS)
+			d = &ch->curr[ccb->ccb_h.target_id];
+		else
+			d = &ch->user[ccb->ccb_h.target_id];
+		cts->protocol = PROTO_ATA;
+		cts->protocol_version = PROTO_VERSION_UNSPECIFIED;
+		cts->transport = XPORT_SATA;
+		cts->transport_version = XPORT_VERSION_UNSPECIFIED;
+		cts->proto_specific.valid = 0;
+		cts->xport_specific.sata.valid = 0;
+		if (cts->type == CTS_TYPE_CURRENT_SETTINGS &&
+		    (ccb->ccb_h.target_id == 15 ||
+		    (ccb->ccb_h.target_id == 0 && !ch->pm_present))) {
+			status = ATA_INL(ch->r_mem, SATA_SS) & SATA_SS_SPD_MASK;
+			if (status & 0x0f0) {
+				cts->xport_specific.sata.revision =
+				    (status & 0x0f0) >> 4;
+				cts->xport_specific.sata.valid |=
+				    CTS_SATA_VALID_REVISION;
+			}
+			cts->xport_specific.sata.caps = d->caps & CTS_SATA_CAPS_D;
+//			if (ch->pm_level)
+//				cts->xport_specific.sata.caps |= CTS_SATA_CAPS_H_PMREQ;
+			cts->xport_specific.sata.caps &=
+			    ch->user[ccb->ccb_h.target_id].caps;
+			cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
+		} else {
+			cts->xport_specific.sata.revision = d->revision;
+			cts->xport_specific.sata.valid |= CTS_SATA_VALID_REVISION;
+			cts->xport_specific.sata.caps = d->caps;
+			cts->xport_specific.sata.valid |= CTS_SATA_VALID_CAPS;
+		}
+		cts->xport_specific.sata.mode = d->mode;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_MODE;
+		cts->xport_specific.sata.bytecount = d->bytecount;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_BYTECOUNT;
+		cts->xport_specific.sata.pm_present = ch->pm_present;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_PM;
+		cts->xport_specific.sata.tags = d->tags;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_TAGS;
+		cts->xport_specific.sata.atapi = d->atapi;
+		cts->xport_specific.sata.valid |= CTS_SATA_VALID_ATAPI;
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	case XPT_RESET_BUS:		/* Reset the specified SCSI bus */
+	case XPT_RESET_DEV:	/* Bus Device Reset the specified SCSI device */
+		mvs_reset(dev);
+		ccb->ccb_h.status = CAM_REQ_CMP;
+		break;
+	case XPT_TERM_IO:		/* Terminate the I/O process */
+		/* XXX Implement */
+		ccb->ccb_h.status = CAM_REQ_INVALID;
+		break;
+	case XPT_PATH_INQ:		/* Path routing inquiry */
+	{
+		struct ccb_pathinq *cpi = &ccb->cpi;
+
+		cpi->version_num = 1; /* XXX??? */
+		cpi->hba_inquiry = PI_SDTR_ABLE;
+		if (!(ch->quirks & MVS_Q_GENI)) {
+			cpi->hba_inquiry |= PI_SATAPM;
+			/* Gen-II is extremely slow with NCQ on PMP. */
+			if ((ch->quirks & MVS_Q_GENIIE) || ch->pm_present == 0)
+				cpi->hba_inquiry |= PI_TAG_ABLE;
+		}
+		cpi->target_sprt = 0;
+		cpi->hba_misc = PIM_SEQSCAN;
+		cpi->hba_eng_cnt = 0;
+		if (!(ch->quirks & MVS_Q_GENI))
+			cpi->max_target = 15;
+		else
+			cpi->max_target = 0;
+		cpi->max_lun = 0;
+		cpi->initiator_id = 0;
+		cpi->bus_id = cam_sim_bus(sim);
+		cpi->base_transfer_speed = 150000;
+		strncpy(cpi->sim_vid, "FreeBSD", SIM_IDLEN);
+		strncpy(cpi->hba_vid, "Marvell", HBA_IDLEN);
+		strncpy(cpi->dev_name, cam_sim_name(sim), DEV_IDLEN);
+		cpi->unit_number = cam_sim_unit(sim);
+		cpi->transport = XPORT_SATA;
+		cpi->transport_version = XPORT_VERSION_UNSPECIFIED;
+		cpi->protocol = PROTO_ATA;
+		cpi->protocol_version = PROTO_VERSION_UNSPECIFIED;
+		cpi->maxio = MAXPHYS;
+		cpi->ccb_h.status = CAM_REQ_CMP;
+		break;
+	}
+	default:
+		ccb->ccb_h.status = CAM_REQ_INVALID;
+		break;
+	}
+	xpt_done(ccb);
+}
+
+static void
+mvspoll(struct cam_sim *sim)
+{
+	struct mvs_channel *ch = (struct mvs_channel *)cam_sim_softc(sim);
+	struct mvs_intr_arg arg;
+
+	arg.arg = ch->dev;
+	arg.cause = 2; /* XXX */
+	mvs_ch_intr(arg.arg);
+}
+
diff --git a/sys/dev/mvs/mvs.h b/sys/dev/mvs/mvs.h
new file mode 100644
index 0000000..9ec4e3d
--- /dev/null
+++ b/sys/dev/mvs/mvs.h
@@ -0,0 +1,650 @@
+/*-
+ * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include "mvs_if.h"
+
+/* Chip registers */
+#define CHIP_PCIEIC		0x1900	/* PCIe Interrupt Cause */
+#define CHIP_PCIEIM		0x1910	/* PCIe Interrupt Mask */
+#define CHIP_PCIIC		0x1d58	/* PCI Interrupt Cause */
+#define CHIP_PCIIM		0x1d5c	/* PCI Interrupt Mask */
+#define CHIP_MIC		0x1d60	/* Main Interrupt Cause */
+#define CHIP_MIM		0x1d64	/* Main Interrupt Mask */
+#define CHIP_SOC_MIC		0x20	/* SoC Main Interrupt Cause */
+#define CHIP_SOC_MIM		0x24	/* SoC Main Interrupt Mask */
+#define IC_ERR_IRQ		 (1 << 0)	/* shift by (2 * port #) */
+#define IC_DONE_IRQ		 (1 << 1)	/* shift by (2 * port #) */
+#define IC_HC0			 0x000001ff	/* bits 0-8 = HC0 */
+#define IC_HC_SHIFT		 9		/* HC1 shift */
+#define IC_HC1			 (IC_HC0 << IC_HC_SHIFT) /* 9-17 = HC1 */
+#define IC_ERR_HC0		 0x00000055	/* HC0 ERR_IRQ */
+#define IC_DONE_HC0		 0x000000aa	/* HC0 DONE_IRQ */
+#define IC_ERR_HC1		 (IC_ERR_HC0 << IC_HC_SHIFT) /* HC1 ERR_IRQ */
+#define IC_DONE_HC1		 (IC_DONE_HC0 << IC_HC_SHIFT) /* HC1 DONE_IRQ */
+#define IC_HC0_COAL_DONE	 (1 << 8)	/* HC0 IRQ coalescing */
+#define IC_HC1_COAL_DONE	 (1 << 17)	/* HC1 IRQ coalescing */
+#define IC_PCI_ERR		 (1 << 18)
+#define IC_TRAN_COAL_LO_DONE	 (1 << 19)	/* transaction coalescing */
+#define IC_TRAN_COAL_HI_DONE	 (1 << 20)	/* transaction coalescing */
+#define IC_ALL_PORTS_COAL_DONE	 (1 << 21)	/* GEN_II(E) IRQ coalescing */
+#define IC_GPIO_INT		 (1 << 22)
+#define IC_SELF_INT		 (1 << 23)
+#define IC_TWSI_INT		 (1 << 24)
+#define IC_MAIN_RSVD		 (0xfe000000)	/* bits 31-25 */
+#define IC_MAIN_RSVD_5		 (0xfff10000)	/* bits 31-19 */
+#define IC_MAIN_RSVD_SOC	 (0xfffffec0)	/* bits 31-9, 7-6 */
+
+#define CHIP_SOC_LED		0x2C	/* SoC LED Configuration */
+
+/* Chip CCC registers */
+#define CHIP_ICC		0x18008
+#define CHIP_ICC_ALL_PORTS	 (1 << 4)	/* all ports irq event */
+#define CHIP_ICT 		0x180cc
+#define CHIP_ITT		0x180d0
+#define CHIP_TRAN_COAL_CAUSE_LO	0x18088
+#define CHIP_TRAN_COAL_CAUSE_HI	0x1808c
+
+/* Host Controller registers */
+#define HC_SIZE			0x10000
+#define HC_OFFSET		0x20000
+#define HC_BASE(hc)		((hc) * HC_SIZE + HC_OFFSET)
+
+#define HC_CFG			0x0	/* Configuration */
+#define HC_CFG_TIMEOUT_MASK	 (0xff << 0)
+#define HC_CFG_NODMABS		 (1 << 8)
+#define HC_CFG_NOEDMABS		 (1 << 9)
+#define HC_CFG_NOPRDBS		 (1 << 10)
+#define HC_CFG_TIMEOUTEN	 (1 << 16)	/* Timer Enable */
+#define HC_CFG_COALDIS(p)	 (1 << ((p) + 24))/* Coalescing Disable*/
+#define HC_RQOP			0x4	/* Request Queue Out-Pointer */
+#define HC_RQIP			0x8	/* Response Queue In-Pointer */
+#define HC_ICT			0xc	/* Interrupt Coalescing Threshold */
+#define HC_ICT_SAICOALT_MASK	 0x000000ff
+#define HC_ITT			0x10	/* Interrupt Time Threshold */
+#define HC_ITT_SAITMTH_MASK	 0x00ffffff
+#define HC_IC			0x14	/* Interrupt Cause */
+#define HC_IC_DONE(p)		 (1 << (p))	/* SaCrpb/DMA Done */
+#define HC_IC_COAL		 (1 << 4)	/* Intr Coalescing */
+#define HC_IC_DEV(p)		 (1 << ((p) + 8)) /* Device Intr */
+
+/* Port registers */
+#define PORT_SIZE		0x2000
+#define PORT_OFFSET		0x2000
+#define PORT_BASE(hc)		((hc) * PORT_SIZE + PORT_OFFSET)
+
+#define EDMA_CFG		0x0	/* Configuration */
+#define EDMA_CFG_RESERVED	 (0x1f << 0)	/* Queue len ? */
+#define EDMA_CFG_ESATANATVCMDQUE (1 << 5)
+#define EDMA_CFG_ERDBSZ		 (1 << 8)
+#define EDMA_CFG_EQUE		 (1 << 9)
+#define EDMA_CFG_ERDBSZEXT	 (1 << 11)
+#define EDMA_CFG_RESERVED2	 (1 << 12)
+#define EDMA_CFG_EWRBUFFERLEN	 (1 << 13)
+#define EDMA_CFG_EDEVERR	 (1 << 14)
+#define EDMA_CFG_EEDMAFBS	 (1 << 16)
+#define EDMA_CFG_ECUTTHROUGHEN	 (1 << 17)
+#define EDMA_CFG_EEARLYCOMPLETIONEN (1 << 18)
+#define EDMA_CFG_EEDMAQUELEN	 (1 << 19)
+#define EDMA_CFG_EHOSTQUEUECACHEEN (1 << 22)
+#define EDMA_CFG_EMASKRXPM	 (1 << 23)
+#define EDMA_CFG_RESUMEDIS	 (1 << 24)
+#define EDMA_CFG_EDMAFBS	 (1 << 26)
+#define EDMA_T			0x4	/* Timer */
+#define EDMA_IEC		0x8	/* Interrupt Error Cause */
+#define EDMA_IEM		0xc	/* Interrupt Error Mask */
+#define EDMA_IE_EDEVERR		 (1 << 2)	/* EDMA Device Error */
+#define EDMA_IE_EDEVDIS		 (1 << 3)	/* EDMA Dev Disconn */
+#define EDMA_IE_EDEVCON		 (1 << 4)	/* EDMA Dev Conn */
+#define EDMA_IE_SERRINT		 (1 << 5)
+#define EDMA_IE_ESELFDIS	 (1 << 7)	/* EDMA Self Disable */
+#define EDMA_IE_ETRANSINT	 (1 << 8)	/* Transport Layer */
+#define EDMA_IE_EIORDYERR	 (1 << 12)	/* EDMA IORdy Error */
+#define EDMA_IE_LINKXERR_SATACRC (1 << 0)	/* SATA CRC error */
+#define EDMA_IE_LINKXERR_INTERNALFIFO	(1 << 1)	/* internal FIFO err */
+#define EDMA_IE_LINKXERR_LINKLAYERRESET	(1 << 2)
+	/* Link Layer is reset by the reception of SYNC primitive from device */
+#define EDMA_IE_LINKXERR_OTHERERRORS	(1 << 3)
+	/*
+	 * Link state errors, coding errors, or running disparity errors occur
+	 * during FIS reception.
+	 */
+#define EDMA_IE_LINKTXERR_FISTXABORTED   (1 << 4)	/* FIS Tx is aborted */
+#define EDMA_IE_LINKCTLRXERR(x)		((x) << 13)	/* Link Ctrl Recv Err */
+#define EDMA_IE_LINKDATARXERR(x)	((x) << 17)	/* Link Data Recv Err */
+#define EDMA_IE_LINKCTLTXERR(x)		((x) << 21)	/* Link Ctrl Tx Error */
+#define EDMA_IE_LINKDATATXERR(x)	((x) << 26)	/* Link Data Tx Error */
+#define EDMA_IE_TRANSPROTERR		(1 << 31)	/* Transport Proto E */
+#define EDMA_IE_TRANSIENT		(EDMA_IE_LINKCTLRXERR(0x0b) | \
+					 EDMA_IE_LINKCTLTXERR(0x1f))
+							/* Non-fatal Errors */
+#define EDMA_REQQBAH		0x10	/* Request Queue Base Address High */
+#define EDMA_REQQIP		0x14	/* Request Queue In-Pointer */
+#define EDMA_REQQOP		0x18	/* Request Queue Out-Pointer */
+#define EDMA_REQQP_ERQQP_SHIFT	 5
+#define EDMA_REQQP_ERQQP_MASK	 0x000003e0
+#define EDMA_REQQP_ERQQBAP_MASK	 0x00000c00
+#define EDMA_REQQP_ERQQBA_MASK	 0xfffff000
+#define EDMA_RESQBAH		0x1c	/* Response Queue Base Address High */
+#define EDMA_RESQIP		0x20	/* Response Queue In-Pointer */
+#define EDMA_RESQOP		0x24	/* Response Queue Out-Pointer */
+#define EDMA_RESQP_ERPQP_SHIFT	 3
+#define EDMA_RESQP_ERPQP_MASK	 0x000000f8
+#define EDMA_RESQP_ERPQBAP_MASK	 0x00000300
+#define EDMA_RESQP_ERPQBA_MASK	 0xfffffc00
+#define EDMA_CMD		0x28	/* Command */
+#define EDMA_CMD_EENEDMA	 (1 << 0)	/* Enable EDMA */
+#define EDMA_CMD_EDSEDMA	 (1 << 1)	/* Disable EDMA */
+#define EDMA_CMD_EATARST	 (1 << 2)	/* ATA Device Reset */
+#define EDMA_CMD_EEDMAFRZ	 (1 << 4)	/* EDMA Freeze */
+#define EDMA_TC			0x2c	/* Test Control */
+#define EDMA_S			0x30	/* Status */
+#define EDMA_S_EDEVQUETAG(s)	 ((s) & 0x0000001f)
+#define EDMA_S_EDEVDIR_WRITE	 (0 << 5)
+#define EDMA_S_EDEVDIR_READ	 (1 << 5)
+#define EDMA_S_ECACHEEMPTY	 (1 << 6)
+#define EDMA_S_EDMAIDLE		 (1 << 7)
+#define EDMA_S_ESTATE(s)	 (((s) & 0x0000ff00) >> 8)
+#define EDMA_S_EIOID(s)		 (((s) & 0x003f0000) >> 16)
+#define EDMA_IORT		0x34	/* IORdy Timeout */
+#define EDMA_CDT		0x40	/* Command Delay Threshold */
+#define EDMA_HC			0x60	/* Halt Condition */
+#define EDMA_UNKN_RESD		0x6C	/* Unknown register */
+#define EDMA_CQDCQOS(x)		(0x90 + ((x) << 2)
+					/* NCQ Done/TCQ Outstanding Status */
+
+/* ATA register defines */
+#define ATA_DATA                        0x100	/* (RW) data */
+#define ATA_FEATURE                     0x104	/* (W) feature */
+#define         ATA_F_DMA                0x01	/* enable DMA */
+#define         ATA_F_OVL                0x02	/* enable overlap */
+#define ATA_ERROR                       0x104	/* (R) error */
+#define         ATA_E_ILI                0x01	/* illegal length */
+#define         ATA_E_NM                 0x02	/* no media */
+#define         ATA_E_ABORT              0x04	/* command aborted */
+#define         ATA_E_MCR                0x08	/* media change request */
+#define         ATA_E_IDNF               0x10	/* ID not found */
+#define         ATA_E_MC                 0x20	/* media changed */
+#define         ATA_E_UNC                0x40	/* uncorrectable data */
+#define         ATA_E_ICRC               0x80	/* UDMA crc error */
+#define		ATA_E_ATAPI_SENSE_MASK	 0xf0	/* ATAPI sense key mask */
+#define ATA_COUNT                       0x108	/* (W) sector count */
+#define ATA_IREASON                     0x108   /* (R) interrupt reason */
+#define         ATA_I_CMD                0x01	/* cmd (1) | data (0) */
+#define         ATA_I_IN                 0x02	/* read (1) | write (0) */
+#define         ATA_I_RELEASE            0x04	/* released bus (1) */
+#define         ATA_I_TAGMASK            0xf8	/* tag mask */
+#define ATA_SECTOR                      0x10c	/* (RW) sector # */
+#define ATA_CYL_LSB                     0x110	/* (RW) cylinder# LSB */
+#define ATA_CYL_MSB                     0x114	/* (RW) cylinder# MSB */
+#define ATA_DRIVE                       0x118	/* (W) Sector/Drive/Head */
+#define         ATA_D_LBA                0x40	/* use LBA addressing */
+#define         ATA_D_IBM                0xa0	/* 512 byte sectors, ECC */
+#define ATA_COMMAND                     0x11c	/* (W) command */
+#define ATA_STATUS                      0x11c	/* (R) status */
+#define         ATA_S_ERROR              0x01	/* error */
+#define         ATA_S_INDEX              0x02	/* index */
+#define         ATA_S_CORR               0x04	/* data corrected */
+#define         ATA_S_DRQ                0x08	/* data request */
+#define         ATA_S_DSC                0x10	/* drive seek completed */
+#define         ATA_S_SERVICE            0x10	/* drive needs service */
+#define         ATA_S_DWF                0x20	/* drive write fault */
+#define         ATA_S_DMA                0x20	/* DMA ready */
+#define         ATA_S_READY              0x40	/* drive ready */
+#define         ATA_S_BUSY               0x80	/* busy */
+#define ATA_CONTROL                     0x120	/* (W) control */
+#define         ATA_A_IDS                0x02	/* disable interrupts */
+#define         ATA_A_RESET              0x04	/* RESET controller */
+#define         ATA_A_4BIT               0x08	/* 4 head bits */
+#define         ATA_A_HOB                0x80	/* High Order Byte enable */
+#define ATA_ALTSTAT                     0x120	/* (R) alternate status */
+#define ATAPI_P_READ                    (ATA_S_DRQ | ATA_I_IN)
+#define ATAPI_P_WRITE                   (ATA_S_DRQ)
+#define ATAPI_P_CMDOUT                  (ATA_S_DRQ | ATA_I_CMD)
+#define ATAPI_P_DONEDRQ                 (ATA_S_DRQ | ATA_I_CMD | ATA_I_IN)
+#define ATAPI_P_DONE                    (ATA_I_CMD | ATA_I_IN)
+#define ATAPI_P_ABORT                   0
+
+/* Basic DMA Registers */
+#define DMA_C				0x224	/* Basic DMA Command */
+#define DMA_C_START			 (1 << 0)
+#define DMA_C_READ			 (1 << 3)
+#define DMA_C_DREGIONVALID		 (1 << 8)
+#define DMA_C_DREGIONLAST		 (1 << 9)
+#define DMA_C_CONTFROMPREV		 (1 << 10)
+#define DMA_C_DRBC(n)			 (((n) & 0xffff) << 16)
+#define DMA_S				0x228	/* Basic DMA Status */
+#define DMA_S_ACT			 (1 << 0) /* Active */
+#define DMA_S_ERR			 (1 << 1) /* Error */
+#define DMA_S_PAUSED			 (1 << 2) /* Paused */
+#define DMA_S_LAST			 (1 << 3) /* Last */
+#define DMA_DTLBA			0x22c	/* Descriptor Table Low Base Address */
+#define DMA_DTLBA_MASK			 0xfffffff0
+#define DMA_DTHBA			0x230	/* Descriptor Table High Base Address */
+#define DMA_DRLA			0x234	/* Data Region Low Address */
+#define DMA_DRHA			0x238	/* Data Region High Address */
+
+/* Serial-ATA Registers */
+#define SATA_SS				0x300	/* SStatus */
+#define        SATA_SS_DET_MASK         0x0000000f
+#define        SATA_SS_DET_NO_DEVICE    0x00000000
+#define        SATA_SS_DET_DEV_PRESENT  0x00000001
+#define        SATA_SS_DET_PHY_ONLINE   0x00000003
+#define        SATA_SS_DET_PHY_OFFLINE  0x00000004
+
+#define        SATA_SS_SPD_MASK         0x000000f0
+#define        SATA_SS_SPD_NO_SPEED     0x00000000
+#define        SATA_SS_SPD_GEN1         0x00000010
+#define        SATA_SS_SPD_GEN2         0x00000020
+#define        SATA_SS_SPD_GEN3         0x00000040
+
+#define        SATA_SS_IPM_MASK         0x00000f00
+#define        SATA_SS_IPM_NO_DEVICE    0x00000000
+#define        SATA_SS_IPM_ACTIVE       0x00000100
+#define        SATA_SS_IPM_PARTIAL      0x00000200
+#define        SATA_SS_IPM_SLUMBER      0x00000600
+#define SATA_SE				0x304	/* SError */
+#define SATA_SEIM			0x340	/* SError Interrupt Mask */
+#define        SATA_SE_DATA_CORRECTED   0x00000001
+#define        SATA_SE_COMM_CORRECTED   0x00000002
+#define        SATA_SE_DATA_ERR         0x00000100
+#define        SATA_SE_COMM_ERR         0x00000200
+#define        SATA_SE_PROT_ERR         0x00000400
+#define        SATA_SE_HOST_ERR         0x00000800
+#define        SATA_SE_PHY_CHANGED      0x00010000
+#define        SATA_SE_PHY_IERROR       0x00020000
+#define        SATA_SE_COMM_WAKE        0x00040000
+#define        SATA_SE_DECODE_ERR       0x00080000
+#define        SATA_SE_PARITY_ERR       0x00100000
+#define        SATA_SE_CRC_ERR          0x00200000
+#define        SATA_SE_HANDSHAKE_ERR    0x00400000
+#define        SATA_SE_LINKSEQ_ERR      0x00800000
+#define        SATA_SE_TRANSPORT_ERR    0x01000000
+#define        SATA_SE_UNKNOWN_FIS      0x02000000
+#define SATA_SC				0x308	/* SControl */
+#define        SATA_SC_DET_MASK         0x0000000f
+#define        SATA_SC_DET_IDLE         0x00000000
+#define        SATA_SC_DET_RESET        0x00000001
+#define        SATA_SC_DET_DISABLE      0x00000004
+
+#define        SATA_SC_SPD_MASK         0x000000f0
+#define        SATA_SC_SPD_NO_SPEED     0x00000000
+#define        SATA_SC_SPD_SPEED_GEN1   0x00000010
+#define        SATA_SC_SPD_SPEED_GEN2   0x00000020
+#define        SATA_SC_SPD_SPEED_GEN3   0x00000040
+
+#define        SATA_SC_IPM_MASK         0x00000f00
+#define        SATA_SC_IPM_NONE         0x00000000
+#define        SATA_SC_IPM_DIS_PARTIAL  0x00000100
+#define        SATA_SC_IPM_DIS_SLUMBER  0x00000200
+
+#define        SATA_SC_SPM_MASK		0x0000f000
+#define        SATA_SC_SPM_NONE		0x00000000
+#define        SATA_SC_SPM_PARTIAL	0x00001000
+#define        SATA_SC_SPM_SLUMBER	0x00002000
+#define        SATA_SC_SPM_ACTIVE	0x00004000
+#define SATA_LTM			0x30c	/* LTMode */
+#define SATA_PHYM3			0x310	/* PHY Mode 3 */
+#define SATA_PHYM4			0x314	/* PHY Mode 4 */
+#define SATA_PHYM1			0x32c	/* PHY Mode 1 */
+#define SATA_PHYM2			0x330	/* PHY Mode 2 */
+#define SATA_BISTC			0x334	/* BIST Control */
+#define SATA_BISTDW1			0x338	/* BIST DW1 */
+#define SATA_BISTDW2			0x33c	/* BIST DW2 */
+#define SATA_SATAICFG			0x050	/* Serial-ATA Interface Configuration */
+#define SATA_SATAICFG_REFCLKCNF_20MHZ	 (0 << 0)
+#define SATA_SATAICFG_REFCLKCNF_25MHZ	 (1 << 0)
+#define SATA_SATAICFG_REFCLKCNF_30MHZ	 (2 << 0)
+#define SATA_SATAICFG_REFCLKCNF_40MHZ	 (3 << 0)
+#define SATA_SATAICFG_REFCLKCNF_MASK	 (3 << 0)
+#define SATA_SATAICFG_REFCLKDIV_1	 (0 << 2)
+#define SATA_SATAICFG_REFCLKDIV_2	 (1 << 2)	/* Used 20 or 25MHz */
+#define SATA_SATAICFG_REFCLKDIV_4	 (2 << 2)	/* Used 40MHz */
+#define SATA_SATAICFG_REFCLKDIV_3	 (3 << 2)	/* Used 30MHz */
+#define SATA_SATAICFG_REFCLKDIV_MASK	 (3 << 2)
+#define SATA_SATAICFG_REFCLKFEEDDIV_50	 (0 << 4)	/* or 100, when Gen2En is 1 */
+#define SATA_SATAICFG_REFCLKFEEDDIV_60	 (1 << 4)	/* or 120. Used 25MHz */
+#define SATA_SATAICFG_REFCLKFEEDDIV_75	 (2 << 4)	/* or 150. Used 20MHz */
+#define SATA_SATAICFG_REFCLKFEEDDIV_90	 (3 << 4)	/* or 180 */
+#define SATA_SATAICFG_REFCLKFEEDDIV_MASK (3 << 4)
+#define SATA_SATAICFG_PHYSSCEN		 (1 << 6)
+#define SATA_SATAICFG_GEN2EN		 (1 << 7)
+#define SATA_SATAICFG_COMMEN		 (1 << 8)
+#define SATA_SATAICFG_PHYSHUTDOWN	 (1 << 9)
+#define SATA_SATAICFG_TARGETMODE	 (1 << 10)	/* 1 = Initiator */
+#define SATA_SATAICFG_COMCHANNEL	 (1 << 11)
+#define SATA_SATAICFG_IGNOREBSY		 (1 << 24)
+#define SATA_SATAICFG_LINKRSTEN		 (1 << 25)
+#define SATA_SATAICFG_CMDRETXDS		 (1 << 26)
+#define SATA_SATAICTL			0x344	/* Serial-ATA Interface Control */
+#define SATA_SATAICTL_PMPTX_MASK	 0x0000000f
+#define SATA_SATAICTL_PMPTX_SHIFT	 0
+#define SATA_SATAICTL_VUM		 (1 << 8)
+#define SATA_SATAICTL_VUS		 (1 << 9)
+#define SATA_SATAICTL_EDMAACT		 (1 << 16)
+#define SATA_SATAICTL_CLEARSTAT		 (1 << 24)
+#define SATA_SATAICTL_SRST		 (1 << 25)
+#define SATA_SATAITC			0x348	/* Serial-ATA Interface Test Control */
+#define SATA_SATAIS			0x34c	/* Serial-ATA Interface Status */
+#define SATA_VU				0x35c	/* Vendor Unique */
+#define SATA_FISC			0x360	/* FIS Configuration */
+#define SATA_FISC_FISWAIT4RDYEN_B0	 (1 << 0) /* Device to Host FIS */
+#define SATA_FISC_FISWAIT4RDYEN_B1	 (1 << 1) /* SDB FIS rcv with <N>bit 0 */
+#define SATA_FISC_FISWAIT4RDYEN_B2	 (1 << 2) /* DMA Activate FIS */
+#define SATA_FISC_FISWAIT4RDYEN_B3	 (1 << 3) /* DMA Setup FIS */
+#define SATA_FISC_FISWAIT4RDYEN_B4	 (1 << 4) /* Data FIS first DW */
+#define SATA_FISC_FISWAIT4RDYEN_B5	 (1 << 5) /* Data FIS entire FIS */
+#define SATA_FISC_FISWAIT4HOSTRDYEN_B0	 (1 << 8)
+				/* Device to Host FIS with <ERR> or <DF> */
+#define SATA_FISC_FISWAIT4HOSTRDYEN_B1	 (1 << 9) /* SDB FIS rcv with <N>bit */
+#define SATA_FISC_FISWAIT4HOSTRDYEN_B2	 (1 << 10) /* SDB FIS rcv with <ERR> */
+#define SATA_FISC_FISWAIT4HOSTRDYEN_B3	 (1 << 11) /* BIST Acivate FIS */
+#define SATA_FISC_FISWAIT4HOSTRDYEN_B4	 (1 << 12) /* PIO Setup FIS */
+#define SATA_FISC_FISWAIT4HOSTRDYEN_B5	 (1 << 13) /* Data FIS with Link error */
+#define SATA_FISC_FISWAIT4HOSTRDYEN_B6	 (1 << 14) /* Unrecognized FIS type */
+#define SATA_FISC_FISWAIT4HOSTRDYEN_B7	 (1 << 15) /* Any FIS */
+#define SATA_FISC_FISDMAACTIVATESYNCRESP (1 << 16)
+#define SATA_FISC_FISUNRECTYPECONT	 (1 << 17)
+#define SATA_FISIC			0x364	/* FIS Interrupt Cause */
+#define SATA_FISIM			0x368	/* FIS Interrupt Mask */
+#define SATA_FISDW0			0x370	/* FIS DW0 */
+#define SATA_FISDW1			0x374	/* FIS DW1 */
+#define SATA_FISDW2			0x378	/* FIS DW2 */
+#define SATA_FISDW3			0x37c	/* FIS DW3 */
+#define SATA_FISDW4			0x380	/* FIS DW4 */
+#define SATA_FISDW5			0x384	/* FIS DW5 */
+#define SATA_FISDW6			0x388	/* FIS DW6 */
+
+#define MVS_MAX_PORTS			8
+#define MVS_MAX_SLOTS			32
+
+/* Pessimistic prognosis on number of required S/G entries */
+#define MVS_SG_ENTRIES		(btoc(MAXPHYS) + 1)
+
+/* EDMA Command Request Block (CRQB) Data */
+struct mvs_crqb {
+	uint32_t cprdbl;	/* cPRD Desriptor Table Base Low Address */
+	uint32_t cprdbh;	/* cPRD Desriptor Table Base High Address */
+	uint16_t ctrlflg;	/* Control Flags */
+#define MVS_CRQB_READ		0x0001
+#define MVS_CRQB_TAG_MASK	0x003e
+#define MVS_CRQB_TAG_SHIFT	1
+#define MVS_CRQB_PMP_MASK	0xf000
+#define MVS_CRQB_PMP_SHIFT	12
+	uint8_t cmd[22];
+} __packed;
+
+struct mvs_crqb_gen2e {
+	uint32_t cprdbl;	/* cPRD Desriptor Table Base Low Address */
+	uint32_t cprdbh;	/* cPRD Desriptor Table Base High Address */
+	uint32_t ctrlflg;	/* Control Flags */
+#define MVS_CRQB2E_READ		0x00000001
+#define MVS_CRQB2E_DTAG_MASK	0x0000003e
+#define MVS_CRQB2E_DTAG_SHIFT	1
+#define MVS_CRQB2E_PMP_MASK	0x0000f000
+#define MVS_CRQB2E_PMP_SHIFT	12
+#define MVS_CRQB2E_CPRD		0x00010000
+#define MVS_CRQB2E_HTAG_MASK	0x003e0000
+#define MVS_CRQB2E_HTAG_SHIFT	17
+	uint32_t drbc;		/* Data Region Byte Count */
+	uint8_t cmd[16];
+} __packed;
+
+/* EDMA Phisical Region Descriptors (ePRD) Table Data Structure */
+struct mvs_eprd {
+	uint32_t prdbal;	/* Address bits[31:1] */
+	uint32_t bytecount;	/* Byte Count */
+#define MVS_EPRD_MASK		0x0000ffff      /* max 64KB */
+#define MVS_EPRD_MAX		(MVS_EPRD_MASK + 1)
+#define MVS_EPRD_EOF		0x80000000
+	uint32_t prdbah;	/* Address bits[63:32] */
+	uint32_t resv;
+} __packed;
+
+/* Command request blocks. 32 commands. First 1Kbyte aligned. */
+#define MVS_CRQB_OFFSET		0
+#define MVS_CRQB_SIZE		32	/* sizeof(struct mvs_crqb) */
+#define MVS_CRQB_MASK		0x000003e0
+#define MVS_CRQB_SHIFT		5
+#define MVS_CRQB_TO_ADDR(slot)	((slot) << MVS_CRQB_SHIFT)
+#define MVS_ADDR_TO_CRQB(addr)	(((addr) & MVS_CRQB_MASK) >> MVS_CRQB_SHIFT)
+/* ePRD blocks. Up to 32 commands, Each 16byte aligned. */
+#define MVS_EPRD_OFFSET		(MVS_CRQB_OFFSET + MVS_CRQB_SIZE * MVS_MAX_SLOTS)
+#define MVS_EPRD_SIZE		(MVS_SG_ENTRIES * 16) /* sizeof(struct mvs_eprd) */
+/* Request work area. */
+#define MVS_WORKRQ_SIZE		(MVS_EPRD_OFFSET + MVS_EPRD_SIZE * MVS_MAX_SLOTS)
+
+/* EDMA Command Response Block (CRPB) Data */
+struct mvs_crpb {
+	uint16_t id;		/* CRPB ID */
+#define MVS_CRPB_TAG_MASK	0x001F
+#define MVS_CRPB_TAG_SHIFT	0
+	uint16_t rspflg;	/* CPRB Response Flags */
+#define MVS_CRPB_EDMASTS_MASK	0x007F
+#define MVS_CRPB_EDMASTS_SHIFT	0
+#define MVS_CRPB_ATASTS_MASK	0xFF00
+#define MVS_CRPB_ATASTS_SHIFT	8
+	uint32_t ts;		/* CPRB Time Stamp */
+} __packed;
+
+/* Command response blocks. 32 commands. First 256byte aligned. */
+#define MVS_CRPB_OFFSET		0
+#define MVS_CRPB_SIZE		sizeof(struct mvs_crpb)
+#define MVS_CRPB_MASK		0x000000f8
+#define MVS_CRPB_SHIFT		3
+#define MVS_CRPB_TO_ADDR(slot)	((slot) << MVS_CRPB_SHIFT)
+#define MVS_ADDR_TO_CRPB(addr)	(((addr) & MVS_CRPB_MASK) >> MVS_CRPB_SHIFT)
+/* Request work area. */
+#define MVS_WORKRP_SIZE		(MVS_CRPB_OFFSET + MVS_CRPB_SIZE * MVS_MAX_SLOTS)
+
+/* misc defines */
+#define ATA_IRQ_RID		0
+#define ATA_INTR_FLAGS		(INTR_MPSAFE|INTR_TYPE_BIO|INTR_ENTROPY)
+
+struct ata_dmaslot {
+    bus_dmamap_t                data_map;       /* Data DMA map */
+    bus_addr_t			addr;		/* Data address */
+    uint16_t			len;		/* Data size */
+};
+
+/* structure holding DMA related information */
+struct mvs_dma {
+    bus_dma_tag_t               workrq_tag;	/* Request workspace DMA tag */
+    bus_dmamap_t                workrq_map;	/* Request workspace DMA map */
+    uint8_t                     *workrq;	/* Request workspace */
+    bus_addr_t                  workrq_bus;	/* Request bus address */
+    bus_dma_tag_t               workrp_tag;	/* Reply workspace DMA tag */
+    bus_dmamap_t                workrp_map;	/* Reply workspace DMA map */
+    uint8_t                     *workrp;	/* Reply workspace */
+    bus_addr_t                  workrp_bus;	/* Reply bus address */
+    bus_dma_tag_t               data_tag;	/* Data DMA tag */
+};
+
+enum mvs_slot_states {
+	MVS_SLOT_EMPTY,
+	MVS_SLOT_LOADING,
+	MVS_SLOT_RUNNING,
+	MVS_SLOT_EXECUTING
+};
+
+struct mvs_slot {
+    device_t                    dev;            /* Device handle */
+    int				slot;           /* Number of this slot */
+    int				tag;            /* Used command tag */
+    enum mvs_slot_states	state;          /* Slot state */
+    union ccb			*ccb;		/* CCB occupying slot */
+    struct ata_dmaslot          dma;            /* DMA data of this slot */
+    struct callout              timeout;        /* Execution timeout */
+};
+
+struct mvs_device {
+	int			revision;
+	int			mode;
+	u_int			bytecount;
+	u_int			atapi;
+	u_int			tags;
+	u_int			caps;
+};
+
+enum mvs_edma_mode {
+	MVS_EDMA_UNKNOWN,
+	MVS_EDMA_OFF,
+	MVS_EDMA_ON,
+	MVS_EDMA_QUEUED,
+	MVS_EDMA_NCQ,
+};
+
+/* structure describing an ATA channel */
+struct mvs_channel {
+	device_t		dev;            /* Device handle */
+	int			unit;           /* Physical channel */
+	struct resource		*r_mem;		/* Memory of this channel */
+	struct resource		*r_irq;         /* Interrupt of this channel */
+	void			*ih;            /* Interrupt handle */
+	struct mvs_dma		dma;            /* DMA data */
+	struct cam_sim		*sim;
+	struct cam_path		*path;
+	int			quirks;
+#define MVS_Q_GENI	1
+#define MVS_Q_GENII	2
+#define MVS_Q_GENIIE	4
+#define MVS_Q_SOC	8
+#define MVS_Q_CT	16
+	int			pm_level;	/* power management level */
+
+	struct mvs_slot		slot[MVS_MAX_SLOTS];
+	union ccb		*hold[MVS_MAX_SLOTS];
+	int			holdtag[MVS_MAX_SLOTS]; /* Tags used for holden commands. */
+	struct mtx		mtx;		/* state lock */
+	int			devices;        /* What is present */
+	int			pm_present;	/* PM presence reported */
+	enum mvs_edma_mode	curr_mode;	/* Current EDMA mode */
+	int			fbs_enabled;	/* FIS-based switching enabled */
+	uint32_t		oslots;		/* Occupied slots */
+	uint32_t		otagspd[16];	/* Occupied device tags */
+	uint32_t		rslots;		/* Running slots */
+	uint32_t		aslots;		/* Slots with atomic commands  */
+	uint32_t		eslots;		/* Slots in error */
+	uint32_t		toslots;	/* Slots in timeout */
+	int			numrslots;	/* Number of running slots */
+	int			numrslotspd[16];/* Number of running slots per dev */
+	int			numpslots;	/* Number of PIO slots */
+	int			numdslots;	/* Number of DMA slots */
+	int			numtslots;	/* Number of NCQ slots */
+	int			numtslotspd[16];/* Number of NCQ slots per dev */
+	int			numhslots;	/* Number of holden slots */
+	int			readlog;	/* Our READ LOG active */
+	int			fatalerr;	/* Fatal error happend */
+	int			lastslot;	/* Last used slot */
+	int			taggedtarget;	/* Last tagged target */
+	int			out_idx;	/* Next written CRQB */
+	int			in_idx;		/* Next read CRPB */
+	u_int			transfersize;	/* PIO transfer size */
+	u_int			donecount;	/* PIO bytes sent/received */
+	u_int			basic_dma;	/* Basic DMA used for ATAPI */
+	u_int			fake_busy;	/* Fake busy bit after command submission */
+	union ccb		*frozen;	/* Frozen command */
+	struct callout		pm_timer;	/* Power management events */
+
+	struct mvs_device	user[16];	/* User-specified settings */
+	struct mvs_device	curr[16];	/* Current settings */
+};
+
+/* structure describing a MVS controller */
+struct mvs_controller {
+	device_t		dev;
+	int			r_rid;
+	struct resource		*r_mem;
+	struct rman		sc_iomem;
+	struct mvs_controller_irq {
+		struct resource		*r_irq;
+		void			*handle;
+		int			r_irq_rid;
+	} irq;
+	int			quirks;
+	int			channels;
+	int			ccc;		/* CCC timeout */
+	int			cccc;		/* CCC commands */
+	struct mtx		mtx;		/* MIM access lock */
+	int			gmim;		/* Globally wanted MIM bits */
+	int			pmim;		/* Port wanted MIM bits */
+	int			mim;		/* Current MIM bits */
+	int			msi;		/* MSI enabled */
+	int			msia;		/* MSI active */
+	struct {
+		void			(*function)(void *);
+		void			*argument;
+	} interrupt[MVS_MAX_PORTS];
+};
+
+enum mvs_err_type {
+	MVS_ERR_NONE,		/* No error */
+	MVS_ERR_INVALID,	/* Error detected by us before submitting. */
+	MVS_ERR_INNOCENT,	/* Innocent victim. */
+	MVS_ERR_TFE,		/* Task File Error. */
+	MVS_ERR_SATA,		/* SATA error. */
+	MVS_ERR_TIMEOUT,	/* Command execution timeout. */
+	MVS_ERR_NCQ,		/* NCQ command error. CCB should be put on hold
+				 * until READ LOG executed to reveal error. */
+};
+
+struct mvs_intr_arg {
+	void *arg;
+	u_int cause;
+};
+
+extern devclass_t mvs_devclass;
+
+/* macros to hide busspace uglyness */
+#define ATA_INB(res, offset) \
+	bus_read_1((res), (offset))
+#define ATA_INW(res, offset) \
+	bus_read_2((res), (offset))
+#define ATA_INL(res, offset) \
+	bus_read_4((res), (offset))
+#define ATA_INSW(res, offset, addr, count) \
+	bus_read_multi_2((res), (offset), (addr), (count))
+#define ATA_INSW_STRM(res, offset, addr, count) \
+	bus_read_multi_stream_2((res), (offset), (addr), (count))
+#define ATA_INSL(res, offset, addr, count) \
+	bus_read_multi_4((res), (offset), (addr), (count))
+#define ATA_INSL_STRM(res, offset, addr, count) \
+	bus_read_multi_stream_4((res), (offset), (addr), (count))
+#define ATA_OUTB(res, offset, value) \
+	bus_write_1((res), (offset), (value))
+#define ATA_OUTW(res, offset, value) \
+	bus_write_2((res), (offset), (value))
+#define ATA_OUTL(res, offset, value) \
+	bus_write_4((res), (offset), (value));
+#define ATA_OUTSW(res, offset, addr, count) \
+	bus_write_multi_2((res), (offset), (addr), (count))
+#define ATA_OUTSW_STRM(res, offset, addr, count) \
+	bus_write_multi_stream_2((res), (offset), (addr), (count))
+#define ATA_OUTSL(res, offset, addr, count) \
+	bus_write_multi_4((res), (offset), (addr), (count))
+#define ATA_OUTSL_STRM(res, offset, addr, count) \
+	bus_write_multi_stream_4((res), (offset), (addr), (count))
diff --git a/sys/dev/mvs/mvs_if.m b/sys/dev/mvs/mvs_if.m
new file mode 100644
index 0000000..e744219
--- /dev/null
+++ b/sys/dev/mvs/mvs_if.m
@@ -0,0 +1,34 @@
+# Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
+# All rights reserved.
+#
+# Redistribution and use in source and binary forms, with or without
+# modification, are permitted provided that the following conditions
+# are met:
+# 1. Redistributions of source code must retain the above copyright
+#    notice, this list of conditions and the following disclaimer,
+#    without modification, immediately at the beginning of the file.
+# 2. Redistributions in binary form must reproduce the above copyright
+#    notice, this list of conditions and the following disclaimer in the
+#    documentation and/or other materials provided with the distribution.
+#
+# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+# NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+# THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+#
+# $FreeBSD$
+
+INTERFACE mvs;
+
+METHOD void edma {
+    device_t    dev;
+    device_t    child;
+    int         mode;
+};
+
diff --git a/sys/dev/mvs/mvs_pci.c b/sys/dev/mvs/mvs_pci.c
new file mode 100644
index 0000000..4fae627
--- /dev/null
+++ b/sys/dev/mvs/mvs_pci.c
@@ -0,0 +1,507 @@
+/*-
+ * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/module.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/malloc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <vm/uma.h>
+#include <machine/stdarg.h>
+#include <machine/resource.h>
+#include <machine/bus.h>
+#include <sys/rman.h>
+#include <dev/pci/pcivar.h>
+#include <dev/pci/pcireg.h>
+#include "mvs.h"
+
+/* local prototypes */
+static int mvs_setup_interrupt(device_t dev);
+static void mvs_intr(void *data);
+static int mvs_suspend(device_t dev);
+static int mvs_resume(device_t dev);
+static int mvs_ctlr_setup(device_t dev);
+
+static struct {
+	uint32_t	id;
+	uint8_t		rev;
+	const char	*name;
+	int		ports;
+	int		quirks;
+} mvs_ids[] = {
+	{0x504011ab, 0x00, "Marvell 88SX5040",	4,	MVS_Q_GENI},
+	{0x504111ab, 0x00, "Marvell 88SX5041",	4,	MVS_Q_GENI},
+	{0x508011ab, 0x00, "Marvell 88SX5080",	8,	MVS_Q_GENI},
+	{0x508111ab, 0x00, "Marvell 88SX5081",	8,	MVS_Q_GENI},
+	{0x604011ab, 0x00, "Marvell 88SX6040",	4,	MVS_Q_GENII},
+	{0x604111ab, 0x00, "Marvell 88SX6041",	4,	MVS_Q_GENII},
+	{0x604211ab, 0x00, "Marvell 88SX6042",	4,	MVS_Q_GENIIE},
+	{0x608011ab, 0x00, "Marvell 88SX6080",	8,	MVS_Q_GENII},
+	{0x608111ab, 0x00, "Marvell 88SX6081",	8,	MVS_Q_GENII},
+	{0x704211ab, 0x00, "Marvell 88SX7042",	4,	MVS_Q_GENIIE|MVS_Q_CT},
+	{0x02419005, 0x00, "Adaptec 1420SA",	4,	MVS_Q_GENII},
+	{0x02439005, 0x00, "Adaptec 1430SA",	4,	MVS_Q_GENIIE|MVS_Q_CT},
+	{0x00000000, 0x00, NULL,	0,	0}
+};
+
+static int
+mvs_probe(device_t dev)
+{
+	char buf[64];
+	int i;
+	uint32_t devid = pci_get_devid(dev);
+	uint8_t revid = pci_get_revid(dev);
+
+	for (i = 0; mvs_ids[i].id != 0; i++) {
+		if (mvs_ids[i].id == devid &&
+		    mvs_ids[i].rev <= revid) {
+			snprintf(buf, sizeof(buf), "%s SATA controller",
+			    mvs_ids[i].name);
+			device_set_desc_copy(dev, buf);
+			return (BUS_PROBE_VENDOR);
+		}
+	}
+	return (ENXIO);
+}
+
+static int
+mvs_attach(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	device_t child;
+	int	error, unit, i;
+	uint32_t devid = pci_get_devid(dev);
+	uint8_t revid = pci_get_revid(dev);
+
+	ctlr->dev = dev;
+	i = 0;
+	while (mvs_ids[i].id != 0 &&
+	    (mvs_ids[i].id != devid ||
+	     mvs_ids[i].rev > revid))
+		i++;
+	ctlr->channels = mvs_ids[i].ports;
+	ctlr->quirks = mvs_ids[i].quirks;
+	resource_int_value(device_get_name(dev),
+	    device_get_unit(dev), "ccc", &ctlr->ccc);
+	ctlr->cccc = 8;
+	resource_int_value(device_get_name(dev),
+	    device_get_unit(dev), "cccc", &ctlr->cccc);
+	if (ctlr->ccc == 0 || ctlr->cccc == 0) {
+		ctlr->ccc = 0;
+		ctlr->cccc = 0;
+	}
+	if (ctlr->ccc > 100000)
+		ctlr->ccc = 100000;
+	device_printf(dev,
+	    "Gen-%s, %d %sGbps ports, Port Multiplier %s%s\n",
+	    ((ctlr->quirks & MVS_Q_GENI) ? "I" :
+	     ((ctlr->quirks & MVS_Q_GENII) ? "II" : "IIe")),
+	    ctlr->channels,
+	    ((ctlr->quirks & MVS_Q_GENI) ? "1.5" : "3"),
+	    ((ctlr->quirks & MVS_Q_GENI) ?
+	    "not supported" : "supported"),
+	    ((ctlr->quirks & MVS_Q_GENIIE) ?
+	    " with FBS" : ""));
+	mtx_init(&ctlr->mtx, "MVS controller lock", NULL, MTX_DEF);
+	/* We should have a memory BAR(0). */
+	ctlr->r_rid = PCIR_BAR(0);
+	if (!(ctlr->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &ctlr->r_rid, RF_ACTIVE)))
+		return ENXIO;
+	/* Setup our own memory management for channels. */
+	ctlr->sc_iomem.rm_type = RMAN_ARRAY;
+	ctlr->sc_iomem.rm_descr = "I/O memory addresses";
+	if ((error = rman_init(&ctlr->sc_iomem)) != 0) {
+		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
+		return (error);
+	}
+	if ((error = rman_manage_region(&ctlr->sc_iomem,
+	    rman_get_start(ctlr->r_mem), rman_get_end(ctlr->r_mem))) != 0) {
+		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
+		rman_fini(&ctlr->sc_iomem);
+		return (error);
+	}
+	pci_enable_busmaster(dev);
+	mvs_ctlr_setup(dev);
+	/* Setup interrupts. */
+	if (mvs_setup_interrupt(dev)) {
+		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
+		rman_fini(&ctlr->sc_iomem);
+		return ENXIO;
+	}
+	/* Attach all channels on this controller */
+	for (unit = 0; unit < ctlr->channels; unit++) {
+		child = device_add_child(dev, "mvsch", -1);
+		if (child == NULL)
+			device_printf(dev, "failed to add channel device\n");
+		else
+			device_set_ivars(child, (void *)(intptr_t)unit);
+	}
+	bus_generic_attach(dev);
+	return 0;
+}
+
+static int
+mvs_detach(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	device_t *children;
+	int nchildren, i;
+
+	/* Detach & delete all children */
+	if (!device_get_children(dev, &children, &nchildren)) {
+		for (i = 0; i < nchildren; i++)
+			device_delete_child(dev, children[i]);
+		free(children, M_TEMP);
+	}
+	/* Free interrupt. */
+	if (ctlr->irq.r_irq) {
+		bus_teardown_intr(dev, ctlr->irq.r_irq,
+		    ctlr->irq.handle);
+		bus_release_resource(dev, SYS_RES_IRQ,
+		    ctlr->irq.r_irq_rid, ctlr->irq.r_irq);
+	}
+	pci_release_msi(dev);
+	/* Free memory. */
+	rman_fini(&ctlr->sc_iomem);
+	if (ctlr->r_mem)
+		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
+	mtx_destroy(&ctlr->mtx);
+	return (0);
+}
+
+static int
+mvs_ctlr_setup(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int i, ccc = ctlr->ccc, cccc = ctlr->cccc, ccim = 0;
+
+	/* Mask chip interrupts */
+	ATA_OUTL(ctlr->r_mem, CHIP_MIM, 0x00000000);
+	/* Mask PCI interrupts */
+	ATA_OUTL(ctlr->r_mem, CHIP_PCIIM, 0x00000000);
+	/* Clear PCI interrupts */
+	ATA_OUTL(ctlr->r_mem, CHIP_PCIIC, 0x00000000);
+	if (ccc && bootverbose) {
+		device_printf(dev,
+		    "CCC with %dus/%dcmd enabled\n",
+		    ctlr->ccc, ctlr->cccc);
+	}
+	ccc *= 150;
+	/* Configure chip-global CCC */
+	if (ctlr->channels > 4 && (ctlr->quirks & MVS_Q_GENI) == 0) {
+		ATA_OUTL(ctlr->r_mem, CHIP_ICT, cccc);
+		ATA_OUTL(ctlr->r_mem, CHIP_ITT, ccc);
+		ATA_OUTL(ctlr->r_mem, CHIP_ICC, ~CHIP_ICC_ALL_PORTS);
+		if (ccc)
+			ccim |= IC_ALL_PORTS_COAL_DONE;
+		ccc = 0;
+		cccc = 0;
+	}
+	for (i = 0; i < ctlr->channels / 4; i++) {
+		/* Configure per-HC CCC */
+		ATA_OUTL(ctlr->r_mem, HC_BASE(i) + HC_ICT, cccc);
+		ATA_OUTL(ctlr->r_mem, HC_BASE(i) + HC_ITT, ccc);
+		if (ccc)
+			ccim |= (IC_HC0_COAL_DONE << (i * IC_HC_SHIFT));
+		/* Clear HC interrupts */
+		ATA_OUTL(ctlr->r_mem, HC_BASE(i) + HC_IC, 0x00000000);
+	}
+	/* Enable chip interrupts */
+	ctlr->gmim = (ccim ? ccim : (IC_DONE_HC0 | IC_DONE_HC1)) |
+	     IC_ERR_HC0 | IC_ERR_HC1;
+	ctlr->mim = ctlr->gmim | ctlr->pmim;
+	ATA_OUTL(ctlr->r_mem, CHIP_MIM, ctlr->mim);
+	/* Enable PCI interrupts */
+	ATA_OUTL(ctlr->r_mem, CHIP_PCIIM, 0x007fffff);
+	return (0);
+}
+
+static void
+mvs_edma(device_t dev, device_t child, int mode)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int unit = ((struct mvs_channel *)device_get_softc(child))->unit;
+	int bit = IC_DONE_IRQ << (unit * 2 + unit / 4) ;
+
+	if (ctlr->ccc == 0)
+		return;
+	/* CCC is not working for non-EDMA mode. Unmask device interrupts. */
+	mtx_lock(&ctlr->mtx);
+	if (mode == MVS_EDMA_OFF)
+		ctlr->pmim |= bit;
+	else
+		ctlr->pmim &= ~bit;
+	ctlr->mim = ctlr->gmim | ctlr->pmim;
+	if (!ctlr->msia)
+		ATA_OUTL(ctlr->r_mem, CHIP_MIM, ctlr->mim);
+	mtx_unlock(&ctlr->mtx);
+}
+
+static int
+mvs_suspend(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+
+	bus_generic_suspend(dev);
+	/* Mask chip interrupts */
+	ATA_OUTL(ctlr->r_mem, CHIP_MIM, 0x00000000);
+	/* Mask PCI interrupts */
+	ATA_OUTL(ctlr->r_mem, CHIP_PCIIM, 0x00000000);
+	return 0;
+}
+
+static int
+mvs_resume(device_t dev)
+{
+
+	mvs_ctlr_setup(dev);
+	return (bus_generic_resume(dev));
+}
+
+static int
+mvs_setup_interrupt(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int msi = 0;
+
+	/* Process hints. */
+	resource_int_value(device_get_name(dev),
+	    device_get_unit(dev), "msi", &msi);
+	if (msi < 0)
+		msi = 0;
+	else if (msi > 0)
+		msi = min(1, pci_msi_count(dev));
+	/* Allocate MSI if needed/present. */
+	if (msi && pci_alloc_msi(dev, &msi) != 0)
+		msi = 0;
+	ctlr->msi = msi;
+	/* Allocate all IRQs. */
+	ctlr->irq.r_irq_rid = msi ? 1 : 0;
+	if (!(ctlr->irq.r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+	    &ctlr->irq.r_irq_rid, RF_SHAREABLE | RF_ACTIVE))) {
+		device_printf(dev, "unable to map interrupt\n");
+		return (ENXIO);
+	}
+	if ((bus_setup_intr(dev, ctlr->irq.r_irq, ATA_INTR_FLAGS, NULL,
+	    mvs_intr, ctlr, &ctlr->irq.handle))) {
+		device_printf(dev, "unable to setup interrupt\n");
+		bus_release_resource(dev, SYS_RES_IRQ,
+		    ctlr->irq.r_irq_rid, ctlr->irq.r_irq);
+		ctlr->irq.r_irq = 0;
+		return (ENXIO);
+	}
+	return (0);
+}
+
+/*
+ * Common case interrupt handler.
+ */
+static void
+mvs_intr(void *data)
+{
+	struct mvs_controller *ctlr = data;
+	struct mvs_intr_arg arg;
+	void (*function)(void *);
+	int p;
+	u_int32_t ic, aic;
+
+	ic = ATA_INL(ctlr->r_mem, CHIP_MIC);
+//device_printf(ctlr->dev, "irq MIC:%08x\n", ic);
+	if (ctlr->msi) {
+		/* We have to to mask MSI during processing. */
+		mtx_lock(&ctlr->mtx);
+		ATA_OUTL(ctlr->r_mem, CHIP_MIM, 0);
+		ctlr->msia = 1; /* Deny MIM update during processing. */
+		mtx_unlock(&ctlr->mtx);
+	} else if (ic == 0)
+		return;
+	/* Acknowledge all-ports CCC interrupt. */
+	if (ic & IC_ALL_PORTS_COAL_DONE)
+		ATA_OUTL(ctlr->r_mem, CHIP_ICC, ~CHIP_ICC_ALL_PORTS);
+	for (p = 0; p < ctlr->channels; p++) {
+		if ((p & 3) == 0) {
+			if (p != 0)
+				ic >>= 1;
+			if ((ic & IC_HC0) == 0) {
+				p += 3;
+				ic >>= 8;
+				continue;
+			}
+			/* Acknowledge interrupts of this HC. */
+			aic = 0;
+			if (ic & (IC_DONE_IRQ << 0))
+				aic |= HC_IC_DONE(0) | HC_IC_DEV(0);
+			if (ic & (IC_DONE_IRQ << 2))
+				aic |= HC_IC_DONE(1) | HC_IC_DEV(1);
+			if (ic & (IC_DONE_IRQ << 4))
+				aic |= HC_IC_DONE(2) | HC_IC_DEV(2);
+			if (ic & (IC_DONE_IRQ << 6))
+				aic |= HC_IC_DONE(3) | HC_IC_DEV(3);
+			if (ic & IC_HC0_COAL_DONE)
+				aic |= HC_IC_COAL;
+			ATA_OUTL(ctlr->r_mem, HC_BASE(p == 4) + HC_IC, ~aic);
+		}
+		/* Call per-port interrupt handler. */
+		arg.cause = ic & (IC_ERR_IRQ|IC_DONE_IRQ);
+		if ((arg.cause != 0) &&
+		    (function = ctlr->interrupt[p].function)) {
+			arg.arg = ctlr->interrupt[p].argument;
+			function(&arg);
+		}
+		ic >>= 2;
+	}
+	if (ctlr->msi) {
+		/* Unmasking MSI triggers next interrupt, if needed. */
+		mtx_lock(&ctlr->mtx);
+		ctlr->msia = 0;	/* Allow MIM update. */
+		ATA_OUTL(ctlr->r_mem, CHIP_MIM, ctlr->mim);
+		mtx_unlock(&ctlr->mtx);
+	}
+}
+
+static struct resource *
+mvs_alloc_resource(device_t dev, device_t child, int type, int *rid,
+		       u_long start, u_long end, u_long count, u_int flags)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int unit = ((struct mvs_channel *)device_get_softc(child))->unit;
+	struct resource *res = NULL;
+	int offset = HC_BASE(unit >> 2) + PORT_BASE(unit & 0x03);
+	long st;
+
+	switch (type) {
+	case SYS_RES_MEMORY:
+		st = rman_get_start(ctlr->r_mem);
+		res = rman_reserve_resource(&ctlr->sc_iomem, st + offset,
+		    st + offset + PORT_SIZE - 1, PORT_SIZE, RF_ACTIVE, child);
+		if (res) {
+			bus_space_handle_t bsh;
+			bus_space_tag_t bst;
+			bsh = rman_get_bushandle(ctlr->r_mem);
+			bst = rman_get_bustag(ctlr->r_mem);
+			bus_space_subregion(bst, bsh, offset, PORT_SIZE, &bsh);
+			rman_set_bushandle(res, bsh);
+			rman_set_bustag(res, bst);
+		}
+		break;
+	case SYS_RES_IRQ:
+		if (*rid == ATA_IRQ_RID)
+			res = ctlr->irq.r_irq;
+		break;
+	}
+	return (res);
+}
+
+static int
+mvs_release_resource(device_t dev, device_t child, int type, int rid,
+			 struct resource *r)
+{
+
+	switch (type) {
+	case SYS_RES_MEMORY:
+		rman_release_resource(r);
+		return (0);
+	case SYS_RES_IRQ:
+		if (rid != ATA_IRQ_RID)
+			return ENOENT;
+		return (0);
+	}
+	return (EINVAL);
+}
+
+static int
+mvs_setup_intr(device_t dev, device_t child, struct resource *irq, 
+		   int flags, driver_filter_t *filter, driver_intr_t *function, 
+		   void *argument, void **cookiep)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int unit = (intptr_t)device_get_ivars(child);
+
+	if (filter != NULL) {
+		printf("mvs.c: we cannot use a filter here\n");
+		return (EINVAL);
+	}
+	ctlr->interrupt[unit].function = function;
+	ctlr->interrupt[unit].argument = argument;
+	return (0);
+}
+
+static int
+mvs_teardown_intr(device_t dev, device_t child, struct resource *irq,
+		      void *cookie)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int unit = (intptr_t)device_get_ivars(child);
+
+	ctlr->interrupt[unit].function = NULL;
+	ctlr->interrupt[unit].argument = NULL;
+	return (0);
+}
+
+static int
+mvs_print_child(device_t dev, device_t child)
+{
+	int retval;
+
+	retval = bus_print_child_header(dev, child);
+	retval += printf(" at channel %d",
+	    (int)(intptr_t)device_get_ivars(child));
+	retval += bus_print_child_footer(dev, child);
+
+	return (retval);
+}
+
+static device_method_t mvs_methods[] = {
+	DEVMETHOD(device_probe,     mvs_probe),
+	DEVMETHOD(device_attach,    mvs_attach),
+	DEVMETHOD(device_detach,    mvs_detach),
+	DEVMETHOD(device_suspend,   mvs_suspend),
+	DEVMETHOD(device_resume,    mvs_resume),
+	DEVMETHOD(bus_print_child,  mvs_print_child),
+	DEVMETHOD(bus_alloc_resource,       mvs_alloc_resource),
+	DEVMETHOD(bus_release_resource,     mvs_release_resource),
+	DEVMETHOD(bus_setup_intr,   mvs_setup_intr),
+	DEVMETHOD(bus_teardown_intr,mvs_teardown_intr),
+	DEVMETHOD(mvs_edma,         mvs_edma),
+	{ 0, 0 }
+};
+static driver_t mvs_driver = {
+        "mvs",
+        mvs_methods,
+        sizeof(struct mvs_controller)
+};
+DRIVER_MODULE(mvs, pci, mvs_driver, mvs_devclass, 0, 0);
+MODULE_VERSION(mvs, 1);
+MODULE_DEPEND(mvs, cam, 1, 1, 1);
+
diff --git a/sys/dev/mvs/mvs_soc.c b/sys/dev/mvs/mvs_soc.c
new file mode 100644
index 0000000..298a873
--- /dev/null
+++ b/sys/dev/mvs/mvs_soc.c
@@ -0,0 +1,437 @@
+/*-
+ * Copyright (c) 2010 Alexander Motin <mav@FreeBSD.org>
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer,
+ *    without modification, immediately at the beginning of the file.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/param.h>
+#include <sys/module.h>
+#include <sys/systm.h>
+#include <sys/kernel.h>
+#include <sys/bus.h>
+#include <sys/endian.h>
+#include <sys/malloc.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <vm/uma.h>
+#include <machine/stdarg.h>
+#include <machine/resource.h>
+#include <machine/bus.h>
+#include <sys/rman.h>
+#include <arm/mv/mvreg.h>
+#include <arm/mv/mvvar.h>
+#include "mvs.h"
+
+/* local prototypes */
+static int mvs_setup_interrupt(device_t dev);
+static void mvs_intr(void *data);
+static int mvs_suspend(device_t dev);
+static int mvs_resume(device_t dev);
+static int mvs_ctlr_setup(device_t dev);
+
+static struct {
+	uint32_t	id;
+	uint8_t		rev;
+	const char	*name;
+	int		ports;
+	int		quirks;
+} mvs_ids[] = {
+	{MV_DEV_88F5182, 0x00,   "Marvell 88F5182",	2, MVS_Q_GENIIE|MVS_Q_SOC},
+	{MV_DEV_88F6281, 0x00,   "Marvell 88F6281",	2, MVS_Q_GENIIE|MVS_Q_SOC},
+	{MV_DEV_MV78100, 0x00,   "Marvell MV78100",	2, MVS_Q_GENIIE|MVS_Q_SOC},
+	{MV_DEV_MV78100_Z0, 0x00,"Marvell MV78100",	2, MVS_Q_GENIIE|MVS_Q_SOC},
+	{0,              0x00,   NULL,			0, 0}
+};
+
+static int
+mvs_probe(device_t dev)
+{
+	char buf[64];
+	int i;
+	uint32_t devid, revid;
+
+	soc_id(&devid, &revid);
+	for (i = 0; mvs_ids[i].id != 0; i++) {
+		if (mvs_ids[i].id == devid &&
+		    mvs_ids[i].rev <= revid) {
+			snprintf(buf, sizeof(buf), "%s SATA controller",
+			    mvs_ids[i].name);
+			device_set_desc_copy(dev, buf);
+			return (BUS_PROBE_VENDOR);
+		}
+	}
+	return (ENXIO);
+}
+
+static int
+mvs_attach(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	device_t child;
+	int	error, unit, i;
+	uint32_t devid, revid;
+
+	soc_id(&devid, &revid);
+	ctlr->dev = dev;
+	i = 0;
+	while (mvs_ids[i].id != 0 &&
+	    (mvs_ids[i].id != devid ||
+	     mvs_ids[i].rev > revid))
+		i++;
+	ctlr->channels = mvs_ids[i].ports;
+	ctlr->quirks = mvs_ids[i].quirks;
+	resource_int_value(device_get_name(dev),
+	    device_get_unit(dev), "ccc", &ctlr->ccc);
+	ctlr->cccc = 8;
+	resource_int_value(device_get_name(dev),
+	    device_get_unit(dev), "cccc", &ctlr->cccc);
+	if (ctlr->ccc == 0 || ctlr->cccc == 0) {
+		ctlr->ccc = 0;
+		ctlr->cccc = 0;
+	}
+	if (ctlr->ccc > 100000)
+		ctlr->ccc = 100000;
+	device_printf(dev,
+	    "Gen-%s, %d %sGbps ports, Port Multiplier %s%s\n",
+	    ((ctlr->quirks & MVS_Q_GENI) ? "I" :
+	     ((ctlr->quirks & MVS_Q_GENII) ? "II" : "IIe")),
+	    ctlr->channels,
+	    ((ctlr->quirks & MVS_Q_GENI) ? "1.5" : "3"),
+	    ((ctlr->quirks & MVS_Q_GENI) ?
+	    "not supported" : "supported"),
+	    ((ctlr->quirks & MVS_Q_GENIIE) ?
+	    " with FBS" : ""));
+	mtx_init(&ctlr->mtx, "MVS controller lock", NULL, MTX_DEF);
+	/* We should have a memory BAR(0). */
+	ctlr->r_rid = 0;
+	if (!(ctlr->r_mem = bus_alloc_resource_any(dev, SYS_RES_MEMORY,
+	    &ctlr->r_rid, RF_ACTIVE)))
+		return ENXIO;
+	/* Setup our own memory management for channels. */
+	ctlr->sc_iomem.rm_type = RMAN_ARRAY;
+	ctlr->sc_iomem.rm_descr = "I/O memory addresses";
+	if ((error = rman_init(&ctlr->sc_iomem)) != 0) {
+		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
+		return (error);
+	}
+	if ((error = rman_manage_region(&ctlr->sc_iomem,
+	    rman_get_start(ctlr->r_mem), rman_get_end(ctlr->r_mem))) != 0) {
+		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
+		rman_fini(&ctlr->sc_iomem);
+		return (error);
+	}
+	mvs_ctlr_setup(dev);
+	/* Setup interrupts. */
+	if (mvs_setup_interrupt(dev)) {
+		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
+		rman_fini(&ctlr->sc_iomem);
+		return ENXIO;
+	}
+	/* Attach all channels on this controller */
+	for (unit = 0; unit < ctlr->channels; unit++) {
+		child = device_add_child(dev, "mvsch", -1);
+		if (child == NULL)
+			device_printf(dev, "failed to add channel device\n");
+		else
+			device_set_ivars(child, (void *)(intptr_t)unit);
+	}
+	bus_generic_attach(dev);
+	return 0;
+}
+
+static int
+mvs_detach(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	device_t *children;
+	int nchildren, i;
+
+	/* Detach & delete all children */
+	if (!device_get_children(dev, &children, &nchildren)) {
+		for (i = 0; i < nchildren; i++)
+			device_delete_child(dev, children[i]);
+		free(children, M_TEMP);
+	}
+	/* Free interrupt. */
+	if (ctlr->irq.r_irq) {
+		bus_teardown_intr(dev, ctlr->irq.r_irq,
+		    ctlr->irq.handle);
+		bus_release_resource(dev, SYS_RES_IRQ,
+		    ctlr->irq.r_irq_rid, ctlr->irq.r_irq);
+	}
+	/* Free memory. */
+	rman_fini(&ctlr->sc_iomem);
+	if (ctlr->r_mem)
+		bus_release_resource(dev, SYS_RES_MEMORY, ctlr->r_rid, ctlr->r_mem);
+	mtx_destroy(&ctlr->mtx);
+	return (0);
+}
+
+static int
+mvs_ctlr_setup(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int ccc = ctlr->ccc, cccc = ctlr->cccc, ccim = 0;
+
+	/* Mask chip interrupts */
+	ATA_OUTL(ctlr->r_mem, CHIP_SOC_MIM, 0x00000000);
+	/* Clear HC interrupts */
+	ATA_OUTL(ctlr->r_mem, HC_IC, 0x00000000);
+	/* Clear chip interrupts */
+	ATA_OUTL(ctlr->r_mem, CHIP_SOC_MIC, 0);
+	/* Configure per-HC CCC */
+	if (ccc && bootverbose) {
+		device_printf(dev,
+		    "CCC with %dus/%dcmd enabled\n",
+		    ctlr->ccc, ctlr->cccc);
+	}
+	ccc *= 150;
+	ATA_OUTL(ctlr->r_mem, HC_ICT, cccc);
+	ATA_OUTL(ctlr->r_mem, HC_ITT, ccc);
+	if (ccc)
+		ccim |= IC_HC0_COAL_DONE;
+	/* Enable chip interrupts */
+	ctlr->gmim = (ccc ? IC_HC0_COAL_DONE : IC_DONE_HC0) | IC_ERR_HC0;
+	ATA_OUTL(ctlr->r_mem, CHIP_SOC_MIM, ctlr->gmim | ctlr->pmim);
+	return (0);
+}
+
+static void
+mvs_edma(device_t dev, device_t child, int mode)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int unit = ((struct mvs_channel *)device_get_softc(child))->unit;
+	int bit = IC_DONE_IRQ << (unit * 2);
+
+	if (ctlr->ccc == 0)
+		return;
+	/* CCC is not working for non-EDMA mode. Unmask device interrupts. */
+	mtx_lock(&ctlr->mtx);
+	if (mode == MVS_EDMA_OFF)
+		ctlr->pmim |= bit;
+	else
+		ctlr->pmim &= ~bit;
+	ATA_OUTL(ctlr->r_mem, CHIP_SOC_MIM, ctlr->gmim | ctlr->pmim);
+	mtx_unlock(&ctlr->mtx);
+}
+
+static int
+mvs_suspend(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+
+	bus_generic_suspend(dev);
+	/* Mask chip interrupts */
+	ATA_OUTL(ctlr->r_mem, CHIP_SOC_MIM, 0x00000000);
+	return 0;
+}
+
+static int
+mvs_resume(device_t dev)
+{
+
+	mvs_ctlr_setup(dev);
+	return (bus_generic_resume(dev));
+}
+
+static int
+mvs_setup_interrupt(device_t dev)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+
+	/* Allocate all IRQs. */
+	ctlr->irq.r_irq_rid = 0;
+	if (!(ctlr->irq.r_irq = bus_alloc_resource_any(dev, SYS_RES_IRQ,
+	    &ctlr->irq.r_irq_rid, RF_SHAREABLE | RF_ACTIVE))) {
+		device_printf(dev, "unable to map interrupt\n");
+		return (ENXIO);
+	}
+	if ((bus_setup_intr(dev, ctlr->irq.r_irq, ATA_INTR_FLAGS, NULL,
+	    mvs_intr, ctlr, &ctlr->irq.handle))) {
+		device_printf(dev, "unable to setup interrupt\n");
+		bus_release_resource(dev, SYS_RES_IRQ,
+		    ctlr->irq.r_irq_rid, ctlr->irq.r_irq);
+		ctlr->irq.r_irq = 0;
+		return (ENXIO);
+	}
+	return (0);
+}
+
+/*
+ * Common case interrupt handler.
+ */
+static void
+mvs_intr(void *data)
+{
+	struct mvs_controller *ctlr = data;
+	struct mvs_intr_arg arg;
+	void (*function)(void *);
+	int p;
+	u_int32_t ic, aic;
+
+	ic = ATA_INL(ctlr->r_mem, CHIP_SOC_MIC);
+//device_printf(ctlr->dev, "irq MIC:%08x\n", ic);
+	if ((ic & IC_HC0) == 0)
+		return;
+	/* Acknowledge interrupts of this HC. */
+	aic = 0;
+	if (ic & (IC_DONE_IRQ << 0))
+		aic |= HC_IC_DONE(0) | HC_IC_DEV(0);
+	if (ic & (IC_DONE_IRQ << 2))
+		aic |= HC_IC_DONE(1) | HC_IC_DEV(1);
+	if (ic & (IC_DONE_IRQ << 4))
+		aic |= HC_IC_DONE(2) | HC_IC_DEV(2);
+	if (ic & (IC_DONE_IRQ << 6))
+		aic |= HC_IC_DONE(3) | HC_IC_DEV(3);
+	if (ic & IC_HC0_COAL_DONE)
+		aic |= HC_IC_COAL;
+	ATA_OUTL(ctlr->r_mem, HC_IC, ~aic);
+	/* Call per-port interrupt handler. */
+	for (p = 0; p < ctlr->channels; p++) {
+		arg.cause = ic & (IC_ERR_IRQ|IC_DONE_IRQ);
+		if ((arg.cause != 0) &&
+		    (function = ctlr->interrupt[p].function)) {
+			arg.arg = ctlr->interrupt[p].argument;
+			function(&arg);
+		}
+		ic >>= 2;
+	}
+}
+
+static struct resource *
+mvs_alloc_resource(device_t dev, device_t child, int type, int *rid,
+		       u_long start, u_long end, u_long count, u_int flags)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int unit = ((struct mvs_channel *)device_get_softc(child))->unit;
+	struct resource *res = NULL;
+	int offset = PORT_BASE(unit & 0x03);
+	long st;
+
+	switch (type) {
+	case SYS_RES_MEMORY:
+		st = rman_get_start(ctlr->r_mem);
+		res = rman_reserve_resource(&ctlr->sc_iomem, st + offset,
+		    st + offset + PORT_SIZE - 1, PORT_SIZE, RF_ACTIVE, child);
+		if (res) {
+			bus_space_handle_t bsh;
+			bus_space_tag_t bst;
+			bsh = rman_get_bushandle(ctlr->r_mem);
+			bst = rman_get_bustag(ctlr->r_mem);
+			bus_space_subregion(bst, bsh, offset, PORT_SIZE, &bsh);
+			rman_set_bushandle(res, bsh);
+			rman_set_bustag(res, bst);
+		}
+		break;
+	case SYS_RES_IRQ:
+		if (*rid == ATA_IRQ_RID)
+			res = ctlr->irq.r_irq;
+		break;
+	}
+	return (res);
+}
+
+static int
+mvs_release_resource(device_t dev, device_t child, int type, int rid,
+			 struct resource *r)
+{
+
+	switch (type) {
+	case SYS_RES_MEMORY:
+		rman_release_resource(r);
+		return (0);
+	case SYS_RES_IRQ:
+		if (rid != ATA_IRQ_RID)
+			return ENOENT;
+		return (0);
+	}
+	return (EINVAL);
+}
+
+static int
+mvs_setup_intr(device_t dev, device_t child, struct resource *irq, 
+		   int flags, driver_filter_t *filter, driver_intr_t *function, 
+		   void *argument, void **cookiep)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int unit = (intptr_t)device_get_ivars(child);
+
+	if (filter != NULL) {
+		printf("mvs.c: we cannot use a filter here\n");
+		return (EINVAL);
+	}
+	ctlr->interrupt[unit].function = function;
+	ctlr->interrupt[unit].argument = argument;
+	return (0);
+}
+
+static int
+mvs_teardown_intr(device_t dev, device_t child, struct resource *irq,
+		      void *cookie)
+{
+	struct mvs_controller *ctlr = device_get_softc(dev);
+	int unit = (intptr_t)device_get_ivars(child);
+
+	ctlr->interrupt[unit].function = NULL;
+	ctlr->interrupt[unit].argument = NULL;
+	return (0);
+}
+
+static int
+mvs_print_child(device_t dev, device_t child)
+{
+	int retval;
+
+	retval = bus_print_child_header(dev, child);
+	retval += printf(" at channel %d",
+	    (int)(intptr_t)device_get_ivars(child));
+	retval += bus_print_child_footer(dev, child);
+
+	return (retval);
+}
+
+static device_method_t mvs_methods[] = {
+	DEVMETHOD(device_probe,     mvs_probe),
+	DEVMETHOD(device_attach,    mvs_attach),
+	DEVMETHOD(device_detach,    mvs_detach),
+	DEVMETHOD(device_suspend,   mvs_suspend),
+	DEVMETHOD(device_resume,    mvs_resume),
+	DEVMETHOD(bus_print_child,  mvs_print_child),
+	DEVMETHOD(bus_alloc_resource,       mvs_alloc_resource),
+	DEVMETHOD(bus_release_resource,     mvs_release_resource),
+	DEVMETHOD(bus_setup_intr,   mvs_setup_intr),
+	DEVMETHOD(bus_teardown_intr,mvs_teardown_intr),
+	DEVMETHOD(mvs_edma,         mvs_edma),
+	{ 0, 0 }
+};
+static driver_t mvs_driver = {
+        "sata",
+        mvs_methods,
+        sizeof(struct mvs_controller)
+};
+DRIVER_MODULE(sata, mbus, mvs_driver, mvs_devclass, 0, 0);
+MODULE_VERSION(sata, 1);
+