Add unmapped I/O support to ata(4) driver.

Main problem there was PIO mode support, that required KVA mapping.
Handle that case using recently added pmap_quick_enter_page(9) KPI,
mapping data pages to KVA one at a time.

2 files changed, 168 insertions, 71 deletions

diff --git a/sys/dev/ata/ata-all.c b/sys/dev/ata/ata-all.c
index 19d6eb2..6e3f5fe 100644
--- a/sys/dev/ata/ata-all.c
+++ b/sys/dev/ata/ata-all.c
@@ -1074,7 +1074,7 @@ ataaction(struct cam_sim *sim, union ccb *ccb)
 		cpi->version_num = 1; /* XXX??? */
 		cpi->hba_inquiry = PI_SDTR_ABLE;
 		cpi->target_sprt = 0;
-		cpi->hba_misc = PIM_SEQSCAN;
+		cpi->hba_misc = PIM_SEQSCAN | PIM_UNMAPPED;
 		cpi->hba_eng_cnt = 0;
 		if (ch->flags & ATA_NO_SLAVE)
 			cpi->max_target = 0;
diff --git a/sys/dev/ata/ata-lowlevel.c b/sys/dev/ata/ata-lowlevel.c
index 459f98c..6b825c0 100644
--- a/sys/dev/ata/ata-lowlevel.c
+++ b/sys/dev/ata/ata-lowlevel.c
@@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/endian.h>
 #include <sys/ata.h>
+#include <sys/bio.h>
 #include <sys/conf.h>
 #include <sys/ctype.h>
 #include <sys/bus.h>
@@ -44,6 +45,12 @@ __FBSDID("$FreeBSD$");
 #include <dev/ata/ata-pci.h>
 #include <ata_if.h>
 
+#include <vm/vm.h>
+#include <vm/pmap.h>
+
+#include <cam/cam.h>
+#include <cam/cam_ccb.h>
+
 /* prototypes */
 static int ata_generic_status(device_t dev);
 static int ata_wait(struct ata_channel *ch, int unit, u_int8_t);
@@ -811,86 +818,176 @@ ata_tf_write(struct ata_request *request)
 static void
 ata_pio_read(struct ata_request *request, int length)
 {
-    struct ata_channel *ch = device_get_softc(request->parent);
-    uint8_t *addr;
-    int size = min(request->transfersize, length);
-    int resid;
-    uint8_t buf[2] __aligned(sizeof(int16_t));
-#ifndef __NO_STRICT_ALIGNMENT
-    int i;
-#endif
+	struct ata_channel *ch = device_get_softc(request->parent);
+	struct bio *bio;
+	uint8_t *addr;
+	vm_offset_t page;
+	int todo, done, off, moff, resid, size, i;
+	uint8_t buf[2] __aligned(2);
+
+	todo = min(request->transfersize, length);
+	page = done = resid = 0;
+	while (done < todo) {
+		size = todo - done;
+
+		/* Prepare data address and limit size (if not sequential). */
+		off = request->donecount + done;
+		if ((request->flags & ATA_R_DATA_IN_CCB) == 0 ||
+		    (request->ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) {
+			addr = (uint8_t *)request->data + off;
+		} else if ((request->ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO) {
+			bio = (struct bio *)request->data;
+			if ((bio->bio_flags & BIO_UNMAPPED) == 0) {
+				addr = (uint8_t *)bio->bio_data + off;
+			} else {
+				moff = bio->bio_ma_offset + off;
+				page = pmap_quick_enter_page(
+				    bio->bio_ma[moff / PAGE_SIZE]);
+				moff %= PAGE_SIZE;
+				size = min(size, PAGE_SIZE - moff);
+				addr = (void *)(page + moff);
+			}
+		} else
+			panic("ata_pio_read: Unsupported CAM data type %x\n",
+			    (request->ccb->ccb_h.flags & CAM_DATA_MASK));
+
+		/* We may have extra byte already red but not stored. */
+		if (resid) {
+			addr[0] = buf[1];
+			addr++;
+			done++;
+			size--;
+		}
 
-    addr = (uint8_t *)request->data + request->donecount;
-    if (__predict_false(ch->flags & ATA_USE_16BIT ||
-      (size % sizeof(int32_t)) || ((uintptr_t)addr % sizeof(int32_t)))) {
+		/* Process main part of data. */
+		resid = size % 2;
+		if (__predict_false((ch->flags & ATA_USE_16BIT) ||
+		    (size % 4) != 0 || ((uintptr_t)addr % 4) != 0)) {
 #ifndef __NO_STRICT_ALIGNMENT
-	if (__predict_false((uintptr_t)addr % sizeof(int16_t))) {
-	    for (i = 0, resid = size & ~1; resid > 0; resid -=
-	      sizeof(int16_t)) {
-		*(uint16_t *)&buf = ATA_IDX_INW_STRM(ch, ATA_DATA);
-	        addr[i++] = buf[0];
-	        addr[i++] = buf[1];
-	    }
-	} else
+			if (__predict_false((uintptr_t)addr % 2)) {
+				for (i = 0; i + 1 < size; i += 2) {
+					*(uint16_t *)&buf =
+					    ATA_IDX_INW_STRM(ch, ATA_DATA);
+					addr[i] = buf[0];
+					addr[i + 1] = buf[1];
+				}
+			} else
 #endif
-	    ATA_IDX_INSW_STRM(ch, ATA_DATA, (void*)addr, size /
-	      sizeof(int16_t));
-	if (size & 1) {
-	    *(uint16_t *)&buf = ATA_IDX_INW_STRM(ch, ATA_DATA);
-	    (addr + (size & ~1))[0] = buf[0];
-	}
-    } else
-	ATA_IDX_INSL_STRM(ch, ATA_DATA, (void*)addr, size / sizeof(int32_t));
-
-    if (request->transfersize < length) {
-	device_printf(request->parent, "WARNING - %s read data overrun %d>%d\n",
-		   ata_cmd2str(request), length, request->transfersize);
-	for (resid = request->transfersize + (size & 1); resid < length;
-	     resid += sizeof(int16_t))
-	    ATA_IDX_INW(ch, ATA_DATA);
-    }
+				ATA_IDX_INSW_STRM(ch, ATA_DATA, (void*)addr,
+				    size / 2);
+
+			/* If we have extra byte of data, leave it for later. */
+			if (resid) {
+				*(uint16_t *)&buf =
+				    ATA_IDX_INW_STRM(ch, ATA_DATA);
+				addr[size - 1] = buf[0];
+			}
+		} else
+			ATA_IDX_INSL_STRM(ch, ATA_DATA, (void*)addr, size / 4);
+
+		if (page) {
+			pmap_quick_remove_page(page);
+			page = 0;
+		}
+		done += size;
+	}
+
+	if (length > done) {
+		device_printf(request->parent,
+		    "WARNING - %s read data overrun %d > %d\n",
+		    ata_cmd2str(request), length, done);
+		for (i = done + resid; i < length; i += 2)
+			ATA_IDX_INW(ch, ATA_DATA);
+	}
 }
 
 static void
 ata_pio_write(struct ata_request *request, int length)
 {
-    struct ata_channel *ch = device_get_softc(request->parent);
-    uint8_t *addr;
-    int size = min(request->transfersize, length);
-    int resid;
-    uint8_t buf[2] __aligned(sizeof(int16_t));
+	struct ata_channel *ch = device_get_softc(request->parent);
+	struct bio *bio;
+	uint8_t *addr;
+	vm_offset_t page;
+	int todo, done, off, moff, resid, size, i;
+	uint8_t buf[2] __aligned(2);
+
+	todo = min(request->transfersize, length);
+	page = done = resid = 0;
+	while (done < todo) {
+		size = todo - done;
+
+		/* Prepare data address and limit size (if not sequential). */
+		off = request->donecount + done;
+		if ((request->flags & ATA_R_DATA_IN_CCB) == 0 ||
+		    (request->ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_VADDR) {
+			addr = (uint8_t *)request->data + off;
+		} else if ((request->ccb->ccb_h.flags & CAM_DATA_MASK) == CAM_DATA_BIO) {
+			bio = (struct bio *)request->data;
+			if ((bio->bio_flags & BIO_UNMAPPED) == 0) {
+				addr = (uint8_t *)bio->bio_data + off;
+			} else {
+				moff = bio->bio_ma_offset + off;
+				page = pmap_quick_enter_page(
+				    bio->bio_ma[moff / PAGE_SIZE]);
+				moff %= PAGE_SIZE;
+				size = min(size, PAGE_SIZE - moff);
+				addr = (void *)(page + moff);
+			}
+		} else
+			panic("ata_pio_write: Unsupported CAM data type %x\n",
+			    (request->ccb->ccb_h.flags & CAM_DATA_MASK));
+
+		/* We may have extra byte to be written first. */
+		if (resid) {
+			buf[1] = addr[0];
+			ATA_IDX_OUTW_STRM(ch, ATA_DATA, *(uint16_t *)&buf);
+			addr++;
+			done++;
+			size--;
+		}
+
+		/* Process main part of data. */
+		resid = size % 2;
+		if (__predict_false((ch->flags & ATA_USE_16BIT) ||
+		    (size % 4) != 0 || ((uintptr_t)addr % 4) != 0)) {
 #ifndef __NO_STRICT_ALIGNMENT
-    int i;
+			if (__predict_false((uintptr_t)addr % 2)) {
+				for (i = 0; i + 1 < size; i += 2) {
+					buf[0] = addr[i];
+					buf[1] = addr[i + 1];
+					ATA_IDX_OUTW_STRM(ch, ATA_DATA,
+					    *(uint16_t *)&buf);
+				}
+			} else
 #endif
+				ATA_IDX_OUTSW_STRM(ch, ATA_DATA, (void*)addr,
+				    size / 2);
+
+			/* If we have extra byte of data, save it for later. */
+			if (resid)
+				buf[0] = addr[size - 1];
+		} else
+			ATA_IDX_OUTSL_STRM(ch, ATA_DATA,
+			    (void*)addr, size / sizeof(int32_t));
+
+		if (page) {
+			pmap_quick_remove_page(page);
+			page = 0;
+		}
+		done += size;
+	}
 
-    size = min(request->transfersize, length);
-    addr = (uint8_t *)request->data + request->donecount;
-    if (__predict_false(ch->flags & ATA_USE_16BIT ||
-      (size % sizeof(int32_t)) || ((uintptr_t)addr % sizeof(int32_t)))) {
-#ifndef __NO_STRICT_ALIGNMENT
-	if (__predict_false((uintptr_t)addr % sizeof(int16_t))) {
-	    for (i = 0, resid = size & ~1; resid > 0; resid -=
-	      sizeof(int16_t)) {
-	        buf[0] = addr[i++];
-	        buf[1] = addr[i++];
+	/* We may have extra byte of data to be written. Pad it with zero. */
+	if (resid) {
+		buf[1] = 0;
 		ATA_IDX_OUTW_STRM(ch, ATA_DATA, *(uint16_t *)&buf);
-	    }
-	} else
-#endif
-	    ATA_IDX_OUTSW_STRM(ch, ATA_DATA, (void*)addr, size /
-	      sizeof(int16_t));
-	if (size & 1) {
-	    buf[0] = (addr + (size & ~1))[0];
-	    ATA_IDX_OUTW_STRM(ch, ATA_DATA, *(uint16_t *)&buf);
-	}
-    } else
-	ATA_IDX_OUTSL_STRM(ch, ATA_DATA, (void*)addr, size / sizeof(int32_t));
-
-    if (request->transfersize < length) {
-	device_printf(request->parent, "WARNING - %s write data underrun %d>%d\n",
-		   ata_cmd2str(request), length, request->transfersize);
-	for (resid = request->transfersize + (size & 1); resid < length;
-	     resid += sizeof(int16_t))
-	    ATA_IDX_OUTW(ch, ATA_DATA, 0);
-    }
+	}
+
+	if (length > done) {
+		device_printf(request->parent,
+		    "WARNING - %s write data underrun %d > %d\n",
+		    ata_cmd2str(request), length, done);
+		for (i = done + resid; i < length; i += 2)
+			ATA_IDX_OUTW(ch, ATA_DATA, 0);
+	}
 }