summaryrefslogtreecommitdiffstats
path: root/sys/dev/md
diff options
context:
space:
mode:
Diffstat (limited to 'sys/dev/md')
-rw-r--r--sys/dev/md/md.c307
1 files changed, 236 insertions, 71 deletions
diff --git a/sys/dev/md/md.c b/sys/dev/md/md.c
index dccd5b3..27ef8b3 100644
--- a/sys/dev/md/md.c
+++ b/sys/dev/md/md.c
@@ -99,6 +99,8 @@
#include <vm/swap_pager.h>
#include <vm/uma.h>
+#include <machine/bus.h>
+
#define MD_MODVER 1
#define MD_SHUTDOWN 0x10000 /* Tell worker thread to terminate. */
@@ -435,7 +437,7 @@ g_md_start(struct bio *bp)
#define MD_MALLOC_MOVE_CMP 5
static int
-md_malloc_move(vm_page_t **mp, int *ma_offs, unsigned sectorsize,
+md_malloc_move_ma(vm_page_t **mp, int *ma_offs, unsigned sectorsize,
void *ptr, u_char fill, int op)
{
struct sf_buf *sf;
@@ -497,7 +499,7 @@ md_malloc_move(vm_page_t **mp, int *ma_offs, unsigned sectorsize,
}
break;
default:
- KASSERT(0, ("md_malloc_move unknown op %d\n", op));
+ KASSERT(0, ("md_malloc_move_ma unknown op %d\n", op));
break;
}
if (error != 0)
@@ -520,10 +522,68 @@ md_malloc_move(vm_page_t **mp, int *ma_offs, unsigned sectorsize,
}
static int
+md_malloc_move_vlist(bus_dma_segment_t **pvlist, int *pma_offs,
+ unsigned len, void *ptr, u_char fill, int op)
+{
+ bus_dma_segment_t *vlist;
+ uint8_t *p, *end, first;
+ off_t *uc;
+ int ma_offs, seg_len;
+
+ vlist = *pvlist;
+ ma_offs = *pma_offs;
+ uc = ptr;
+
+ for (; len != 0; len -= seg_len) {
+ seg_len = imin(vlist->ds_len - ma_offs, len);
+ p = (uint8_t *)(uintptr_t)vlist->ds_addr + ma_offs;
+ switch (op) {
+ case MD_MALLOC_MOVE_ZERO:
+ bzero(p, seg_len);
+ break;
+ case MD_MALLOC_MOVE_FILL:
+ memset(p, fill, seg_len);
+ break;
+ case MD_MALLOC_MOVE_READ:
+ bcopy(ptr, p, seg_len);
+ cpu_flush_dcache(p, seg_len);
+ break;
+ case MD_MALLOC_MOVE_WRITE:
+ bcopy(p, ptr, seg_len);
+ break;
+ case MD_MALLOC_MOVE_CMP:
+ end = p + seg_len;
+ first = *uc = *p;
+ /* Confirm all following bytes match the first */
+ while (++p < end) {
+ if (*p != first)
+ return (EDOOFUS);
+ }
+ break;
+ default:
+ KASSERT(0, ("md_malloc_move_vlist unknown op %d\n", op));
+ break;
+ }
+
+ ma_offs += seg_len;
+ if (ma_offs == vlist->ds_len) {
+ ma_offs = 0;
+ vlist++;
+ }
+ ptr = (uint8_t *)ptr + seg_len;
+ }
+ *pvlist = vlist;
+ *pma_offs = ma_offs;
+
+ return (0);
+}
+
+static int
mdstart_malloc(struct md_s *sc, struct bio *bp)
{
u_char *dst;
vm_page_t *m;
+ bus_dma_segment_t *vlist;
int i, error, error1, ma_offs, notmapped;
off_t secno, nsec, uc;
uintptr_t sp, osp;
@@ -538,10 +598,16 @@ mdstart_malloc(struct md_s *sc, struct bio *bp)
}
notmapped = (bp->bio_flags & BIO_UNMAPPED) != 0;
+ vlist = (bp->bio_flags & BIO_VLIST) != 0 ?
+ (bus_dma_segment_t *)bp->bio_data : NULL;
if (notmapped) {
m = bp->bio_ma;
ma_offs = bp->bio_ma_offset;
dst = NULL;
+ KASSERT(vlist == NULL, ("vlists cannot be unmapped"));
+ } else if (vlist != NULL) {
+ ma_offs = bp->bio_ma_offset;
+ dst = NULL;
} else {
dst = bp->bio_data;
}
@@ -557,23 +623,36 @@ mdstart_malloc(struct md_s *sc, struct bio *bp)
} else if (bp->bio_cmd == BIO_READ) {
if (osp == 0) {
if (notmapped) {
- error = md_malloc_move(&m, &ma_offs,
+ error = md_malloc_move_ma(&m, &ma_offs,
sc->sectorsize, NULL, 0,
MD_MALLOC_MOVE_ZERO);
+ } else if (vlist != NULL) {
+ error = md_malloc_move_vlist(&vlist,
+ &ma_offs, sc->sectorsize, NULL, 0,
+ MD_MALLOC_MOVE_ZERO);
} else
bzero(dst, sc->sectorsize);
} else if (osp <= 255) {
if (notmapped) {
- error = md_malloc_move(&m, &ma_offs,
+ error = md_malloc_move_ma(&m, &ma_offs,
sc->sectorsize, NULL, osp,
MD_MALLOC_MOVE_FILL);
+ } else if (vlist != NULL) {
+ error = md_malloc_move_vlist(&vlist,
+ &ma_offs, sc->sectorsize, NULL, osp,
+ MD_MALLOC_MOVE_FILL);
} else
memset(dst, osp, sc->sectorsize);
} else {
if (notmapped) {
- error = md_malloc_move(&m, &ma_offs,
+ error = md_malloc_move_ma(&m, &ma_offs,
sc->sectorsize, (void *)osp, 0,
MD_MALLOC_MOVE_READ);
+ } else if (vlist != NULL) {
+ error = md_malloc_move_vlist(&vlist,
+ &ma_offs, sc->sectorsize,
+ (void *)osp, 0,
+ MD_MALLOC_MOVE_READ);
} else {
bcopy((void *)osp, dst, sc->sectorsize);
cpu_flush_dcache(dst, sc->sectorsize);
@@ -583,10 +662,15 @@ mdstart_malloc(struct md_s *sc, struct bio *bp)
} else if (bp->bio_cmd == BIO_WRITE) {
if (sc->flags & MD_COMPRESS) {
if (notmapped) {
- error1 = md_malloc_move(&m, &ma_offs,
+ error1 = md_malloc_move_ma(&m, &ma_offs,
sc->sectorsize, &uc, 0,
MD_MALLOC_MOVE_CMP);
i = error1 == 0 ? sc->sectorsize : 0;
+ } else if (vlist != NULL) {
+ error1 = md_malloc_move_vlist(&vlist,
+ &ma_offs, sc->sectorsize, &uc, 0,
+ MD_MALLOC_MOVE_CMP);
+ i = error1 == 0 ? sc->sectorsize : 0;
} else {
uc = dst[0];
for (i = 1; i < sc->sectorsize; i++) {
@@ -611,10 +695,15 @@ mdstart_malloc(struct md_s *sc, struct bio *bp)
break;
}
if (notmapped) {
- error = md_malloc_move(&m,
+ error = md_malloc_move_ma(&m,
&ma_offs, sc->sectorsize,
(void *)sp, 0,
MD_MALLOC_MOVE_WRITE);
+ } else if (vlist != NULL) {
+ error = md_malloc_move_vlist(
+ &vlist, &ma_offs,
+ sc->sectorsize, (void *)sp,
+ 0, MD_MALLOC_MOVE_WRITE);
} else {
bcopy(dst, (void *)sp,
sc->sectorsize);
@@ -622,10 +711,15 @@ mdstart_malloc(struct md_s *sc, struct bio *bp)
error = s_write(sc->indir, secno, sp);
} else {
if (notmapped) {
- error = md_malloc_move(&m,
+ error = md_malloc_move_ma(&m,
&ma_offs, sc->sectorsize,
(void *)osp, 0,
MD_MALLOC_MOVE_WRITE);
+ } else if (vlist != NULL) {
+ error = md_malloc_move_vlist(
+ &vlist, &ma_offs,
+ sc->sectorsize, (void *)osp,
+ 0, MD_MALLOC_MOVE_WRITE);
} else {
bcopy(dst, (void *)osp,
sc->sectorsize);
@@ -641,26 +735,78 @@ mdstart_malloc(struct md_s *sc, struct bio *bp)
if (error != 0)
break;
secno++;
- if (!notmapped)
+ if (!notmapped && vlist == NULL)
dst += sc->sectorsize;
}
bp->bio_resid = 0;
return (error);
}
+static void
+mdcopyto_vlist(void *src, bus_dma_segment_t *vlist, off_t offset, off_t len)
+{
+ off_t seg_len;
+
+ while (offset >= vlist->ds_len) {
+ offset -= vlist->ds_len;
+ vlist++;
+ }
+
+ while (len != 0) {
+ seg_len = omin(len, vlist->ds_len - offset);
+ bcopy(src, (void *)(uintptr_t)(vlist->ds_addr + offset),
+ seg_len);
+ offset = 0;
+ src = (uint8_t *)src + seg_len;
+ len -= seg_len;
+ vlist++;
+ }
+}
+
+static void
+mdcopyfrom_vlist(bus_dma_segment_t *vlist, off_t offset, void *dst, off_t len)
+{
+ off_t seg_len;
+
+ while (offset >= vlist->ds_len) {
+ offset -= vlist->ds_len;
+ vlist++;
+ }
+
+ while (len != 0) {
+ seg_len = omin(len, vlist->ds_len - offset);
+ bcopy((void *)(uintptr_t)(vlist->ds_addr + offset), dst,
+ seg_len);
+ offset = 0;
+ dst = (uint8_t *)dst + seg_len;
+ len -= seg_len;
+ vlist++;
+ }
+}
+
static int
mdstart_preload(struct md_s *sc, struct bio *bp)
{
+ uint8_t *p;
+ p = sc->pl_ptr + bp->bio_offset;
switch (bp->bio_cmd) {
case BIO_READ:
- bcopy(sc->pl_ptr + bp->bio_offset, bp->bio_data,
- bp->bio_length);
+ if ((bp->bio_flags & BIO_VLIST) != 0) {
+ mdcopyto_vlist(p, (bus_dma_segment_t *)bp->bio_data,
+ bp->bio_ma_offset, bp->bio_length);
+ } else {
+ bcopy(p, bp->bio_data, bp->bio_length);
+ }
cpu_flush_dcache(bp->bio_data, bp->bio_length);
break;
case BIO_WRITE:
- bcopy(bp->bio_data, sc->pl_ptr + bp->bio_offset,
- bp->bio_length);
+ if ((bp->bio_flags & BIO_VLIST) != 0) {
+ mdcopyfrom_vlist((bus_dma_segment_t *)bp->bio_data,
+ bp->bio_ma_offset, p, bp->bio_length);
+ } else {
+ bcopy(bp->bio_data, p, bp->bio_length);
+ }
break;
}
bp->bio_resid = 0;
@@ -673,16 +819,23 @@ mdstart_vnode(struct md_s *sc, struct bio *bp)
int error;
struct uio auio;
struct iovec aiov;
+ struct iovec *piov;
struct mount *mp;
struct vnode *vp;
struct buf *pb;
+ bus_dma_segment_t *vlist;
struct thread *td;
- off_t end, zerosize;
+ off_t len, zerosize;
+ int ma_offs;
switch (bp->bio_cmd) {
case BIO_READ:
+ auio.uio_rw = UIO_READ;
+ break;
case BIO_WRITE:
case BIO_DELETE:
+ auio.uio_rw = UIO_WRITE;
+ break;
case BIO_FLUSH:
break;
default:
@@ -691,6 +844,9 @@ mdstart_vnode(struct md_s *sc, struct bio *bp)
td = curthread;
vp = sc->vnode;
+ pb = NULL;
+ piov = NULL;
+ ma_offs = bp->bio_ma_offset;
/*
* VNODE I/O
@@ -709,73 +865,66 @@ mdstart_vnode(struct md_s *sc, struct bio *bp)
return (error);
}
- bzero(&auio, sizeof(auio));
+ auio.uio_offset = (vm_ooffset_t)bp->bio_offset;
+ auio.uio_resid = bp->bio_length;
+ auio.uio_segflg = UIO_SYSSPACE;
+ auio.uio_td = td;
- /*
- * Special case for BIO_DELETE. On the surface, this is very
- * similar to BIO_WRITE, except that we write from our own
- * fixed-length buffer, so we have to loop. The net result is
- * that the two cases end up having very little in common.
- */
if (bp->bio_cmd == BIO_DELETE) {
+ /*
+ * Emulate BIO_DELETE by writing zeros.
+ */
zerosize = ZERO_REGION_SIZE -
(ZERO_REGION_SIZE % sc->sectorsize);
- auio.uio_iov = &aiov;
- auio.uio_iovcnt = 1;
- auio.uio_offset = (vm_ooffset_t)bp->bio_offset;
- auio.uio_segflg = UIO_SYSSPACE;
- auio.uio_rw = UIO_WRITE;
- auio.uio_td = td;
- end = bp->bio_offset + bp->bio_length;
- (void) vn_start_write(vp, &mp, V_WAIT);
- vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
- error = 0;
- while (auio.uio_offset < end) {
- aiov.iov_base = __DECONST(void *, zero_region);
- aiov.iov_len = end - auio.uio_offset;
- if (aiov.iov_len > zerosize)
- aiov.iov_len = zerosize;
- auio.uio_resid = aiov.iov_len;
- error = VOP_WRITE(vp, &auio,
- sc->flags & MD_ASYNC ? 0 : IO_SYNC, sc->cred);
- if (error != 0)
- break;
+ auio.uio_iovcnt = howmany(bp->bio_length, zerosize);
+ piov = malloc(sizeof(*piov) * auio.uio_iovcnt, M_MD, M_WAITOK);
+ auio.uio_iov = piov;
+ len = bp->bio_length;
+ while (len > 0) {
+ piov->iov_base = __DECONST(void *, zero_region);
+ piov->iov_len = len;
+ if (len > zerosize)
+ piov->iov_len = zerosize;
+ len -= piov->iov_len;
+ piov++;
}
- VOP_UNLOCK(vp, 0);
- vn_finished_write(mp);
- bp->bio_resid = end - auio.uio_offset;
- return (error);
- }
-
- KASSERT(bp->bio_length <= MAXPHYS, ("bio_length %jd",
- (uintmax_t)bp->bio_length));
- if ((bp->bio_flags & BIO_UNMAPPED) == 0) {
- pb = NULL;
- aiov.iov_base = bp->bio_data;
- } else {
+ piov = auio.uio_iov;
+ } else if ((bp->bio_flags & BIO_VLIST) != 0) {
+ piov = malloc(sizeof(*piov) * bp->bio_ma_n, M_MD, M_WAITOK);
+ auio.uio_iov = piov;
+ vlist = (bus_dma_segment_t *)bp->bio_data;
+ len = bp->bio_length;
+ while (len > 0) {
+ piov->iov_base = (void *)(uintptr_t)(vlist->ds_addr +
+ ma_offs);
+ piov->iov_len = vlist->ds_len - ma_offs;
+ if (piov->iov_len > len)
+ piov->iov_len = len;
+ len -= piov->iov_len;
+ ma_offs = 0;
+ vlist++;
+ piov++;
+ }
+ auio.uio_iovcnt = piov - auio.uio_iov;
+ piov = auio.uio_iov;
+ } else if ((bp->bio_flags & BIO_UNMAPPED) != 0) {
pb = getpbuf(&md_vnode_pbuf_freecnt);
pmap_qenter((vm_offset_t)pb->b_data, bp->bio_ma, bp->bio_ma_n);
- aiov.iov_base = (void *)((vm_offset_t)pb->b_data +
- bp->bio_ma_offset);
+ aiov.iov_base = (void *)((vm_offset_t)pb->b_data + ma_offs);
+ aiov.iov_len = bp->bio_length;
+ auio.uio_iov = &aiov;
+ auio.uio_iovcnt = 1;
+ } else {
+ aiov.iov_base = bp->bio_data;
+ aiov.iov_len = bp->bio_length;
+ auio.uio_iov = &aiov;
+ auio.uio_iovcnt = 1;
}
- aiov.iov_len = bp->bio_length;
- auio.uio_iov = &aiov;
- auio.uio_iovcnt = 1;
- auio.uio_offset = (vm_ooffset_t)bp->bio_offset;
- auio.uio_segflg = UIO_SYSSPACE;
- if (bp->bio_cmd == BIO_READ)
- auio.uio_rw = UIO_READ;
- else if (bp->bio_cmd == BIO_WRITE)
- auio.uio_rw = UIO_WRITE;
- else
- panic("wrong BIO_OP in mdstart_vnode");
- auio.uio_resid = bp->bio_length;
- auio.uio_td = td;
/*
* When reading set IO_DIRECT to try to avoid double-caching
* the data. When writing IO_DIRECT is not optimal.
*/
- if (bp->bio_cmd == BIO_READ) {
+ if (auio.uio_rw == UIO_READ) {
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred);
VOP_UNLOCK(vp, 0);
@@ -787,10 +936,15 @@ mdstart_vnode(struct md_s *sc, struct bio *bp)
VOP_UNLOCK(vp, 0);
vn_finished_write(mp);
}
- if ((bp->bio_flags & BIO_UNMAPPED) != 0) {
+
+ if (pb) {
pmap_qremove((vm_offset_t)pb->b_data, bp->bio_ma_n);
relpbuf(pb, &md_vnode_pbuf_freecnt);
}
+
+ if (piov != NULL)
+ free(piov, M_MD);
+
bp->bio_resid = auio.uio_resid;
return (error);
}
@@ -801,6 +955,7 @@ mdstart_swap(struct md_s *sc, struct bio *bp)
vm_page_t m;
u_char *p;
vm_pindex_t i, lastp;
+ bus_dma_segment_t *vlist;
int rv, ma_offs, offs, len, lastend;
switch (bp->bio_cmd) {
@@ -813,7 +968,10 @@ mdstart_swap(struct md_s *sc, struct bio *bp)
}
p = bp->bio_data;
- ma_offs = (bp->bio_flags & BIO_UNMAPPED) == 0 ? 0 : bp->bio_ma_offset;
+ ma_offs = (bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0 ?
+ bp->bio_ma_offset : 0;
+ vlist = (bp->bio_flags & BIO_VLIST) != 0 ?
+ (bus_dma_segment_t *)bp->bio_data : NULL;
/*
* offs is the offset at which to start operating on the
@@ -853,6 +1011,10 @@ mdstart_swap(struct md_s *sc, struct bio *bp)
if ((bp->bio_flags & BIO_UNMAPPED) != 0) {
pmap_copy_pages(&m, offs, bp->bio_ma,
ma_offs, len);
+ } else if ((bp->bio_flags & BIO_VLIST) != 0) {
+ physcopyout_vlist(VM_PAGE_TO_PHYS(m) + offs,
+ vlist, ma_offs, len);
+ cpu_flush_dcache(p, len);
} else {
physcopyout(VM_PAGE_TO_PHYS(m) + offs, p, len);
cpu_flush_dcache(p, len);
@@ -869,6 +1031,9 @@ mdstart_swap(struct md_s *sc, struct bio *bp)
if ((bp->bio_flags & BIO_UNMAPPED) != 0) {
pmap_copy_pages(bp->bio_ma, ma_offs, &m,
offs, len);
+ } else if ((bp->bio_flags & BIO_VLIST) != 0) {
+ physcopyin_vlist(vlist, ma_offs,
+ VM_PAGE_TO_PHYS(m) + offs, len);
} else {
physcopyin(p, VM_PAGE_TO_PHYS(m) + offs, len);
}
OpenPOWER on IntegriCloud