diff options
Diffstat (limited to 'sys/dev/md')
-rw-r--r-- | sys/dev/md/md.c | 307 |
1 files changed, 236 insertions, 71 deletions
diff --git a/sys/dev/md/md.c b/sys/dev/md/md.c index dccd5b3..27ef8b3 100644 --- a/sys/dev/md/md.c +++ b/sys/dev/md/md.c @@ -99,6 +99,8 @@ #include <vm/swap_pager.h> #include <vm/uma.h> +#include <machine/bus.h> + #define MD_MODVER 1 #define MD_SHUTDOWN 0x10000 /* Tell worker thread to terminate. */ @@ -435,7 +437,7 @@ g_md_start(struct bio *bp) #define MD_MALLOC_MOVE_CMP 5 static int -md_malloc_move(vm_page_t **mp, int *ma_offs, unsigned sectorsize, +md_malloc_move_ma(vm_page_t **mp, int *ma_offs, unsigned sectorsize, void *ptr, u_char fill, int op) { struct sf_buf *sf; @@ -497,7 +499,7 @@ md_malloc_move(vm_page_t **mp, int *ma_offs, unsigned sectorsize, } break; default: - KASSERT(0, ("md_malloc_move unknown op %d\n", op)); + KASSERT(0, ("md_malloc_move_ma unknown op %d\n", op)); break; } if (error != 0) @@ -520,10 +522,68 @@ md_malloc_move(vm_page_t **mp, int *ma_offs, unsigned sectorsize, } static int +md_malloc_move_vlist(bus_dma_segment_t **pvlist, int *pma_offs, + unsigned len, void *ptr, u_char fill, int op) +{ + bus_dma_segment_t *vlist; + uint8_t *p, *end, first; + off_t *uc; + int ma_offs, seg_len; + + vlist = *pvlist; + ma_offs = *pma_offs; + uc = ptr; + + for (; len != 0; len -= seg_len) { + seg_len = imin(vlist->ds_len - ma_offs, len); + p = (uint8_t *)(uintptr_t)vlist->ds_addr + ma_offs; + switch (op) { + case MD_MALLOC_MOVE_ZERO: + bzero(p, seg_len); + break; + case MD_MALLOC_MOVE_FILL: + memset(p, fill, seg_len); + break; + case MD_MALLOC_MOVE_READ: + bcopy(ptr, p, seg_len); + cpu_flush_dcache(p, seg_len); + break; + case MD_MALLOC_MOVE_WRITE: + bcopy(p, ptr, seg_len); + break; + case MD_MALLOC_MOVE_CMP: + end = p + seg_len; + first = *uc = *p; + /* Confirm all following bytes match the first */ + while (++p < end) { + if (*p != first) + return (EDOOFUS); + } + break; + default: + KASSERT(0, ("md_malloc_move_vlist unknown op %d\n", op)); + break; + } + + ma_offs += seg_len; + if (ma_offs == vlist->ds_len) { + ma_offs = 0; + vlist++; + } + ptr = (uint8_t *)ptr + seg_len; + } + *pvlist = vlist; + *pma_offs = ma_offs; + + return (0); +} + +static int mdstart_malloc(struct md_s *sc, struct bio *bp) { u_char *dst; vm_page_t *m; + bus_dma_segment_t *vlist; int i, error, error1, ma_offs, notmapped; off_t secno, nsec, uc; uintptr_t sp, osp; @@ -538,10 +598,16 @@ mdstart_malloc(struct md_s *sc, struct bio *bp) } notmapped = (bp->bio_flags & BIO_UNMAPPED) != 0; + vlist = (bp->bio_flags & BIO_VLIST) != 0 ? + (bus_dma_segment_t *)bp->bio_data : NULL; if (notmapped) { m = bp->bio_ma; ma_offs = bp->bio_ma_offset; dst = NULL; + KASSERT(vlist == NULL, ("vlists cannot be unmapped")); + } else if (vlist != NULL) { + ma_offs = bp->bio_ma_offset; + dst = NULL; } else { dst = bp->bio_data; } @@ -557,23 +623,36 @@ mdstart_malloc(struct md_s *sc, struct bio *bp) } else if (bp->bio_cmd == BIO_READ) { if (osp == 0) { if (notmapped) { - error = md_malloc_move(&m, &ma_offs, + error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, NULL, 0, MD_MALLOC_MOVE_ZERO); + } else if (vlist != NULL) { + error = md_malloc_move_vlist(&vlist, + &ma_offs, sc->sectorsize, NULL, 0, + MD_MALLOC_MOVE_ZERO); } else bzero(dst, sc->sectorsize); } else if (osp <= 255) { if (notmapped) { - error = md_malloc_move(&m, &ma_offs, + error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, NULL, osp, MD_MALLOC_MOVE_FILL); + } else if (vlist != NULL) { + error = md_malloc_move_vlist(&vlist, + &ma_offs, sc->sectorsize, NULL, osp, + MD_MALLOC_MOVE_FILL); } else memset(dst, osp, sc->sectorsize); } else { if (notmapped) { - error = md_malloc_move(&m, &ma_offs, + error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_READ); + } else if (vlist != NULL) { + error = md_malloc_move_vlist(&vlist, + &ma_offs, sc->sectorsize, + (void *)osp, 0, + MD_MALLOC_MOVE_READ); } else { bcopy((void *)osp, dst, sc->sectorsize); cpu_flush_dcache(dst, sc->sectorsize); @@ -583,10 +662,15 @@ mdstart_malloc(struct md_s *sc, struct bio *bp) } else if (bp->bio_cmd == BIO_WRITE) { if (sc->flags & MD_COMPRESS) { if (notmapped) { - error1 = md_malloc_move(&m, &ma_offs, + error1 = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, &uc, 0, MD_MALLOC_MOVE_CMP); i = error1 == 0 ? sc->sectorsize : 0; + } else if (vlist != NULL) { + error1 = md_malloc_move_vlist(&vlist, + &ma_offs, sc->sectorsize, &uc, 0, + MD_MALLOC_MOVE_CMP); + i = error1 == 0 ? sc->sectorsize : 0; } else { uc = dst[0]; for (i = 1; i < sc->sectorsize; i++) { @@ -611,10 +695,15 @@ mdstart_malloc(struct md_s *sc, struct bio *bp) break; } if (notmapped) { - error = md_malloc_move(&m, + error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, (void *)sp, 0, MD_MALLOC_MOVE_WRITE); + } else if (vlist != NULL) { + error = md_malloc_move_vlist( + &vlist, &ma_offs, + sc->sectorsize, (void *)sp, + 0, MD_MALLOC_MOVE_WRITE); } else { bcopy(dst, (void *)sp, sc->sectorsize); @@ -622,10 +711,15 @@ mdstart_malloc(struct md_s *sc, struct bio *bp) error = s_write(sc->indir, secno, sp); } else { if (notmapped) { - error = md_malloc_move(&m, + error = md_malloc_move_ma(&m, &ma_offs, sc->sectorsize, (void *)osp, 0, MD_MALLOC_MOVE_WRITE); + } else if (vlist != NULL) { + error = md_malloc_move_vlist( + &vlist, &ma_offs, + sc->sectorsize, (void *)osp, + 0, MD_MALLOC_MOVE_WRITE); } else { bcopy(dst, (void *)osp, sc->sectorsize); @@ -641,26 +735,78 @@ mdstart_malloc(struct md_s *sc, struct bio *bp) if (error != 0) break; secno++; - if (!notmapped) + if (!notmapped && vlist == NULL) dst += sc->sectorsize; } bp->bio_resid = 0; return (error); } +static void +mdcopyto_vlist(void *src, bus_dma_segment_t *vlist, off_t offset, off_t len) +{ + off_t seg_len; + + while (offset >= vlist->ds_len) { + offset -= vlist->ds_len; + vlist++; + } + + while (len != 0) { + seg_len = omin(len, vlist->ds_len - offset); + bcopy(src, (void *)(uintptr_t)(vlist->ds_addr + offset), + seg_len); + offset = 0; + src = (uint8_t *)src + seg_len; + len -= seg_len; + vlist++; + } +} + +static void +mdcopyfrom_vlist(bus_dma_segment_t *vlist, off_t offset, void *dst, off_t len) +{ + off_t seg_len; + + while (offset >= vlist->ds_len) { + offset -= vlist->ds_len; + vlist++; + } + + while (len != 0) { + seg_len = omin(len, vlist->ds_len - offset); + bcopy((void *)(uintptr_t)(vlist->ds_addr + offset), dst, + seg_len); + offset = 0; + dst = (uint8_t *)dst + seg_len; + len -= seg_len; + vlist++; + } +} + static int mdstart_preload(struct md_s *sc, struct bio *bp) { + uint8_t *p; + p = sc->pl_ptr + bp->bio_offset; switch (bp->bio_cmd) { case BIO_READ: - bcopy(sc->pl_ptr + bp->bio_offset, bp->bio_data, - bp->bio_length); + if ((bp->bio_flags & BIO_VLIST) != 0) { + mdcopyto_vlist(p, (bus_dma_segment_t *)bp->bio_data, + bp->bio_ma_offset, bp->bio_length); + } else { + bcopy(p, bp->bio_data, bp->bio_length); + } cpu_flush_dcache(bp->bio_data, bp->bio_length); break; case BIO_WRITE: - bcopy(bp->bio_data, sc->pl_ptr + bp->bio_offset, - bp->bio_length); + if ((bp->bio_flags & BIO_VLIST) != 0) { + mdcopyfrom_vlist((bus_dma_segment_t *)bp->bio_data, + bp->bio_ma_offset, p, bp->bio_length); + } else { + bcopy(bp->bio_data, p, bp->bio_length); + } break; } bp->bio_resid = 0; @@ -673,16 +819,23 @@ mdstart_vnode(struct md_s *sc, struct bio *bp) int error; struct uio auio; struct iovec aiov; + struct iovec *piov; struct mount *mp; struct vnode *vp; struct buf *pb; + bus_dma_segment_t *vlist; struct thread *td; - off_t end, zerosize; + off_t len, zerosize; + int ma_offs; switch (bp->bio_cmd) { case BIO_READ: + auio.uio_rw = UIO_READ; + break; case BIO_WRITE: case BIO_DELETE: + auio.uio_rw = UIO_WRITE; + break; case BIO_FLUSH: break; default: @@ -691,6 +844,9 @@ mdstart_vnode(struct md_s *sc, struct bio *bp) td = curthread; vp = sc->vnode; + pb = NULL; + piov = NULL; + ma_offs = bp->bio_ma_offset; /* * VNODE I/O @@ -709,73 +865,66 @@ mdstart_vnode(struct md_s *sc, struct bio *bp) return (error); } - bzero(&auio, sizeof(auio)); + auio.uio_offset = (vm_ooffset_t)bp->bio_offset; + auio.uio_resid = bp->bio_length; + auio.uio_segflg = UIO_SYSSPACE; + auio.uio_td = td; - /* - * Special case for BIO_DELETE. On the surface, this is very - * similar to BIO_WRITE, except that we write from our own - * fixed-length buffer, so we have to loop. The net result is - * that the two cases end up having very little in common. - */ if (bp->bio_cmd == BIO_DELETE) { + /* + * Emulate BIO_DELETE by writing zeros. + */ zerosize = ZERO_REGION_SIZE - (ZERO_REGION_SIZE % sc->sectorsize); - auio.uio_iov = &aiov; - auio.uio_iovcnt = 1; - auio.uio_offset = (vm_ooffset_t)bp->bio_offset; - auio.uio_segflg = UIO_SYSSPACE; - auio.uio_rw = UIO_WRITE; - auio.uio_td = td; - end = bp->bio_offset + bp->bio_length; - (void) vn_start_write(vp, &mp, V_WAIT); - vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); - error = 0; - while (auio.uio_offset < end) { - aiov.iov_base = __DECONST(void *, zero_region); - aiov.iov_len = end - auio.uio_offset; - if (aiov.iov_len > zerosize) - aiov.iov_len = zerosize; - auio.uio_resid = aiov.iov_len; - error = VOP_WRITE(vp, &auio, - sc->flags & MD_ASYNC ? 0 : IO_SYNC, sc->cred); - if (error != 0) - break; + auio.uio_iovcnt = howmany(bp->bio_length, zerosize); + piov = malloc(sizeof(*piov) * auio.uio_iovcnt, M_MD, M_WAITOK); + auio.uio_iov = piov; + len = bp->bio_length; + while (len > 0) { + piov->iov_base = __DECONST(void *, zero_region); + piov->iov_len = len; + if (len > zerosize) + piov->iov_len = zerosize; + len -= piov->iov_len; + piov++; } - VOP_UNLOCK(vp, 0); - vn_finished_write(mp); - bp->bio_resid = end - auio.uio_offset; - return (error); - } - - KASSERT(bp->bio_length <= MAXPHYS, ("bio_length %jd", - (uintmax_t)bp->bio_length)); - if ((bp->bio_flags & BIO_UNMAPPED) == 0) { - pb = NULL; - aiov.iov_base = bp->bio_data; - } else { + piov = auio.uio_iov; + } else if ((bp->bio_flags & BIO_VLIST) != 0) { + piov = malloc(sizeof(*piov) * bp->bio_ma_n, M_MD, M_WAITOK); + auio.uio_iov = piov; + vlist = (bus_dma_segment_t *)bp->bio_data; + len = bp->bio_length; + while (len > 0) { + piov->iov_base = (void *)(uintptr_t)(vlist->ds_addr + + ma_offs); + piov->iov_len = vlist->ds_len - ma_offs; + if (piov->iov_len > len) + piov->iov_len = len; + len -= piov->iov_len; + ma_offs = 0; + vlist++; + piov++; + } + auio.uio_iovcnt = piov - auio.uio_iov; + piov = auio.uio_iov; + } else if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pb = getpbuf(&md_vnode_pbuf_freecnt); pmap_qenter((vm_offset_t)pb->b_data, bp->bio_ma, bp->bio_ma_n); - aiov.iov_base = (void *)((vm_offset_t)pb->b_data + - bp->bio_ma_offset); + aiov.iov_base = (void *)((vm_offset_t)pb->b_data + ma_offs); + aiov.iov_len = bp->bio_length; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; + } else { + aiov.iov_base = bp->bio_data; + aiov.iov_len = bp->bio_length; + auio.uio_iov = &aiov; + auio.uio_iovcnt = 1; } - aiov.iov_len = bp->bio_length; - auio.uio_iov = &aiov; - auio.uio_iovcnt = 1; - auio.uio_offset = (vm_ooffset_t)bp->bio_offset; - auio.uio_segflg = UIO_SYSSPACE; - if (bp->bio_cmd == BIO_READ) - auio.uio_rw = UIO_READ; - else if (bp->bio_cmd == BIO_WRITE) - auio.uio_rw = UIO_WRITE; - else - panic("wrong BIO_OP in mdstart_vnode"); - auio.uio_resid = bp->bio_length; - auio.uio_td = td; /* * When reading set IO_DIRECT to try to avoid double-caching * the data. When writing IO_DIRECT is not optimal. */ - if (bp->bio_cmd == BIO_READ) { + if (auio.uio_rw == UIO_READ) { vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); error = VOP_READ(vp, &auio, IO_DIRECT, sc->cred); VOP_UNLOCK(vp, 0); @@ -787,10 +936,15 @@ mdstart_vnode(struct md_s *sc, struct bio *bp) VOP_UNLOCK(vp, 0); vn_finished_write(mp); } - if ((bp->bio_flags & BIO_UNMAPPED) != 0) { + + if (pb) { pmap_qremove((vm_offset_t)pb->b_data, bp->bio_ma_n); relpbuf(pb, &md_vnode_pbuf_freecnt); } + + if (piov != NULL) + free(piov, M_MD); + bp->bio_resid = auio.uio_resid; return (error); } @@ -801,6 +955,7 @@ mdstart_swap(struct md_s *sc, struct bio *bp) vm_page_t m; u_char *p; vm_pindex_t i, lastp; + bus_dma_segment_t *vlist; int rv, ma_offs, offs, len, lastend; switch (bp->bio_cmd) { @@ -813,7 +968,10 @@ mdstart_swap(struct md_s *sc, struct bio *bp) } p = bp->bio_data; - ma_offs = (bp->bio_flags & BIO_UNMAPPED) == 0 ? 0 : bp->bio_ma_offset; + ma_offs = (bp->bio_flags & (BIO_UNMAPPED|BIO_VLIST)) != 0 ? + bp->bio_ma_offset : 0; + vlist = (bp->bio_flags & BIO_VLIST) != 0 ? + (bus_dma_segment_t *)bp->bio_data : NULL; /* * offs is the offset at which to start operating on the @@ -853,6 +1011,10 @@ mdstart_swap(struct md_s *sc, struct bio *bp) if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pmap_copy_pages(&m, offs, bp->bio_ma, ma_offs, len); + } else if ((bp->bio_flags & BIO_VLIST) != 0) { + physcopyout_vlist(VM_PAGE_TO_PHYS(m) + offs, + vlist, ma_offs, len); + cpu_flush_dcache(p, len); } else { physcopyout(VM_PAGE_TO_PHYS(m) + offs, p, len); cpu_flush_dcache(p, len); @@ -869,6 +1031,9 @@ mdstart_swap(struct md_s *sc, struct bio *bp) if ((bp->bio_flags & BIO_UNMAPPED) != 0) { pmap_copy_pages(bp->bio_ma, ma_offs, &m, offs, len); + } else if ((bp->bio_flags & BIO_VLIST) != 0) { + physcopyin_vlist(vlist, ma_offs, + VM_PAGE_TO_PHYS(m) + offs, len); } else { physcopyin(p, VM_PAGE_TO_PHYS(m) + offs, len); } |