diff options
author | glebius <glebius@FreeBSD.org> | 2014-11-23 12:01:52 +0000 |
---|---|---|
committer | glebius <glebius@FreeBSD.org> | 2014-11-23 12:01:52 +0000 |
commit | b4ef8e602d9f4d2aaffedc22322c8db21a20c289 (patch) | |
tree | f1f00e6a4e1bf77d75d8ea1deacb1498df360944 /sys/vm/vnode_pager.c | |
parent | 10696c171b9b4f5224bf7aa76a0c80e7c072a26b (diff) | |
download | FreeBSD-src-b4ef8e602d9f4d2aaffedc22322c8db21a20c289.zip FreeBSD-src-b4ef8e602d9f4d2aaffedc22322c8db21a20c289.tar.gz |
Merge from projects/sendfile:
o Provide a new VOP_GETPAGES_ASYNC(), which works like VOP_GETPAGES(), but
doesn't sleep. It returns immediately, and will execute the I/O done handler
function that must be supplied as argument.
o Provide VOP_GETPAGES_ASYNC() for the FFS, which uses vnode_pager.
o Extend pagertab to support pgo_getpages_async method, and implement this
method for vnode_pager.
Reviewed by: kib
Tested by: pho
Sponsored by: Netflix
Sponsored by: Nginx, Inc.
Diffstat (limited to 'sys/vm/vnode_pager.c')
-rw-r--r-- | sys/vm/vnode_pager.c | 165 |
1 files changed, 122 insertions, 43 deletions
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index 2aeef7e..1178b14 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -82,16 +82,23 @@ static int vnode_pager_addr(struct vnode *vp, vm_ooffset_t address, static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m); static int vnode_pager_input_old(vm_object_t object, vm_page_t m); static void vnode_pager_dealloc(vm_object_t); +static int vnode_pager_local_getpages0(struct vnode *, vm_page_t *, int, int, + vop_getpages_iodone_t, void *); static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int); +static int vnode_pager_getpages_async(vm_object_t, vm_page_t *, int, int, + vop_getpages_iodone_t, void *); static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, int, int *); static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *); static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t, vm_ooffset_t, struct ucred *cred); +static int vnode_pager_generic_getpages_done(struct buf *); +static void vnode_pager_generic_getpages_done_async(struct buf *); struct pagerops vnodepagerops = { .pgo_alloc = vnode_pager_alloc, .pgo_dealloc = vnode_pager_dealloc, .pgo_getpages = vnode_pager_getpages, + .pgo_getpages_async = vnode_pager_getpages_async, .pgo_putpages = vnode_pager_putpages, .pgo_haspage = vnode_pager_haspage, }; @@ -664,16 +671,51 @@ vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) return rtval; } +static int +vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count, + int reqpage, vop_getpages_iodone_t iodone, void *arg) +{ + struct vnode *vp; + int rtval; + + vp = object->handle; + VM_OBJECT_WUNLOCK(object); + rtval = VOP_GETPAGES_ASYNC(vp, m, count * PAGE_SIZE, reqpage, 0, + iodone, arg); + KASSERT(rtval != EOPNOTSUPP, + ("vnode_pager: FS getpages_async not implemented\n")); + VM_OBJECT_WLOCK(object); + return (rtval); +} + /* - * The implementation of VOP_GETPAGES() for local filesystems, where - * partially valid pages can only occur at the end of file. + * The implementation of VOP_GETPAGES() and VOP_GETPAGES_ASYNC() for + * local filesystems, where partially valid pages can only occur at + * the end of file. */ int vnode_pager_local_getpages(struct vop_getpages_args *ap) { + + return (vnode_pager_local_getpages0(ap->a_vp, ap->a_m, ap->a_count, + ap->a_reqpage, NULL, NULL)); +} + +int +vnode_pager_local_getpages_async(struct vop_getpages_async_args *ap) +{ + + return (vnode_pager_local_getpages0(ap->a_vp, ap->a_m, ap->a_count, + ap->a_reqpage, ap->a_iodone, ap->a_arg)); +} + +static int +vnode_pager_local_getpages0(struct vnode *vp, vm_page_t *m, int bytecount, + int reqpage, vop_getpages_iodone_t iodone, void *arg) +{ vm_page_t mreq; - mreq = ap->a_m[ap->a_reqpage]; + mreq = m[reqpage]; /* * Since the caller has busied the requested page, that page's valid @@ -688,13 +730,15 @@ vnode_pager_local_getpages(struct vop_getpages_args *ap) * pages, since no i/o is done to read its content. */ if (mreq->valid != 0) { - vm_pager_free_nonreq(mreq->object, ap->a_m, ap->a_reqpage, - round_page(ap->a_count) / PAGE_SIZE); + vm_pager_free_nonreq(mreq->object, m, reqpage, + round_page(bytecount) / PAGE_SIZE); + if (iodone != NULL) + iodone(arg, m, reqpage, 0); return (VM_PAGER_OK); } - return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m, - ap->a_count, ap->a_reqpage)); + return (vnode_pager_generic_getpages(vp, m, bytecount, reqpage, + iodone, arg)); } /* @@ -703,11 +747,10 @@ vnode_pager_local_getpages(struct vop_getpages_args *ap) */ int vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount, - int reqpage) + int reqpage, vop_getpages_iodone_t iodone, void *arg) { vm_object_t object; - vm_offset_t kva; - off_t foff, tfoff, nextoff; + off_t foff; int i, j, size, bsize, first; daddr_t firstaddr, reqblock; struct bufobj *bo; @@ -899,7 +942,7 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount, } bp = getpbuf(&vnode_pbuf_freecnt); - kva = (vm_offset_t)bp->b_data; + bp->b_kvaalloc = bp->b_data; /* * and map the pages to be read into the kva, if the filesystem @@ -911,15 +954,11 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount, bp->b_kvabase = unmapped_buf; bp->b_offset = 0; bp->b_flags |= B_UNMAPPED; - bp->b_npages = count; - for (i = 0; i < count; i++) - bp->b_pages[i] = m[i]; } else - pmap_qenter(kva, m, count); + pmap_qenter((vm_offset_t)bp->b_kvaalloc, m, count); /* build a minimal buffer header */ bp->b_iocmd = BIO_READ; - bp->b_iodone = bdone; KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred")); KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred")); bp->b_rcred = crhold(curthread->td_ucred); @@ -930,6 +969,10 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount, bp->b_bcount = size; bp->b_bufsize = size; bp->b_runningbufspace = bp->b_bufsize; + for (i = 0; i < count; i++) + bp->b_pages[i] = m[i]; + bp->b_npages = count; + bp->b_pager.pg_reqpage = reqpage; atomic_add_long(&runningbufspace, bp->b_runningbufspace); PCPU_INC(cnt.v_vnodein); @@ -937,43 +980,79 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount, /* do the input */ bp->b_iooffset = dbtob(bp->b_blkno); - bstrategy(bp); - bwait(bp, PVM, "vnread"); + if (iodone != NULL) { /* async */ + bp->b_pager.pg_iodone = iodone; + bp->b_caller1 = arg; + bp->b_iodone = vnode_pager_generic_getpages_done_async; + bp->b_flags |= B_ASYNC; + BUF_KERNPROC(bp); + bstrategy(bp); + /* Good bye! */ + } else { + bp->b_iodone = bdone; + bstrategy(bp); + bwait(bp, PVM, "vnread"); + error = vnode_pager_generic_getpages_done(bp); + for (int i = 0; i < bp->b_npages; i++) + bp->b_pages[i] = NULL; + bp->b_vp = NULL; + pbrelbo(bp); + relpbuf(bp, &vnode_pbuf_freecnt); + } + + return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK); +} + +static void +vnode_pager_generic_getpages_done_async(struct buf *bp) +{ + int error; + + error = vnode_pager_generic_getpages_done(bp); + bp->b_pager.pg_iodone(bp->b_caller1, bp->b_pages, + bp->b_pager.pg_reqpage, error); + for (int i = 0; i < bp->b_npages; i++) + bp->b_pages[i] = NULL; + bp->b_vp = NULL; + pbrelbo(bp); + relpbuf(bp, &vnode_pbuf_freecnt); +} + +static int +vnode_pager_generic_getpages_done(struct buf *bp) +{ + vm_object_t object; + off_t tfoff, nextoff; + int i, error; - if ((bp->b_ioflags & BIO_ERROR) != 0) - error = EIO; + error = (bp->b_ioflags & BIO_ERROR) != 0 ? EIO : 0; + object = bp->b_vp->v_object; - if (error == 0 && size != count * PAGE_SIZE) { + if (error == 0 && bp->b_bcount != bp->b_npages * PAGE_SIZE) { if ((bp->b_flags & B_UNMAPPED) != 0) { bp->b_flags &= ~B_UNMAPPED; - pmap_qenter(kva, m, count); + pmap_qenter((vm_offset_t)bp->b_kvaalloc, bp->b_pages, + bp->b_npages); } - bzero((caddr_t)kva + size, PAGE_SIZE * count - size); + bzero(bp->b_kvaalloc + bp->b_bcount, + PAGE_SIZE * bp->b_npages - bp->b_bcount); } if ((bp->b_flags & B_UNMAPPED) == 0) - pmap_qremove(kva, count); - if ((vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) { - bp->b_data = (caddr_t)kva; - bp->b_kvabase = (caddr_t)kva; + pmap_qremove((vm_offset_t)bp->b_kvaalloc, bp->b_npages); + if ((bp->b_vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) { + bp->b_data = bp->b_kvaalloc; + bp->b_kvabase = bp->b_kvaalloc; bp->b_flags &= ~B_UNMAPPED; - for (i = 0; i < count; i++) - bp->b_pages[i] = NULL; } - /* - * free the buffer header back to the swap buffer pool - */ - bp->b_vp = NULL; - pbrelbo(bp); - relpbuf(bp, &vnode_pbuf_freecnt); - VM_OBJECT_WLOCK(object); - for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) { + for (i = 0, tfoff = IDX_TO_OFF(bp->b_pages[0]->pindex); + i < bp->b_npages; i++, tfoff = nextoff) { vm_page_t mt; nextoff = tfoff + PAGE_SIZE; - mt = m[i]; + mt = bp->b_pages[i]; if (nextoff <= object->un_pager.vnp.vnp_size) { /* @@ -999,14 +1078,14 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount, ("%s: page %p is dirty", __func__, mt)); } - if (i != reqpage) + if (i != bp->b_pager.pg_reqpage) vm_page_readahead_finish(mt); } VM_OBJECT_WUNLOCK(object); - if (error) { - printf("vnode_pager_getpages: I/O read error\n"); - } - return (error ? VM_PAGER_ERROR : VM_PAGER_OK); + if (error != 0) + printf("%s: I/O read error %d\n", __func__, error); + + return (error); } /* |