Merge from projects/sendfile:

o Provide a new VOP_GETPAGES_ASYNC(), which works like VOP_GETPAGES(), but
  doesn't sleep. It returns immediately, and will execute the I/O done handler
  function that must be supplied as argument.
o Provide VOP_GETPAGES_ASYNC() for the FFS, which uses vnode_pager.
o Extend pagertab to support pgo_getpages_async method, and implement this
  method for vnode_pager.

Reviewed by:	kib
Tested by:	pho
Sponsored by:	Netflix
Sponsored by:	Nginx, Inc.

1 files changed, 122 insertions, 43 deletions

diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index 2aeef7e..1178b14 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -82,16 +82,23 @@ static int vnode_pager_addr(struct vnode *vp, vm_ooffset_t address,
 static int vnode_pager_input_smlfs(vm_object_t object, vm_page_t m);
 static int vnode_pager_input_old(vm_object_t object, vm_page_t m);
 static void vnode_pager_dealloc(vm_object_t);
+static int vnode_pager_local_getpages0(struct vnode *, vm_page_t *, int, int,
+    vop_getpages_iodone_t, void *);
 static int vnode_pager_getpages(vm_object_t, vm_page_t *, int, int);
+static int vnode_pager_getpages_async(vm_object_t, vm_page_t *, int, int,
+    vop_getpages_iodone_t, void *);
 static void vnode_pager_putpages(vm_object_t, vm_page_t *, int, int, int *);
 static boolean_t vnode_pager_haspage(vm_object_t, vm_pindex_t, int *, int *);
 static vm_object_t vnode_pager_alloc(void *, vm_ooffset_t, vm_prot_t,
     vm_ooffset_t, struct ucred *cred);
+static int vnode_pager_generic_getpages_done(struct buf *);
+static void vnode_pager_generic_getpages_done_async(struct buf *);
 
 struct pagerops vnodepagerops = {
 	.pgo_alloc =	vnode_pager_alloc,
 	.pgo_dealloc =	vnode_pager_dealloc,
 	.pgo_getpages =	vnode_pager_getpages,
+	.pgo_getpages_async = vnode_pager_getpages_async,
 	.pgo_putpages =	vnode_pager_putpages,
 	.pgo_haspage =	vnode_pager_haspage,
 };
@@ -664,16 +671,51 @@ vnode_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
 	return rtval;
 }
 
+static int
+vnode_pager_getpages_async(vm_object_t object, vm_page_t *m, int count,
+    int reqpage, vop_getpages_iodone_t iodone, void *arg)
+{
+	struct vnode *vp;
+	int rtval;
+
+	vp = object->handle;
+	VM_OBJECT_WUNLOCK(object);
+	rtval = VOP_GETPAGES_ASYNC(vp, m, count * PAGE_SIZE, reqpage, 0,
+	    iodone, arg);
+	KASSERT(rtval != EOPNOTSUPP,
+	    ("vnode_pager: FS getpages_async not implemented\n"));
+	VM_OBJECT_WLOCK(object);
+	return (rtval);
+}
+
 /*
- * The implementation of VOP_GETPAGES() for local filesystems, where
- * partially valid pages can only occur at the end of file.
+ * The implementation of VOP_GETPAGES() and VOP_GETPAGES_ASYNC() for
+ * local filesystems, where partially valid pages can only occur at
+ * the end of file.
  */
 int
 vnode_pager_local_getpages(struct vop_getpages_args *ap)
 {
+
+	return (vnode_pager_local_getpages0(ap->a_vp, ap->a_m, ap->a_count,
+	    ap->a_reqpage, NULL, NULL));
+}
+
+int
+vnode_pager_local_getpages_async(struct vop_getpages_async_args *ap)
+{
+
+	return (vnode_pager_local_getpages0(ap->a_vp, ap->a_m, ap->a_count,
+	    ap->a_reqpage, ap->a_iodone, ap->a_arg));
+}
+
+static int
+vnode_pager_local_getpages0(struct vnode *vp, vm_page_t *m, int bytecount,
+    int reqpage, vop_getpages_iodone_t iodone, void *arg)
+{
 	vm_page_t mreq;
 
-	mreq = ap->a_m[ap->a_reqpage];
+	mreq = m[reqpage];
 
 	/*
 	 * Since the caller has busied the requested page, that page's valid
@@ -688,13 +730,15 @@ vnode_pager_local_getpages(struct vop_getpages_args *ap)
 	 * pages, since no i/o is done to read its content.
 	 */
 	if (mreq->valid != 0) {
-		vm_pager_free_nonreq(mreq->object, ap->a_m, ap->a_reqpage,
-		    round_page(ap->a_count) / PAGE_SIZE);
+		vm_pager_free_nonreq(mreq->object, m, reqpage,
+		    round_page(bytecount) / PAGE_SIZE);
+		if (iodone != NULL)
+			iodone(arg, m, reqpage, 0);
 		return (VM_PAGER_OK);
 	}
 
-	return (vnode_pager_generic_getpages(ap->a_vp, ap->a_m,
-	    ap->a_count, ap->a_reqpage));
+	return (vnode_pager_generic_getpages(vp, m, bytecount, reqpage,
+	    iodone, arg));
 }
 
 /*
@@ -703,11 +747,10 @@ vnode_pager_local_getpages(struct vop_getpages_args *ap)
  */
 int
 vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount,
-    int reqpage)
+    int reqpage, vop_getpages_iodone_t iodone, void *arg)
 {
 	vm_object_t object;
-	vm_offset_t kva;
-	off_t foff, tfoff, nextoff;
+	off_t foff;
 	int i, j, size, bsize, first;
 	daddr_t firstaddr, reqblock;
 	struct bufobj *bo;
@@ -899,7 +942,7 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount,
 	}
 
 	bp = getpbuf(&vnode_pbuf_freecnt);
-	kva = (vm_offset_t)bp->b_data;
+	bp->b_kvaalloc = bp->b_data;
 
 	/*
 	 * and map the pages to be read into the kva, if the filesystem
@@ -911,15 +954,11 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount,
 		bp->b_kvabase = unmapped_buf;
 		bp->b_offset = 0;
 		bp->b_flags |= B_UNMAPPED;
-		bp->b_npages = count;
-		for (i = 0; i < count; i++)
-			bp->b_pages[i] = m[i];
 	} else
-		pmap_qenter(kva, m, count);
+		pmap_qenter((vm_offset_t)bp->b_kvaalloc, m, count);
 
 	/* build a minimal buffer header */
 	bp->b_iocmd = BIO_READ;
-	bp->b_iodone = bdone;
 	KASSERT(bp->b_rcred == NOCRED, ("leaking read ucred"));
 	KASSERT(bp->b_wcred == NOCRED, ("leaking write ucred"));
 	bp->b_rcred = crhold(curthread->td_ucred);
@@ -930,6 +969,10 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount,
 	bp->b_bcount = size;
 	bp->b_bufsize = size;
 	bp->b_runningbufspace = bp->b_bufsize;
+	for (i = 0; i < count; i++)
+		bp->b_pages[i] = m[i];
+	bp->b_npages = count;
+	bp->b_pager.pg_reqpage = reqpage;
 	atomic_add_long(&runningbufspace, bp->b_runningbufspace);
 
 	PCPU_INC(cnt.v_vnodein);
@@ -937,43 +980,79 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount,
 
 	/* do the input */
 	bp->b_iooffset = dbtob(bp->b_blkno);
-	bstrategy(bp);
 
-	bwait(bp, PVM, "vnread");
+	if (iodone != NULL) { /* async */
+		bp->b_pager.pg_iodone = iodone;
+		bp->b_caller1 = arg;
+		bp->b_iodone = vnode_pager_generic_getpages_done_async;
+		bp->b_flags |= B_ASYNC;
+		BUF_KERNPROC(bp);
+		bstrategy(bp);
+		/* Good bye! */
+	} else {
+		bp->b_iodone = bdone;
+		bstrategy(bp);
+		bwait(bp, PVM, "vnread");
+		error = vnode_pager_generic_getpages_done(bp);
+		for (int i = 0; i < bp->b_npages; i++)
+			bp->b_pages[i] = NULL;
+		bp->b_vp = NULL;
+		pbrelbo(bp);
+		relpbuf(bp, &vnode_pbuf_freecnt);
+	}
+
+	return (error != 0 ? VM_PAGER_ERROR : VM_PAGER_OK);
+}
+
+static void
+vnode_pager_generic_getpages_done_async(struct buf *bp)
+{
+	int error;
+
+	error = vnode_pager_generic_getpages_done(bp);
+	bp->b_pager.pg_iodone(bp->b_caller1, bp->b_pages,
+	  bp->b_pager.pg_reqpage, error);
+	for (int i = 0; i < bp->b_npages; i++)
+		bp->b_pages[i] = NULL;
+	bp->b_vp = NULL;
+	pbrelbo(bp);
+	relpbuf(bp, &vnode_pbuf_freecnt);
+}
+
+static int
+vnode_pager_generic_getpages_done(struct buf *bp)
+{
+	vm_object_t object;
+	off_t tfoff, nextoff;
+	int i, error;
 
-	if ((bp->b_ioflags & BIO_ERROR) != 0)
-		error = EIO;
+	error = (bp->b_ioflags & BIO_ERROR) != 0 ? EIO : 0;
+	object = bp->b_vp->v_object;
 
-	if (error == 0 && size != count * PAGE_SIZE) {
+	if (error == 0 && bp->b_bcount != bp->b_npages * PAGE_SIZE) {
 		if ((bp->b_flags & B_UNMAPPED) != 0) {
 			bp->b_flags &= ~B_UNMAPPED;
-			pmap_qenter(kva, m, count);
+			pmap_qenter((vm_offset_t)bp->b_kvaalloc, bp->b_pages,
+			    bp->b_npages);
 		}
-		bzero((caddr_t)kva + size, PAGE_SIZE * count - size);
+		bzero(bp->b_kvaalloc + bp->b_bcount,
+		    PAGE_SIZE * bp->b_npages - bp->b_bcount);
 	}
 	if ((bp->b_flags & B_UNMAPPED) == 0)
-		pmap_qremove(kva, count);
-	if ((vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) {
-		bp->b_data = (caddr_t)kva;
-		bp->b_kvabase = (caddr_t)kva;
+		pmap_qremove((vm_offset_t)bp->b_kvaalloc, bp->b_npages);
+	if ((bp->b_vp->v_mount->mnt_kern_flag & MNTK_UNMAPPED_BUFS) != 0) {
+		bp->b_data = bp->b_kvaalloc;
+		bp->b_kvabase = bp->b_kvaalloc;
 		bp->b_flags &= ~B_UNMAPPED;
-		for (i = 0; i < count; i++)
-			bp->b_pages[i] = NULL;
 	}
 
-	/*
-	 * free the buffer header back to the swap buffer pool
-	 */
-	bp->b_vp = NULL;
-	pbrelbo(bp);
-	relpbuf(bp, &vnode_pbuf_freecnt);
-
 	VM_OBJECT_WLOCK(object);
-	for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) {
+	for (i = 0, tfoff = IDX_TO_OFF(bp->b_pages[0]->pindex);
+	    i < bp->b_npages; i++, tfoff = nextoff) {
 		vm_page_t mt;
 
 		nextoff = tfoff + PAGE_SIZE;
-		mt = m[i];
+		mt = bp->b_pages[i];
 
 		if (nextoff <= object->un_pager.vnp.vnp_size) {
 			/*
@@ -999,14 +1078,14 @@ vnode_pager_generic_getpages(struct vnode *vp, vm_page_t *m, int bytecount,
 			    ("%s: page %p is dirty", __func__, mt));
 		}
 		
-		if (i != reqpage)
+		if (i != bp->b_pager.pg_reqpage)
 			vm_page_readahead_finish(mt);
 	}
 	VM_OBJECT_WUNLOCK(object);
-	if (error) {
-		printf("vnode_pager_getpages: I/O read error\n");
-	}
-	return (error ? VM_PAGER_ERROR : VM_PAGER_OK);
+	if (error != 0)
+		printf("%s: I/O read error %d\n", __func__, error);
+
+	return (error);
 }
 
 /*