Introduce and use a new VM interface for temporarily pinning pages. This

new interface replaces the combined use of vm_fault_quick() and
pmap_extract_and_hold() throughout the kernel.

In collaboration with:	kib@

11 files changed, 116 insertions, 291 deletions

diff --git a/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c b/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c
index 33f1dd1..bb0015f 100644
--- a/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c
+++ b/sys/dev/cxgb/ulp/tom/cxgb_cpl_socket.c
@@ -90,7 +90,6 @@ __FBSDID("$FreeBSD$");
 #include <ulp/tom/cxgb_t3_ddp.h>
 #include <ulp/tom/cxgb_toepcb.h>
 #include <ulp/tom/cxgb_tcp.h>
-#include <ulp/tom/cxgb_vm.h>
 
 
 static int	(*pru_sosend)(struct socket *so, struct sockaddr *addr,
@@ -218,8 +217,9 @@ cxgb_hold_iovec_pages(struct uio *uio, vm_page_t *m, int *held, vm_prot_t prot)
 		
 		count = min(count, npages);
 
-		err = vm_fault_hold_user_pages(map,
-			(vm_offset_t)iov->iov_base, mp, count, prot);
+		/* The following return value is not used. XXX */
+		err = vm_fault_quick_hold_pages(map,
+		    (vm_offset_t)iov->iov_base, iov->iov_len, prot, mp, count);
 		mp += count;
 		totcount += count;
 		curbytes = iov->iov_len;
@@ -503,7 +503,7 @@ cxgb_sosend(struct socket *so, struct sockaddr *addr, struct uio *uio,
 	 *  - the number of bytes to be transferred exceeds the threshold
 	 *  - the number of bytes currently in flight won't exceed the in-flight
 	 *    threshold XXX TODO
-	 *  - vm_fault_hold_user_pages succeeds
+	 *  - vm_fault_quick_hold_pages succeeds
 	 *  - blocking socket XXX for now
 	 *
 	 */
@@ -970,7 +970,7 @@ cxgb_soreceive(struct socket *so, struct sockaddr **psa, struct uio *uio,
 	 *  - the number of bytes to be transferred exceeds the threshold
 	 *  - the number of bytes currently in flight won't exceed the in-flight
 	 *    threshold XXX TODO
-	 *  - vm_fault_hold_user_pages succeeds
+	 *  - vm_fault_quick_hold_pages succeeds
 	 *  - blocking socket XXX for now
 	 *  - iovcnt is 1
 	 *
diff --git a/sys/dev/cxgb/ulp/tom/cxgb_ddp.c b/sys/dev/cxgb/ulp/tom/cxgb_ddp.c
index a188f62..fe3b075 100644
--- a/sys/dev/cxgb/ulp/tom/cxgb_ddp.c
+++ b/sys/dev/cxgb/ulp/tom/cxgb_ddp.c
@@ -90,7 +90,6 @@ __FBSDID("$FreeBSD$");
 #include <ulp/tom/cxgb_t3_ddp.h>
 #include <ulp/tom/cxgb_toepcb.h>
 #include <ulp/tom/cxgb_tcp.h>
-#include <ulp/tom/cxgb_vm.h>
 
 
 #define MAX_SCHEDULE_TIMEOUT	300
@@ -130,14 +129,6 @@ t3_pin_pages(bus_dma_tag_t tag, bus_dmamap_t dmamap, vm_offset_t addr,
 	struct ddp_gather_list *p;
 	vm_map_t map;
 	
-	/*
-	 * XXX need x86 agnostic check
-	 */
-	if (addr + len > VM_MAXUSER_ADDRESS)
-		return (EFAULT);
-
-
-	
 	pg_off = addr & PAGE_MASK;
 	npages = (pg_off + len + PAGE_SIZE - 1) >> PAGE_SHIFT;
 	p = malloc(sizeof(struct ddp_gather_list) + npages * sizeof(vm_page_t *),
@@ -146,10 +137,11 @@ t3_pin_pages(bus_dma_tag_t tag, bus_dmamap_t dmamap, vm_offset_t addr,
 		return (ENOMEM);
 
 	map = &curthread->td_proc->p_vmspace->vm_map;
-	err = vm_fault_hold_user_pages(map, addr, p->dgl_pages, npages,
-	    VM_PROT_READ | VM_PROT_WRITE);
-	if (err)
+	if (vm_fault_quick_hold_pages(map, addr, len, VM_PROT_READ |
+	    VM_PROT_WRITE, p->dgl_pages, npages) < 0) {
+		err = EFAULT;
 		goto free_gl;
+	}
 
 	if (gl && gl->dgl_offset == pg_off && gl->dgl_nelem >= npages &&
 	    gl->dgl_length >= len) {
diff --git a/sys/dev/cxgb/ulp/tom/cxgb_vm.c b/sys/dev/cxgb/ulp/tom/cxgb_vm.c
deleted file mode 100644
index e84d206..0000000
--- a/sys/dev/cxgb/ulp/tom/cxgb_vm.c
+++ /dev/null
@@ -1,143 +0,0 @@
-/**************************************************************************
-
-Copyright (c) 2007-2008, Chelsio Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Neither the name of the Chelsio Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-***************************************************************************/
-
-#include <sys/cdefs.h>
-__FBSDID("$FreeBSD$");
-
-#include <sys/param.h>
-#include <sys/systm.h>
-#include <sys/types.h>
-#include <sys/fcntl.h>
-#include <sys/kernel.h>
-#include <sys/limits.h>
-#include <sys/lock.h>
-#include <sys/mbuf.h>
-#include <sys/condvar.h>
-#include <sys/mutex.h>
-#include <sys/proc.h>
-#include <sys/syslog.h>
-
-#include <vm/vm.h>
-#include <vm/vm_page.h>
-#include <vm/vm_map.h>
-#include <vm/vm_extern.h>
-#include <vm/pmap.h>
-#include <ulp/tom/cxgb_vm.h>
-
-/*
- * This routine takes a user's map, array of pages, number of pages, and flags
- * and then does the following:
- *  - validate that the user has access to those pages (flags indicates read
- *	or write) - if not fail
- *  - validate that count is enough to hold range number of pages - if not fail
- *  - fault in any non-resident pages
- *  - if the user is doing a read force a write fault for any COWed pages
- *  - if the user is doing a read mark all pages as dirty
- *  - hold all pages
- */
-int
-vm_fault_hold_user_pages(vm_map_t map, vm_offset_t addr, vm_page_t *mp,
-    int count, vm_prot_t prot)
-{
-	vm_offset_t end, va;
-	int faults, rv;
-	pmap_t pmap;
-	vm_page_t m, *pages;
-	
-	pmap = vm_map_pmap(map);
-	pages = mp;
-	addr &= ~PAGE_MASK;
-	/*
-	 * Check that virtual address range is legal
-	 * This check is somewhat bogus as on some architectures kernel
-	 * and user do not share VA - however, it appears that all FreeBSD
-	 * architectures define it
-	 */
-	end = addr + (count * PAGE_SIZE);
-	if (end > VM_MAXUSER_ADDRESS) {
-		log(LOG_WARNING, "bad address passed to vm_fault_hold_user_pages");
-		return (EFAULT);
-	}
-
-	/*
-	 * First optimistically assume that all pages are resident 
-	 * (and R/W if for write) if so just mark pages as held (and 
-	 * dirty if for write) and return
-	 */
-	for (pages = mp, faults = 0, va = addr; va < end;
-	     va += PAGE_SIZE, pages++) {
-		/*
-		 * it would be really nice if we had an unlocked
-		 * version of this so we were only acquiring the 
-		 * pmap lock 1 time as opposed to potentially
-		 * many dozens of times
-		 */
-		*pages = m = pmap_extract_and_hold(pmap, va, prot);
-		if (m == NULL) {
-			faults++;
-			continue;
-		}
-		/*
-		 * Preemptively mark dirty - the pages
-		 * will never have the modified bit set if
-		 * they are only changed via DMA
-		 */
-		if (prot & VM_PROT_WRITE) {
-			vm_page_lock_queues();
-			vm_page_dirty(m);
-			vm_page_unlock_queues();
-		}
-		
-	}
-	
-	if (faults == 0)
-		return (0);
-	
-	/*
-	 * Pages either have insufficient permissions or are not present
-	 * trigger a fault where neccessary
-	 */
-	for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++) {
-		if (*pages == NULL && (rv = vm_fault_hold(map, va, prot,
-		    VM_FAULT_NORMAL, pages)) != KERN_SUCCESS)
-			goto error;
-	}
-	return (0);
-error:	
-	log(LOG_WARNING,
-	    "vm_fault bad return rv=%d va=0x%zx\n", rv, va);
-	for (pages = mp, va = addr; va < end; va += PAGE_SIZE, pages++)
-		if (*pages) {
-			vm_page_lock(*pages);
-			vm_page_unhold(*pages);
-			vm_page_unlock(*pages);
-			*pages = NULL;
-		}
-	return (EFAULT);
-}
diff --git a/sys/dev/cxgb/ulp/tom/cxgb_vm.h b/sys/dev/cxgb/ulp/tom/cxgb_vm.h
deleted file mode 100644
index 6647d55..0000000
--- a/sys/dev/cxgb/ulp/tom/cxgb_vm.h
+++ /dev/null
@@ -1,38 +0,0 @@
-/**************************************************************************
-
-Copyright (c) 2007-2008, Chelsio Inc.
-All rights reserved.
-
-Redistribution and use in source and binary forms, with or without
-modification, are permitted provided that the following conditions are met:
-
- 1. Redistributions of source code must retain the above copyright notice,
-    this list of conditions and the following disclaimer.
-
- 2. Neither the name of the Chelsio Corporation nor the names of its
-    contributors may be used to endorse or promote products derived from
-    this software without specific prior written permission.
-
-THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
-AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
-IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
-ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR CONTRIBUTORS BE
-LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
-CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
-SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
-INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
-CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
-ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
-POSSIBILITY OF SUCH DAMAGE.
-
-
-$FreeBSD$
-
-***************************************************************************/
-#ifndef CXGB_VM_H_
-#define CXGB_VM_H_
-
-int vm_fault_hold_user_pages(vm_map_t map, vm_offset_t addr,
-    vm_page_t *mp, int count, vm_prot_t prot);
-
-#endif
diff --git a/sys/dev/drm/via_dmablit.c b/sys/dev/drm/via_dmablit.c
index 9be42ff..11dc2aa 100644
--- a/sys/dev/drm/via_dmablit.c
+++ b/sys/dev/drm/via_dmablit.c
@@ -177,11 +177,10 @@ via_free_sg_info(drm_via_sg_info_t *vsg)
 		free(vsg->desc_pages, DRM_MEM_DRIVER);
 	case dr_via_pages_locked:
 		for (i=0; i < vsg->num_pages; ++i) {
-			if ( NULL != (page = vsg->pages[i])) {
-				vm_page_lock(page);
-				vm_page_unwire(page, 0);
-				vm_page_unlock(page);
-			}
+			page = vsg->pages[i];
+			vm_page_lock(page);
+			vm_page_unwire(page, 0);
+			vm_page_unlock(page);
 		}
 	case dr_via_pages_alloc:
 		free(vsg->pages, DRM_MEM_DRIVER);
@@ -224,41 +223,31 @@ via_lock_all_dma_pages(drm_via_sg_info_t *vsg,  drm_via_dmablit_t *xfer)
 {
 	unsigned long first_pfn = VIA_PFN(xfer->mem_addr);
 	vm_page_t m;
-	vm_map_t map;
 	int i;
 
-	map = &curproc->p_vmspace->vm_map;
-
 	vsg->num_pages = VIA_PFN(xfer->mem_addr +
 	    (xfer->num_lines * xfer->mem_stride -1)) - first_pfn + 1;
 
-	/* Make sure that the user has access to these pages */
-	for(i = 0; i < vsg->num_pages; i++) {
-		if (vm_fault_quick((caddr_t)xfer->mem_addr + IDX_TO_OFF(i),
-		    VM_PROT_RW) < 0)
-			return (-EACCES);
-	}
-
 	if (NULL == (vsg->pages = malloc(sizeof(vm_page_t) * vsg->num_pages,
-	    DRM_MEM_DRIVER, M_NOWAIT | M_ZERO)))
+	    DRM_MEM_DRIVER, M_NOWAIT)))
 		return -ENOMEM;
 
-	for(i = 0; i < vsg->num_pages; i++) {
-		m = pmap_extract_and_hold(map->pmap,
-		    (vm_offset_t)xfer->mem_addr + IDX_TO_OFF(i), VM_PROT_RW);
-		if (m == NULL)
-			break;
+	vsg->state = dr_via_pages_alloc;
+
+	if (vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
+	    (vm_offset_t)xfer->mem_addr, vsg->num_pages * PAGE_SIZE,
+	    VM_PROT_READ | VM_PROT_WRITE, vsg->pages, vsg->num_pages) < 0)
+		return -EACCES;
+
+	for (i = 0; i < vsg->num_pages; i++) {
+		m = vsg->pages[i];
 		vm_page_lock(m);
 		vm_page_wire(m);
 		vm_page_unhold(m);
 		vm_page_unlock(m);
-		vsg->pages[i] = m;
 	}
 	vsg->state = dr_via_pages_locked;
 
-	if (i != vsg->num_pages)
-		return -EINVAL;
-
 	DRM_DEBUG("DMA pages locked\n");
 
 	return 0;
diff --git a/sys/kern/sys_pipe.c b/sys/kern/sys_pipe.c
index 444b424..65839d6 100644
--- a/sys/kern/sys_pipe.c
+++ b/sys/kern/sys_pipe.c
@@ -747,10 +747,8 @@ pipe_build_write_buffer(wpipe, uio)
 	struct pipe *wpipe;
 	struct uio *uio;
 {
-	pmap_t pmap;
 	u_int size;
 	int i;
-	vm_offset_t addr, endaddr;
 
 	PIPE_LOCK_ASSERT(wpipe, MA_NOTOWNED);
 	KASSERT(wpipe->pipe_state & PIPE_DIRECTW,
@@ -760,25 +758,10 @@ pipe_build_write_buffer(wpipe, uio)
 	if (size > wpipe->pipe_buffer.size)
 		size = wpipe->pipe_buffer.size;
 
-	pmap = vmspace_pmap(curproc->p_vmspace);
-	endaddr = round_page((vm_offset_t)uio->uio_iov->iov_base + size);
-	addr = trunc_page((vm_offset_t)uio->uio_iov->iov_base);
-	if (endaddr < addr)
+	if ((i = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
+	    (vm_offset_t)uio->uio_iov->iov_base, size, VM_PROT_READ,
+	    wpipe->pipe_map.ms, PIPENPAGES)) < 0)
 		return (EFAULT);
-	for (i = 0; addr < endaddr; addr += PAGE_SIZE, i++) {
-		/*
-		 * vm_fault_quick() can sleep.
-		 */
-	race:
-		if (vm_fault_quick((caddr_t)addr, VM_PROT_READ) < 0) {
-			vm_page_unhold_pages(wpipe->pipe_map.ms, i);
-			return (EFAULT);
-		}
-		wpipe->pipe_map.ms[i] = pmap_extract_and_hold(pmap, addr,
-		    VM_PROT_READ);
-		if (wpipe->pipe_map.ms[i] == NULL)
-			goto race;
-	}
 
 /*
  * set up the control block
diff --git a/sys/kern/uipc_cow.c b/sys/kern/uipc_cow.c
index 853b70b..f7ef798 100644
--- a/sys/kern/uipc_cow.c
+++ b/sys/kern/uipc_cow.c
@@ -103,24 +103,20 @@ socow_setup(struct mbuf *m0, struct uio *uio)
 	struct vmspace *vmspace;
 	struct vm_map *map;
 	vm_offset_t offset, uva;
+	vm_size_t len;
 
 	socow_stats.attempted++;
 	vmspace = curproc->p_vmspace;
 	map = &vmspace->vm_map;
 	uva = (vm_offset_t) uio->uio_iov->iov_base;
 	offset = uva & PAGE_MASK;
+	len = PAGE_SIZE - offset;
 
 	/*
 	 * Verify that access to the given address is allowed from user-space.
 	 */
-	if (vm_fault_quick((caddr_t)uva, VM_PROT_READ) < 0)
-		return (0);
-
-       /* 
-	* verify page is mapped & not already wired for i/o
-	*/
-	pp = pmap_extract_and_hold(map->pmap, uva, VM_PROT_READ);
-	if (pp == NULL) {
+	if (vm_fault_quick_hold_pages(map, uva, len, &pp, 1, VM_PROT_READ) <
+	    0) {
 		socow_stats.fail_not_mapped++;
 		return(0);
 	}
@@ -165,7 +161,7 @@ socow_setup(struct mbuf *m0, struct uio *uio)
 	 */
 	MEXTADD(m0, sf_buf_kva(sf), PAGE_SIZE, socow_iodone,
 	    (void*)sf_buf_kva(sf), sf, M_RDONLY, EXT_SFBUF);
-	m0->m_len = PAGE_SIZE - offset;
+	m0->m_len = len;
 	m0->m_data = (caddr_t)sf_buf_kva(sf) + offset;
 	socow_stats.success++;
 
diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 7d04e33..0147bff 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -3855,46 +3855,19 @@ vm_hold_free_pages(struct buf *bp, int newbsize)
 int
 vmapbuf(struct buf *bp)
 {
-	caddr_t addr, kva;
+	caddr_t kva;
 	vm_prot_t prot;
-	int pidx, i;
-	struct vm_page *m;
-	struct pmap *pmap = &curproc->p_vmspace->vm_pmap;
+	int pidx;
 
 	if (bp->b_bufsize < 0)
 		return (-1);
 	prot = VM_PROT_READ;
 	if (bp->b_iocmd == BIO_READ)
 		prot |= VM_PROT_WRITE;	/* Less backwards than it looks */
-	for (addr = (caddr_t)trunc_page((vm_offset_t)bp->b_data), pidx = 0;
-	     addr < bp->b_data + bp->b_bufsize;
-	     addr += PAGE_SIZE, pidx++) {
-		/*
-		 * Do the vm_fault if needed; do the copy-on-write thing
-		 * when reading stuff off device into memory.
-		 *
-		 * NOTE! Must use pmap_extract() because addr may be in
-		 * the userland address space, and kextract is only guarenteed
-		 * to work for the kernland address space (see: sparc64 port).
-		 */
-retry:
-		if (vm_fault_quick(addr >= bp->b_data ? addr : bp->b_data,
-		    prot) < 0) {
-			for (i = 0; i < pidx; ++i) {
-				vm_page_lock(bp->b_pages[i]);
-				vm_page_unhold(bp->b_pages[i]);
-				vm_page_unlock(bp->b_pages[i]);
-				bp->b_pages[i] = NULL;
-			}
-			return(-1);
-		}
-		m = pmap_extract_and_hold(pmap, (vm_offset_t)addr, prot);
-		if (m == NULL)
-			goto retry;
-		bp->b_pages[pidx] = m;
-	}
-	if (pidx > btoc(MAXPHYS))
-		panic("vmapbuf: mapped more than MAXPHYS");
+	if ((pidx = vm_fault_quick_hold_pages(&curproc->p_vmspace->vm_map,
+	    (vm_offset_t)bp->b_data, bp->b_bufsize, prot, bp->b_pages,
+	    btoc(MAXPHYS))) < 0)
+		return (-1);
 	pmap_qenter((vm_offset_t)bp->b_saveaddr, bp->b_pages, pidx);
 	
 	kva = bp->b_saveaddr;
diff --git a/sys/net/bpf_zerocopy.c b/sys/net/bpf_zerocopy.c
index 91efd64..7a3e6cd 100644
--- a/sys/net/bpf_zerocopy.c
+++ b/sys/net/bpf_zerocopy.c
@@ -161,12 +161,8 @@ zbuf_sfbuf_get(struct vm_map *map, vm_offset_t uaddr)
 	struct sf_buf *sf;
 	vm_page_t pp;
 
-	if (vm_fault_quick((caddr_t) uaddr, VM_PROT_READ | VM_PROT_WRITE) <
-	    0)
-		return (NULL);
-	pp = pmap_extract_and_hold(map->pmap, uaddr, VM_PROT_READ |
-	    VM_PROT_WRITE);
-	if (pp == NULL)
+	if (vm_fault_quick_hold_pages(map, uaddr, PAGE_SIZE, VM_PROT_READ |
+	    VM_PROT_WRITE, &pp, 1) < 0)
 		return (NULL);
 	vm_page_lock(pp);
 	vm_page_wire(pp);
diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h
index b4ad02b..4c1e304 100644
--- a/sys/vm/vm_extern.h
+++ b/sys/vm/vm_extern.h
@@ -63,6 +63,8 @@ void vm_fault_copy_entry(vm_map_t, vm_map_t, vm_map_entry_t, vm_map_entry_t,
     vm_ooffset_t *);
 int vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type,
     int fault_flags, vm_page_t *m_hold);
+int vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len,
+    vm_prot_t prot, vm_page_t *ma, int max_count);
 void vm_fault_unwire(vm_map_t, vm_offset_t, vm_offset_t, boolean_t);
 int vm_fault_wire(vm_map_t, vm_offset_t, vm_offset_t, boolean_t);
 int vm_forkproc(struct thread *, struct proc *, struct thread *, struct vmspace *, int);
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 57b72e5..8e649d2 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -1045,6 +1045,81 @@ vm_fault_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry)
 }
 
 /*
+ * Hold each of the physical pages that are mapped by the specified range of
+ * virtual addresses, ["addr", "addr" + "len"), if those mappings are valid
+ * and allow the specified types of access, "prot".  If all of the implied
+ * pages are successfully held, then the number of held pages is returned
+ * together with pointers to those pages in the array "ma".  However, if any
+ * of the pages cannot be held, -1 is returned.
+ */
+int
+vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len,
+    vm_prot_t prot, vm_page_t *ma, int max_count)
+{
+	vm_offset_t end, va;
+	vm_page_t *mp;
+	int count;
+	boolean_t pmap_failed;
+
+	end = round_page(addr + len);	
+	addr = trunc_page(addr);
+
+	/*
+	 * Check for illegal addresses.
+	 */
+	if (addr < vm_map_min(map) || addr > end || end > vm_map_max(map))
+		return (-1);
+
+	count = howmany(end - addr, PAGE_SIZE);
+	if (count > max_count)
+		panic("vm_fault_quick_hold_pages: count > max_count");
+
+	/*
+	 * Most likely, the physical pages are resident in the pmap, so it is
+	 * faster to try pmap_extract_and_hold() first.
+	 */
+	pmap_failed = FALSE;
+	for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE) {
+		*mp = pmap_extract_and_hold(map->pmap, va, prot);
+		if (*mp == NULL)
+			pmap_failed = TRUE;
+		else if ((prot & VM_PROT_WRITE) != 0 &&
+		    (*ma)->dirty != VM_PAGE_BITS_ALL) {
+			/*
+			 * Explicitly dirty the physical page.  Otherwise, the
+			 * caller's changes may go unnoticed because they are
+			 * performed through an unmanaged mapping or by a DMA
+			 * operation.
+			 */
+			vm_page_lock_queues();
+			vm_page_dirty(*mp);
+			vm_page_unlock_queues();
+		}
+	}
+	if (pmap_failed) {
+		/*
+		 * One or more pages could not be held by the pmap.  Either no
+		 * page was mapped at the specified virtual address or that
+		 * mapping had insufficient permissions.  Attempt to fault in
+		 * and hold these pages.
+		 */
+		for (mp = ma, va = addr; va < end; mp++, va += PAGE_SIZE)
+			if (*mp == NULL && vm_fault_hold(map, va, prot,
+			    VM_FAULT_NORMAL, mp) != KERN_SUCCESS)
+				goto error;
+	}
+	return (count);
+error:	
+	for (mp = ma; mp < ma + count; mp++)
+		if (*mp != NULL) {
+			vm_page_lock(*mp);
+			vm_page_unhold(*mp);
+			vm_page_unlock(*mp);
+		}
+	return (-1);
+}
+
+/*
  *	vm_fault_quick:
  *
  *	Ensure that the requested virtual address, which may be in userland,