Introduce vm_reserv_reclaim_contig(). This function is used by

contigmalloc(9) as a last resort to steal pages from an inactive,
partially-used superpage reservation.

Rename vm_reserv_reclaim() to vm_reserv_reclaim_inactive() and
refactor it so that a separate subroutine is responsible for breaking
the selected reservation.  This subroutine is also used by
vm_reserv_reclaim_contig().

4 files changed, 99 insertions, 25 deletions

diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 637a6c6..9fb3d36 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1072,7 +1072,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
 			m = vm_phys_alloc_pages(object != NULL ?
 			    VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT, 0);
 #if VM_NRESERVLEVEL > 0
-			if (m == NULL && vm_reserv_reclaim()) {
+			if (m == NULL && vm_reserv_reclaim_inactive()) {
 				m = vm_phys_alloc_pages(object != NULL ?
 				    VM_FREEPOOL_DEFAULT : VM_FREEPOOL_DIRECT,
 				    0);
diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c
index b477b5b..3af40cf 100644
--- a/sys/vm/vm_phys.c
+++ b/sys/vm/vm_phys.c
@@ -54,6 +54,7 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_object.h>
 #include <vm/vm_page.h>
 #include <vm/vm_phys.h>
+#include <vm/vm_reserv.h>
 
 struct vm_freelist {
 	struct pglist pl;
@@ -607,6 +608,9 @@ vm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high,
 	/* Compute the queue that is the best fit for npages. */
 	for (order = 0; (1 << order) < npages; order++);
 	mtx_lock(&vm_page_queue_free_mtx);
+#if VM_NRESERVLEVEL > 0
+retry:
+#endif
 	for (flind = 0; flind < vm_nfreelists; flind++) {
 		for (oind = min(order, VM_NFREEORDER - 1); oind < VM_NFREEORDER; oind++) {
 			for (pind = 0; pind < VM_NFREEPOOL; pind++) {
@@ -664,6 +668,10 @@ vm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high,
 			}
 		}
 	}
+#if VM_NRESERVLEVEL > 0
+	if (vm_reserv_reclaim_contig(size, low, high, alignment, boundary))
+		goto retry;
+#endif
 	mtx_unlock(&vm_page_queue_free_mtx);
 	return (NULL);
 done:
diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c
index 3d0690c..4ea9079 100644
--- a/sys/vm/vm_reserv.c
+++ b/sys/vm/vm_reserv.c
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2002-2006 Rice University
- * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
+ * Copyright (c) 2007-2008 Alan L. Cox <alc@cs.rice.edu>
  * All rights reserved.
  *
  * This software was developed for the FreeBSD Project by Alan L. Cox,
@@ -170,6 +170,7 @@ static vm_reserv_t	vm_reserv_from_page(vm_page_t m);
 static boolean_t	vm_reserv_has_pindex(vm_reserv_t rv,
 			    vm_pindex_t pindex);
 static void		vm_reserv_populate(vm_reserv_t rv);
+static void		vm_reserv_reclaim(vm_reserv_t rv);
 
 /*
  * Describes the current state of the partially-populated reservation queue.
@@ -568,6 +569,37 @@ vm_reserv_reactivate_page(vm_page_t m)
 }
 
 /*
+ * Breaks the given partially-populated reservation, releasing its cached and
+ * free pages to the physical memory allocator.
+ *
+ * The free page queue lock must be held.
+ */
+static void
+vm_reserv_reclaim(vm_reserv_t rv)
+{
+	int i;
+
+	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
+	KASSERT(rv->inpartpopq,
+	    ("vm_reserv_reclaim: reserv %p's inpartpopq is corrupted", rv));
+	TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq);
+	rv->inpartpopq = FALSE;
+	KASSERT(rv->object != NULL,
+	    ("vm_reserv_reclaim: reserv %p is free", rv));
+	LIST_REMOVE(rv, objq);
+	rv->object = NULL;
+	for (i = 0; i < VM_LEVEL_0_NPAGES; i++) {
+		if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0)
+			vm_phys_free_pages(&rv->pages[i], 0);
+		else
+			rv->popcnt--;
+	}
+	KASSERT(rv->popcnt == 0,
+	    ("vm_reserv_reclaim: reserv %p's popcnt is corrupted", rv));
+	vm_reserv_reclaimed++;
+}
+
+/*
  * Breaks the reservation at the head of the partially-populated reservation
  * queue, releasing its cached and free pages to the physical memory
  * allocator.  Returns TRUE if a reservation is broken and FALSE otherwise.
@@ -575,38 +607,69 @@ vm_reserv_reactivate_page(vm_page_t m)
  * The free page queue lock must be held.
  */
 boolean_t
-vm_reserv_reclaim(void)
+vm_reserv_reclaim_inactive(void)
 {
 	vm_reserv_t rv;
-	int i;
 
 	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
 	if ((rv = TAILQ_FIRST(&vm_rvq_partpop)) != NULL) {
-		KASSERT(rv->inpartpopq,
-		    ("vm_reserv_reclaim: reserv %p's inpartpopq is corrupted",
-		    rv));
-		TAILQ_REMOVE(&vm_rvq_partpop, rv, partpopq);
-		rv->inpartpopq = FALSE;
-		KASSERT(rv->object != NULL,
-		    ("vm_reserv_reclaim: reserv %p is free", rv));
-		LIST_REMOVE(rv, objq);
-		rv->object = NULL;
-		for (i = 0; i < VM_LEVEL_0_NPAGES; i++) {
-			if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0)
-				vm_phys_free_pages(&rv->pages[i], 0);
-			else
-				rv->popcnt--;
-		}
-		KASSERT(rv->popcnt == 0,
-		    ("vm_reserv_reclaim: reserv %p's popcnt is corrupted",
-		    rv));
-		vm_reserv_reclaimed++;
+		vm_reserv_reclaim(rv);
 		return (TRUE);
 	}
 	return (FALSE);
 }
 
 /*
+ * Searches the partially-populated reservation queue for the least recently
+ * active reservation with unused pages, i.e., cached or free, that satisfy the
+ * given request for contiguous physical memory.  If a satisfactory reservation
+ * is found, it is broken.  Returns TRUE if a reservation is broken and FALSE
+ * otherwise.
+ *
+ * The free page queue lock must be held.
+ */
+boolean_t
+vm_reserv_reclaim_contig(vm_paddr_t size, vm_paddr_t low, vm_paddr_t high,
+    unsigned long alignment, unsigned long boundary)
+{
+	vm_paddr_t pa, pa_length;
+	vm_reserv_t rv;
+	int i;
+
+	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
+	if (size > VM_LEVEL_0_SIZE - PAGE_SIZE)
+		return (FALSE);
+	TAILQ_FOREACH(rv, &vm_rvq_partpop, partpopq) {
+		pa = VM_PAGE_TO_PHYS(&rv->pages[VM_LEVEL_0_NPAGES - 1]);
+		if (pa + PAGE_SIZE - size < low) {
+			/* this entire reservation is too low; go to next */
+			continue;
+		}
+		pa_length = 0;
+		for (i = 0; i < VM_LEVEL_0_NPAGES; i++)
+			if ((rv->pages[i].flags & (PG_CACHED | PG_FREE)) != 0) {
+				pa_length += PAGE_SIZE;
+				if (pa_length == PAGE_SIZE) {
+					pa = VM_PAGE_TO_PHYS(&rv->pages[i]);
+					if (pa + size > high) {
+						/* skip to next reservation */
+						break;
+					} else if (pa < low ||
+					    (pa & (alignment - 1)) != 0 ||
+					    ((pa ^ (pa + size - 1)) &
+					    ~(boundary - 1)) != 0)
+						pa_length = 0;
+				} else if (pa_length >= size) {
+					vm_reserv_reclaim(rv);
+					return (TRUE);
+				}
+			} else
+				pa_length = 0;
+	}
+	return (FALSE);
+}
+
+/*
  * Transfers the reservation underlying the given page to a new object.
  *
  * The object must be locked.
diff --git a/sys/vm/vm_reserv.h b/sys/vm/vm_reserv.h
index 7cf814b..9eb1d06 100644
--- a/sys/vm/vm_reserv.h
+++ b/sys/vm/vm_reserv.h
@@ -1,6 +1,6 @@
 /*-
  * Copyright (c) 2002-2006 Rice University
- * Copyright (c) 2007 Alan L. Cox <alc@cs.rice.edu>
+ * Copyright (c) 2007-2008 Alan L. Cox <alc@cs.rice.edu>
  * All rights reserved.
  *
  * This software was developed for the FreeBSD Project by Alan L. Cox,
@@ -48,7 +48,10 @@ boolean_t	vm_reserv_free_page(vm_page_t m);
 void		vm_reserv_init(void);
 int		vm_reserv_level_iffullpop(vm_page_t m);
 boolean_t	vm_reserv_reactivate_page(vm_page_t m);
-boolean_t	vm_reserv_reclaim(void);
+boolean_t	vm_reserv_reclaim_contig(vm_paddr_t size, vm_paddr_t low,
+		    vm_paddr_t high, unsigned long alignment,
+		    unsigned long boundary);
+boolean_t	vm_reserv_reclaim_inactive(void);
 void		vm_reserv_rename(vm_page_t m, vm_object_t new_object,
 		    vm_object_t old_object, vm_pindex_t old_object_offset);
 vm_paddr_t	vm_reserv_startup(vm_offset_t *vaddr, vm_paddr_t end,