Introduce vm_page_next() and vm_page_prev(), and use them in

vm_pageout_clean().  When iterating over a range of pages, these functions
can be cheaper than vm_page_lookup() because their implementation takes
advantage of the vm_object's memq being ordered.

Reviewed by:	kib@
MFC after:	3 weeks

3 files changed, 46 insertions, 13 deletions

diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index fbef16a..56effed 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -879,6 +879,42 @@ vm_page_lookup(vm_object_t object, vm_pindex_t pindex)
 }
 
 /*
+ * Returns the given page's successor (by pindex) within the object if it is
+ * resident; if none is found, NULL is returned.
+ *
+ * The object must be locked.
+ */
+vm_page_t
+vm_page_next(vm_page_t m)
+{
+	vm_page_t next;
+
+	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
+	if ((next = TAILQ_NEXT(m, listq)) != NULL &&
+	    next->pindex != m->pindex + 1)
+		next = NULL;
+	return (next);
+}
+
+/*
+ * Returns the given page's predecessor (by pindex) within the object if it is
+ * resident; if none is found, NULL is returned.
+ *
+ * The object must be locked.
+ */
+vm_page_t
+vm_page_prev(vm_page_t m)
+{
+	vm_page_t prev;
+
+	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
+	if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL &&
+	    prev->pindex != m->pindex - 1)
+		prev = NULL;
+	return (prev);
+}
+
+/*
  *	vm_page_rename:
  *
  *	Move the given memory entry from its
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 3133ee4..ee2f8d8 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -359,7 +359,9 @@ void vm_page_dontneed(vm_page_t);
 void vm_page_deactivate (vm_page_t);
 void vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t);
 vm_page_t vm_page_lookup (vm_object_t, vm_pindex_t);
+vm_page_t vm_page_next(vm_page_t m);
 int vm_page_pa_tryrelock(pmap_t, vm_paddr_t, vm_paddr_t *);
+vm_page_t vm_page_prev(vm_page_t m);
 void vm_page_remove (vm_page_t);
 void vm_page_rename (vm_page_t, vm_object_t, vm_pindex_t);
 void vm_page_requeue(vm_page_t m);
diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index 62163de..cdfddf6 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -320,7 +320,7 @@ static int
 vm_pageout_clean(vm_page_t m)
 {
 	vm_object_t object;
-	vm_page_t mc[2*vm_pageout_page_count];
+	vm_page_t mc[2*vm_pageout_page_count], pb, ps;
 	int pageout_count;
 	int ib, is, page_base;
 	vm_pindex_t pindex = m->pindex;
@@ -347,7 +347,7 @@ vm_pageout_clean(vm_page_t m)
 		return 0;
 	}
 
-	mc[vm_pageout_page_count] = m;
+	mc[vm_pageout_page_count] = pb = ps = m;
 	pageout_count = 1;
 	page_base = vm_pageout_page_count;
 	ib = 1;
@@ -382,11 +382,8 @@ more:
 			break;
 		}
 
-		if ((p = vm_page_lookup(object, pindex - ib)) == NULL) {
-			ib = 0;
-			break;
-		}
-		if ((p->oflags & VPO_BUSY) || p->busy) {
+		if ((p = vm_page_prev(pb)) == NULL ||
+		    (p->oflags & VPO_BUSY) != 0 || p->busy != 0) {
 			ib = 0;
 			break;
 		}
@@ -400,7 +397,7 @@ more:
 			break;
 		}
 		vm_page_unlock(p);
-		mc[--page_base] = p;
+		mc[--page_base] = pb = p;
 		++pageout_count;
 		++ib;
 		/*
@@ -415,11 +412,9 @@ more:
 	    pindex + is < object->size) {
 		vm_page_t p;
 
-		if ((p = vm_page_lookup(object, pindex + is)) == NULL)
+		if ((p = vm_page_next(ps)) == NULL ||
+		    (p->oflags & VPO_BUSY) != 0 || p->busy != 0)
 			break;
-		if ((p->oflags & VPO_BUSY) || p->busy) {
-			break;
-		}
 		vm_page_lock(p);
 		vm_page_test_dirty(p);
 		if (p->dirty == 0 ||
@@ -429,7 +424,7 @@ more:
 			break;
 		}
 		vm_page_unlock(p);
-		mc[page_base + pageout_count] = p;
+		mc[page_base + pageout_count] = ps = p;
 		++pageout_count;
 		++is;
 	}