Reduce diffs against HEAD:

Reimplement vm_page_cache_free() as a range operation.

5 files changed, 57 insertions, 140 deletions

diff --git a/sys/fs/tmpfs/tmpfs_vnops.c b/sys/fs/tmpfs/tmpfs_vnops.c
index bedf774..09eff14 100644
--- a/sys/fs/tmpfs/tmpfs_vnops.c
+++ b/sys/fs/tmpfs/tmpfs_vnops.c
@@ -670,13 +670,8 @@ lookupvpg:
 		VM_OBJECT_UNLOCK(vobj);
 		error = uiomove_fromphys(&vpg, offset, tlen, uio);
 	} else {
-		vpg = vm_page_is_cached(vobj, idx);
-		if (vpg != NULL) {
-			mtx_lock(&vm_page_queue_free_mtx);
-			if (vpg->object == vobj)
-				vm_page_cache_free(vpg);
-			mtx_unlock(&vm_page_queue_free_mtx);
-		}
+		if (vm_page_is_cached(vobj, idx))
+			vm_page_cache_free(vobj, idx, idx + 1);
 		VM_OBJECT_UNLOCK(vobj);
 		vpg = NULL;
 	}
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 971b6ae..f997d08 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -674,8 +674,6 @@ void
 vm_object_terminate(vm_object_t object)
 {
 	vm_page_t p, p_next;
-	vm_pindex_t start;
-	struct vnode *vp;
 
 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 
@@ -739,38 +737,6 @@ vm_object_terminate(vm_object_t object)
 		vm_page_unlock(p);
 	}
 	vm_radix_reclaim_allnodes(&object->rtree);
-	vp = NULL;
-	if (!vm_object_cache_is_empty(object)) {
-		start = 0;
-		mtx_lock(&vm_page_queue_free_mtx);
-		while ((p = vm_radix_lookup_ge(&object->cache,
-		    start)) != NULL) {
-			MPASS(p->object == object);
-			p->object = NULL;
-			p->valid = 0;
-
-			/* Clear PG_CACHED and set PG_FREE. */
-			p->flags ^= PG_CACHED | PG_FREE;
-			cnt.v_cache_count--;
-			cnt.v_free_count++;
-
-			/*
-			 * At least one cached page was removed and
-			 * in the end all the cached pages will be
-			 * reclaimed.  If the object is a vnode,
-			 * drop a reference to it.
-			 */
-			if (object->type == OBJT_VNODE)
-				vp = object->handle;
-
-			/* Point to the next available index. */
-			start = p->pindex + 1;
-			if (start < p->pindex)
-				break;
-		}
-		vm_radix_reclaim_allnodes(&object->cache);
-		mtx_unlock(&vm_page_queue_free_mtx);
-	}
 	/*
 	 * If the object contained any pages, then reset it to an empty state.
 	 * None of the object's fields, including "resident_page_count", were
@@ -782,13 +748,13 @@ vm_object_terminate(vm_object_t object)
 		if (object->type == OBJT_VNODE)
 			vdrop(object->handle);
 	}
-	if (vp)
-		vdrop(vp);
 
 #if VM_NRESERVLEVEL > 0
 	if (__predict_false(!LIST_EMPTY(&object->rvq)))
 		vm_reserv_break_all(object);
 #endif
+	if (!vm_object_cache_is_empty(object))
+		vm_page_cache_free(object, 0, 0);
 
 	/*
 	 * Let the pager know object is dead.
@@ -1679,9 +1645,6 @@ vm_object_qcollapse(vm_object_t object)
 void
 vm_object_collapse(vm_object_t object)
 {
-	vm_page_t p;
-	vm_pindex_t start, tmpindex;
-
 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 	
 	while (TRUE) {
@@ -1762,30 +1725,11 @@ vm_object_collapse(vm_object_t object)
 				    object,
 				    OFF_TO_IDX(object->backing_object_offset), TRUE);
 
-				if (!vm_object_cache_is_empty(backing_object)) {
-
-					/*
-					 * Free any cached pages from
-					 * backing_object.
-					 */
-					start = 0;
-					mtx_lock(&vm_page_queue_free_mtx);
-					while ((p =
-				    vm_radix_lookup_ge(&backing_object->cache,
-					    start)) != NULL) {
-						tmpindex = p->pindex;
-						vm_page_cache_free(p);
-
-						/*
-						 * Point to the next available
-						 * index.
-						 */
-						start = tmpindex + 1;
-						if (start < tmpindex)
-							break;
-					}
-					mtx_unlock(&vm_page_queue_free_mtx);
-				}
+				/*
+				 * Free any cached pages from backing_object.
+				 */
+				if (!vm_object_cache_is_empty(object))
+					vm_page_cache_free(backing_object, 0, 0);
 			}
 			/*
 			 * Object now shadows whatever backing_object did.
@@ -1906,7 +1850,6 @@ void
 vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
     int options)
 {
-	struct vnode *vp;
 	vm_page_t p, next;
 	int wirings;
 
@@ -1914,11 +1857,8 @@ vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
 	KASSERT((object->flags & OBJ_UNMANAGED) == 0 ||
 	    (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED,
 	    ("vm_object_page_remove: illegal options for object %p", object));
-	if (object->resident_page_count == 0) {
-		if (vm_object_cache_is_empty(object))
-			return;
+	if (object->resident_page_count == 0)
 		goto skipmemq;
-	}
 	vm_object_pip_add(object, 1);
 again:
 	p = vm_page_find_least(object, start);
@@ -1990,22 +1930,8 @@ again:
 	}
 	vm_object_pip_wakeup(object);
 skipmemq:
-	vp = NULL;
-	if (!vm_object_cache_is_empty(object)) {
-		mtx_lock(&vm_page_queue_free_mtx);
-		while ((p = vm_radix_lookup_ge(&object->cache,
-		    start)) != NULL) {
-			if (p->pindex >= end)
-				break;
-			vm_page_cache_free(p);
-			if (vm_object_cache_is_empty(object) &&
-			    object->type == OBJT_VNODE)
-				vp = object->handle;
-		}
-		mtx_unlock(&vm_page_queue_free_mtx);
-	}
-	if (vp)
-		vdrop(vp);
+	if (!vm_object_cache_is_empty(object))
+		vm_page_cache_free(object, start, end);
 }
 
 /*
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index f9c8e9c..02c7de1 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -999,6 +999,47 @@ vm_page_rename(vm_page_t m, vm_object_t new_object, vm_pindex_t new_pindex)
 }
 
 /*
+ *	Convert all of the given object's cached pages that have a
+ *	pindex within the given range into free pages.  If the value
+ *	zero is given for "end", then the range's upper bound is
+ *	infinity.  If the given object is backed by a vnode and it
+ *	transitions from having one or more cached pages to none, the
+ *	vnode's hold count is reduced.
+ */
+void
+vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
+{
+	vm_page_t m;
+	boolean_t empty;
+
+	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+
+	mtx_lock(&vm_page_queue_free_mtx);
+	if (vm_object_cache_is_empty(object)) {
+		mtx_unlock(&vm_page_queue_free_mtx);
+		return;
+	}
+	while ((m = vm_radix_lookup_ge(&object->cache, start)) != NULL) {
+		if (end != 0 && m->pindex >= end)
+			break;
+		vm_radix_remove(&object->cache, m->pindex);
+		m->object = NULL;
+		m->valid = 0;
+
+		/* Clear PG_CACHED and set PG_FREE. */
+		m->flags ^= PG_CACHED | PG_FREE;
+		KASSERT((m->flags & (PG_CACHED | PG_FREE)) == PG_FREE,
+		    ("vm_page_cache_free: page %p has inconsistent flags", m));
+		cnt.v_cache_count--;
+		cnt.v_free_count++;
+	}
+	empty = vm_object_cache_is_empty(object);
+	mtx_unlock(&vm_page_queue_free_mtx);
+	if (object->type == OBJT_VNODE && empty)
+		vdrop(object->handle);
+}
+
+/*
  *	Returns the cached page that is associated with the given
  *	object and offset.  If, however, none exists, returns NULL.
  *
@@ -1034,36 +1075,6 @@ vm_page_cache_remove(vm_page_t m)
 }
 
 /*
- *	Move a given cached page from an object's cached pages to
- *	the free list.
- *
- *	The free page queue mtx and object lock must be locked.
- */
-void
-vm_page_cache_free(vm_page_t m)
-{
-	vm_object_t object;
-
-	object = m->object;
-	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
-	mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
-	KASSERT((m->flags & PG_CACHED) != 0,
-	    ("vm_page_cache_free: page %p is not cached", m));
-
-	/*
-	 * Replicate vm_page_cache_remove with a version that can collapse
-	 * internal nodes since the object lock is held.
-	 */
-	vm_radix_remove(&object->cache, m->pindex);
-	m->object = NULL;
-	m->valid = 0;
-	/* Clear PG_CACHED and set PG_FREE. */
-	m->flags ^= PG_CACHED | PG_FREE;
-	cnt.v_cache_count--;
-	cnt.v_free_count++;
-}
-
-/*
  *	Transfer all of the cached pages with offset greater than or
  *	equal to 'offidxstart' from the original object's cache to the
  *	new object's cache.  However, any cached pages with offset
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 888aa08..6e0141e 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -376,8 +376,8 @@ vm_page_t vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
 vm_page_t vm_page_alloc_freelist(int, int);
 vm_page_t vm_page_grab (vm_object_t, vm_pindex_t, int);
 void vm_page_cache(vm_page_t);
+void vm_page_cache_free(vm_object_t, vm_pindex_t, vm_pindex_t);
 void vm_page_cache_transfer(vm_object_t, vm_pindex_t, vm_object_t);
-void vm_page_cache_free(vm_page_t);
 int vm_page_try_to_cache (vm_page_t);
 int vm_page_try_to_free (vm_page_t);
 void vm_page_dontneed(vm_page_t);
diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c
index 0f7f3cd..a6d78f4 100644
--- a/sys/vm/vnode_pager.c
+++ b/sys/vm/vnode_pager.c
@@ -373,7 +373,6 @@ vnode_pager_setsize(vp, nsize)
 	vm_ooffset_t nsize;
 {
 	vm_object_t object;
-	struct vnode *drop;
 	vm_page_t m;
 	vm_pindex_t nobjsize;
 
@@ -439,23 +438,9 @@ vnode_pager_setsize(vp, nsize)
 			 */
 			vm_page_clear_dirty(m, base, PAGE_SIZE - base);
 		} else if ((nsize & PAGE_MASK) &&
-		    (m = vm_page_is_cached(object,
-		    OFF_TO_IDX(nsize))) != NULL) {
-			drop = NULL;
-			mtx_lock(&vm_page_queue_free_mtx);
-			if (m->object == object) {
-
-				/*
-				 * Eliminate any cached page as we would have
-				 * to do too much work to save it.
-				 */
-				vm_page_cache_free(m);
-				if (vm_object_cache_is_empty(object))
-					drop = vp;
-			}
-			mtx_unlock(&vm_page_queue_free_mtx);
-			if (drop)
-				vdrop(drop);
+		    vm_page_is_cached(object, OFF_TO_IDX(nsize))) {
+			vm_page_cache_free(object, OFF_TO_IDX(nsize),
+			    nobjsize);
 		}
 	}
 	object->un_pager.vnp.vnp_size = nsize;