Merge from vmc-playground:

Introduce a new KPI that verifies if the page cache is empty for a
specified vm_object.  This KPI does not make assumptions about the
locking in order to be used also for building assertions at init and
destroy time.
It is mostly used to hide implementation details of the page cache.

Sponsored by:	EMC / Isilon storage division
Reviewed by:	jeff
Reviewed by:	alc (vm_radix based version)
Tested by:	flo, pho, jhb, davide

3 files changed, 22 insertions, 12 deletions

diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 1e588e3..372782b 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -173,7 +173,7 @@ vm_object_zdtor(void *mem, int size, void *arg)
 	    ("object %p has reservations",
 	    object));
 #endif
-	KASSERT(object->cache == NULL,
+	KASSERT(vm_object_cache_is_empty(object),
 	    ("object %p has cached pages",
 	    object));
 	KASSERT(object->paging_in_progress == 0,
@@ -752,7 +752,7 @@ vm_object_terminate(vm_object_t object)
 	if (__predict_false(!LIST_EMPTY(&object->rvq)))
 		vm_reserv_break_all(object);
 #endif
-	if (__predict_false(object->cache != NULL))
+	if (__predict_false(!vm_object_cache_is_empty(object)))
 		vm_page_cache_free(object, 0, 0);
 
 	/*
@@ -1377,7 +1377,7 @@ retry:
 		 * should still be OBJT_DEFAULT and orig_object should not
 		 * contain any cached pages within the specified range.
 		 */
-		if (__predict_false(orig_object->cache != NULL))
+		if (__predict_false(!vm_object_cache_is_empty(orig_object)))
 			vm_page_cache_transfer(orig_object, offidxstart,
 			    new_object);
 	}
@@ -1726,7 +1726,8 @@ vm_object_collapse(vm_object_t object)
 				/*
 				 * Free any cached pages from backing_object.
 				 */
-				if (__predict_false(backing_object->cache != NULL))
+				if (__predict_false(
+				    !vm_object_cache_is_empty(backing_object)))
 					vm_page_cache_free(backing_object, 0, 0);
 			}
 			/*
@@ -1920,7 +1921,7 @@ again:
 	}
 	vm_object_pip_wakeup(object);
 skipmemq:
-	if (__predict_false(object->cache != NULL))
+	if (__predict_false(!vm_object_cache_is_empty(object)))
 		vm_page_cache_free(object, start, end);
 }
 
diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h
index 76caa75..319324f 100644
--- a/sys/vm/vm_object.h
+++ b/sys/vm/vm_object.h
@@ -229,6 +229,13 @@ void vm_object_pip_wakeup(vm_object_t object);
 void vm_object_pip_wakeupn(vm_object_t object, short i);
 void vm_object_pip_wait(vm_object_t object, char *waitid);
 
+static __inline boolean_t
+vm_object_cache_is_empty(vm_object_t object)
+{
+
+	return (object->cache == NULL);
+}
+
 vm_object_t vm_object_allocate (objtype_t, vm_pindex_t);
 boolean_t vm_object_coalesce(vm_object_t, vm_ooffset_t, vm_size_t, vm_size_t,
    boolean_t);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 44f7193..d7190f1 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1129,7 +1129,7 @@ vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
 	boolean_t empty;
 
 	mtx_lock(&vm_page_queue_free_mtx);
-	if (__predict_false(object->cache == NULL)) {
+	if (__predict_false(vm_object_cache_is_empty(object))) {
 		mtx_unlock(&vm_page_queue_free_mtx);
 		return;
 	}
@@ -1173,7 +1173,7 @@ vm_page_cache_free(vm_object_t object, vm_pindex_t start, vm_pindex_t end)
 		cnt.v_cache_count--;
 		cnt.v_free_count++;
 	}
-	empty = object->cache == NULL;
+	empty = vm_object_cache_is_empty(object);
 	mtx_unlock(&vm_page_queue_free_mtx);
 	if (object->type == OBJT_VNODE && empty)
 		vdrop(object->handle);
@@ -1257,7 +1257,7 @@ vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart,
 	 * not.
 	 */
 	VM_OBJECT_LOCK_ASSERT(new_object, MA_OWNED);
-	KASSERT(new_object->cache == NULL,
+	KASSERT(vm_object_cache_is_empty(new_object),
 	    ("vm_page_cache_transfer: object %p has cached pages",
 	    new_object));
 	mtx_lock(&vm_page_queue_free_mtx);
@@ -1300,7 +1300,7 @@ vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart,
 			m_next->left = m;
 			new_object->cache = m_next;
 		}
-		KASSERT(new_object->cache == NULL ||
+		KASSERT(vm_object_cache_is_empty(new_object) ||
 		    new_object->type == OBJT_SWAP,
 		    ("vm_page_cache_transfer: object %p's type is incompatible"
 		    " with cached pages", new_object));
@@ -1327,7 +1327,7 @@ vm_page_is_cached(vm_object_t object, vm_pindex_t pindex)
 	 * exist.
 	 */
 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
-	if (__predict_true(object->cache == NULL))
+	if (__predict_true(vm_object_cache_is_empty(object)))
 		return (FALSE);
 	mtx_lock(&vm_page_queue_free_mtx);
 	m = vm_page_cache_lookup(object, pindex);
@@ -1465,7 +1465,8 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req)
 			m->valid = 0;
 		m_object = m->object;
 		vm_page_cache_remove(m);
-		if (m_object->type == OBJT_VNODE && m_object->cache == NULL)
+		if (m_object->type == OBJT_VNODE &&
+		    vm_object_cache_is_empty(m_object))
 			vp = m_object->handle;
 	} else {
 		KASSERT(VM_PAGE_IS_FREE(m),
@@ -1722,7 +1723,8 @@ vm_page_alloc_init(vm_page_t m)
 		m->valid = 0;
 		m_object = m->object;
 		vm_page_cache_remove(m);
-		if (m_object->type == OBJT_VNODE && m_object->cache == NULL)
+		if (m_object->type == OBJT_VNODE &&
+		    vm_object_cache_is_empty(m_object))
 			drop = m_object->handle;
 	} else {
 		KASSERT(VM_PAGE_IS_FREE(m),