On Alan's advice, rather than do a wholesale conversion on a single

architecture from page queue lock to a hashed array of page locks
(based on a patch by Jeff Roberson), I've implemented page lock
support in the MI code and have only moved vm_page's hold_count
out from under page queue mutex to page lock. This changes
pmap_extract_and_hold on all pmaps.

Supported by: Bitgravity Inc.

Discussed with: alc, jeffr, and kib

1 files changed, 75 insertions, 11 deletions

diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index 258da8b..95517de 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -252,7 +252,9 @@ vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next)
 	TAILQ_INSERT_AFTER(&vm_page_queues[queue].pl,
 			   m, &marker, pageq);
 	vm_page_unlock_queues();
+	vm_page_unlock(m);
 	VM_OBJECT_LOCK(object);
+	vm_page_lock(m);
 	vm_page_lock_queues();
 
 	/* Page queue might have changed. */
@@ -275,8 +277,7 @@ vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next)
  * late and we cannot do anything that will mess with the page.
  */
 static int
-vm_pageout_clean(m)
-	vm_page_t m;
+vm_pageout_clean(vm_page_t m)
 {
 	vm_object_t object;
 	vm_page_t mc[2*vm_pageout_page_count];
@@ -284,7 +285,8 @@ vm_pageout_clean(m)
 	int ib, is, page_base;
 	vm_pindex_t pindex = m->pindex;
 
-	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
+	vm_page_lock_assert(m, MA_NOTOWNED);
+	vm_page_lock(m);
 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 
 	/*
@@ -301,6 +303,7 @@ vm_pageout_clean(m)
 	 */
 	if ((m->hold_count != 0) ||
 	    ((m->busy != 0) || (m->oflags & VPO_BUSY))) {
+		vm_page_unlock(m);
 		return 0;
 	}
 
@@ -347,13 +350,19 @@ more:
 			ib = 0;
 			break;
 		}
+		vm_page_lock(p);
+		vm_page_lock_queues();
 		vm_page_test_dirty(p);
 		if (p->dirty == 0 ||
 		    p->queue != PQ_INACTIVE ||
 		    p->hold_count != 0) {	/* may be undergoing I/O */
+			vm_page_unlock(p);
+			vm_page_unlock_queues();
 			ib = 0;
 			break;
 		}
+		vm_page_unlock_queues();
+		vm_page_unlock(p);
 		mc[--page_base] = p;
 		++pageout_count;
 		++ib;
@@ -374,12 +383,18 @@ more:
 		if ((p->oflags & VPO_BUSY) || p->busy) {
 			break;
 		}
+		vm_page_lock(p);
+		vm_page_lock_queues();
 		vm_page_test_dirty(p);
 		if (p->dirty == 0 ||
 		    p->queue != PQ_INACTIVE ||
 		    p->hold_count != 0) {	/* may be undergoing I/O */
+			vm_page_unlock_queues();
+			vm_page_unlock(p);
 			break;
 		}
+		vm_page_unlock_queues();
+		vm_page_unlock(p);
 		mc[page_base + pageout_count] = p;
 		++pageout_count;
 		++is;
@@ -393,6 +408,7 @@ more:
 	if (ib && pageout_count < vm_pageout_page_count)
 		goto more;
 
+	vm_page_unlock(m);
 	/*
 	 * we allow reads during pageouts...
 	 */
@@ -416,7 +432,6 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags)
 	int numpagedout = 0;
 	int i;
 
-	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 	/*
 	 * Initiate I/O.  Bump the vm_page_t->busy counter and
@@ -433,17 +448,21 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags)
 		    ("vm_pageout_flush: partially invalid page %p index %d/%d",
 			mc[i], i, count));
 		vm_page_io_start(mc[i]);
+		vm_page_lock(mc[i]);
+		vm_page_lock_queues();
 		pmap_remove_write(mc[i]);
+		vm_page_unlock(mc[i]);
+		vm_page_unlock_queues();
 	}
-	vm_page_unlock_queues();
 	vm_object_pip_add(object, count);
 
 	vm_pager_put_pages(object, mc, count, flags, pageout_status);
 
-	vm_page_lock_queues();
 	for (i = 0; i < count; i++) {
 		vm_page_t mt = mc[i];
 
+		vm_page_lock(mt);
+		vm_page_lock_queues();
 		KASSERT(pageout_status[i] == VM_PAGER_PEND ||
 		    (mt->flags & PG_WRITEABLE) == 0,
 		    ("vm_pageout_flush: page %p is not write protected", mt));
@@ -485,6 +504,8 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags)
 			if (vm_page_count_severe())
 				vm_page_try_to_cache(mt);
 		}
+		vm_page_unlock_queues();
+		vm_page_unlock(mt);
 	}
 	return numpagedout;
 }
@@ -527,10 +548,12 @@ vm_pageout_object_deactivate_pages(pmap, first_object, desired)
 		 * scan the objects entire memory queue
 		 */
 		p = TAILQ_FIRST(&object->memq);
-		vm_page_lock_queues();
 		while (p != NULL) {
+			vm_page_lock(p);
+			vm_page_lock_queues();
 			if (pmap_resident_count(pmap) <= desired) {
 				vm_page_unlock_queues();
+				vm_page_unlock(p);
 				goto unlock_return;
 			}
 			next = TAILQ_NEXT(p, listq);
@@ -540,6 +563,8 @@ vm_pageout_object_deactivate_pages(pmap, first_object, desired)
 			    p->busy != 0 ||
 			    (p->oflags & VPO_BUSY) ||
 			    !pmap_page_exists_quick(pmap, p)) {
+				vm_page_unlock_queues();
+				vm_page_unlock(p);
 				p = next;
 				continue;
 			}
@@ -573,9 +598,10 @@ vm_pageout_object_deactivate_pages(pmap, first_object, desired)
 			} else if (p->queue == PQ_INACTIVE) {
 				pmap_remove_all(p);
 			}
+			vm_page_unlock_queues();
+			vm_page_unlock(p);
 			p = next;
 		}
-		vm_page_unlock_queues();
 		if ((backing_object = object->backing_object) == NULL)
 			goto unlock_return;
 		VM_OBJECT_LOCK(backing_object);
@@ -742,7 +768,6 @@ rescan0:
 		}
 
 		next = TAILQ_NEXT(m, pageq);
-		object = m->object;
 
 		/*
 		 * skip marker pages
@@ -758,18 +783,26 @@ rescan0:
 			addl_page_shortage++;
 			continue;
 		}
+
+		if (!vm_page_trylock(m) || (object = m->object) == NULL) {
+			addl_page_shortage++;
+			continue;
+		}
+
 		/*
 		 * Don't mess with busy pages, keep in the front of the
 		 * queue, most likely are being paged out.
 		 */
 		if (!VM_OBJECT_TRYLOCK(object) &&
 		    (!vm_pageout_fallback_object_lock(m, &next) ||
-		     m->hold_count != 0)) {
+			m->hold_count != 0)) {
 			VM_OBJECT_UNLOCK(object);
+			vm_page_unlock(m);
 			addl_page_shortage++;
 			continue;
 		}
 		if (m->busy || (m->oflags & VPO_BUSY)) {
+			vm_page_unlock(m);
 			VM_OBJECT_UNLOCK(object);
 			addl_page_shortage++;
 			continue;
@@ -798,6 +831,7 @@ rescan0:
 			vm_page_activate(m);
 			VM_OBJECT_UNLOCK(object);
 			m->act_count += (actcount + ACT_ADVANCE);
+			vm_page_unlock(m);
 			continue;
 		}
 
@@ -813,6 +847,7 @@ rescan0:
 			vm_page_activate(m);
 			VM_OBJECT_UNLOCK(object);
 			m->act_count += (actcount + ACT_ADVANCE + 1);
+			vm_page_unlock(m);
 			continue;
 		}
 
@@ -898,6 +933,7 @@ rescan0:
 			 * Those objects are in a "rundown" state.
 			 */
 			if (!swap_pageouts_ok || (object->flags & OBJ_DEAD)) {
+				vm_page_unlock(m);
 				VM_OBJECT_UNLOCK(object);
 				vm_page_requeue(m);
 				continue;
@@ -937,6 +973,8 @@ rescan0:
 			 * of time.
 			 */
 			if (object->type == OBJT_VNODE) {
+				vm_page_unlock_queues();
+				vm_page_unlock(m);
 				vp = object->handle;
 				if (vp->v_type == VREG &&
 				    vn_start_write(vp, &mp, V_NOWAIT) != 0) {
@@ -944,11 +982,11 @@ rescan0:
 					++pageout_lock_miss;
 					if (object->flags & OBJ_MIGHTBEDIRTY)
 						vnodes_skipped++;
+					vm_page_lock_queues();
 					goto unlock_and_continue;
 				}
 				KASSERT(mp != NULL,
 				    ("vp %p with NULL v_mount", vp));
-				vm_page_unlock_queues();
 				vm_object_reference_locked(object);
 				VM_OBJECT_UNLOCK(object);
 				vfslocked = VFS_LOCK_GIANT(vp->v_mount);
@@ -963,6 +1001,7 @@ rescan0:
 					goto unlock_and_continue;
 				}
 				VM_OBJECT_LOCK(object);
+				vm_page_lock(m);
 				vm_page_lock_queues();
 				/*
 				 * The page might have been moved to another
@@ -973,6 +1012,7 @@ rescan0:
 				if (VM_PAGE_GETQUEUE(m) != PQ_INACTIVE ||
 				    m->object != object ||
 				    TAILQ_NEXT(m, pageq) != &marker) {
+					vm_page_unlock(m);
 					if (object->flags & OBJ_MIGHTBEDIRTY)
 						vnodes_skipped++;
 					goto unlock_and_continue;
@@ -985,6 +1025,7 @@ rescan0:
 				 * statistics are more correct if we don't.
 				 */
 				if (m->busy || (m->oflags & VPO_BUSY)) {
+					vm_page_unlock(m);
 					goto unlock_and_continue;
 				}
 
@@ -993,12 +1034,14 @@ rescan0:
 				 * be undergoing I/O, so skip it
 				 */
 				if (m->hold_count) {
+					vm_page_unlock(m);
 					vm_page_requeue(m);
 					if (object->flags & OBJ_MIGHTBEDIRTY)
 						vnodes_skipped++;
 					goto unlock_and_continue;
 				}
 			}
+			vm_page_unlock(m);
 
 			/*
 			 * If a page is dirty, then it is either being washed
@@ -1010,11 +1053,14 @@ rescan0:
 			 * the (future) cleaned page.  Otherwise we could wind
 			 * up laundering or cleaning too many pages.
 			 */
+			vm_page_unlock_queues();
 			if (vm_pageout_clean(m) != 0) {
 				--page_shortage;
 				--maxlaunder;
 			}
+			vm_page_lock_queues();
 unlock_and_continue:
+			vm_page_lock_assert(m, MA_NOTOWNED);
 			VM_OBJECT_UNLOCK(object);
 			if (mp != NULL) {
 				vm_page_unlock_queues();
@@ -1028,8 +1074,10 @@ unlock_and_continue:
 			next = TAILQ_NEXT(&marker, pageq);
 			TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl,
 				     &marker, pageq);
+			vm_page_lock_assert(m, MA_NOTOWNED);
 			continue;
 		}
+		vm_page_unlock(m);
 		VM_OBJECT_UNLOCK(object);
 	}
 
@@ -1048,6 +1096,7 @@ unlock_and_continue:
 	 */
 	pcount = cnt.v_active_count;
 	m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
+	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 
 	while ((m != NULL) && (pcount-- > 0) && (page_shortage > 0)) {
 
@@ -1060,9 +1109,14 @@ unlock_and_continue:
 			m = next;
 			continue;
 		}
+		if (!vm_page_trylock(m) || (object = m->object) == NULL) {
+			m = next;
+			continue;
+		}
 		if (!VM_OBJECT_TRYLOCK(object) &&
 		    !vm_pageout_fallback_object_lock(m, &next)) {
 			VM_OBJECT_UNLOCK(object);
+			vm_page_unlock(m);
 			m = next;
 			continue;
 		}
@@ -1073,6 +1127,7 @@ unlock_and_continue:
 		if ((m->busy != 0) ||
 		    (m->oflags & VPO_BUSY) ||
 		    (m->hold_count != 0)) {
+			vm_page_unlock(m);
 			VM_OBJECT_UNLOCK(object);
 			vm_page_requeue(m);
 			m = next;
@@ -1132,6 +1187,7 @@ unlock_and_continue:
 				vm_page_requeue(m);
 			}
 		}
+		vm_page_unlock(m);
 		VM_OBJECT_UNLOCK(object);
 		m = next;
 	}
@@ -1317,9 +1373,15 @@ vm_pageout_page_stats()
 			m = next;
 			continue;
 		}
+		vm_page_lock_assert(m, MA_NOTOWNED);
+		if (vm_page_trylock(m) == 0 || (object = m->object) == NULL) {
+			m = next;
+			continue;
+		}
 		if (!VM_OBJECT_TRYLOCK(object) &&
 		    !vm_pageout_fallback_object_lock(m, &next)) {
 			VM_OBJECT_UNLOCK(object);
+			vm_page_unlock(m);
 			m = next;
 			continue;
 		}
@@ -1330,6 +1392,7 @@ vm_pageout_page_stats()
 		if ((m->busy != 0) ||
 		    (m->oflags & VPO_BUSY) ||
 		    (m->hold_count != 0)) {
+			vm_page_unlock(m);
 			VM_OBJECT_UNLOCK(object);
 			vm_page_requeue(m);
 			m = next;
@@ -1366,6 +1429,7 @@ vm_pageout_page_stats()
 				vm_page_requeue(m);
 			}
 		}
+		vm_page_unlock(m);
 		VM_OBJECT_UNLOCK(object);
 		m = next;
 	}