When sleeping on a busy page, use the lock from the containing object

rather than the global page queues lock.

3 files changed, 15 insertions, 18 deletions

diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index a2f9ff6..8e7b42e 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -1097,19 +1097,19 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage)
 	 * cleared on completion.  If an I/O error occurs, SWAPBLK_NONE
 	 * is set in the meta-data.
 	 */
-	vm_page_lock_queues();
+	VM_OBJECT_LOCK(object);
 	while ((mreq->flags & PG_SWAPINPROG) != 0) {
+		vm_page_lock_queues();
 		vm_page_flag_set(mreq, PG_WANTED | PG_REFERENCED);
+		vm_page_unlock_queues();
 		cnt.v_intrans++;
-		if (msleep(mreq, &vm_page_queue_mtx, PSWP, "swread", hz*20)) {
+		if (msleep(mreq, VM_OBJECT_MTX(object), PSWP, "swread", hz*20)) {
 			printf(
 "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n",
 			    bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount);
 		}
 	}
-	vm_page_unlock_queues();
 
-	VM_OBJECT_LOCK(object);
 	/*
 	 * mreq is left busied after completion, but all the other pages
 	 * are freed.  If we had an unrecoverable read error the page will
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 20cb80d..57a6c49 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -1154,10 +1154,10 @@ shadowlookup:
 		}
 		if ((m->flags & PG_BUSY) || m->busy) {
 			vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
+			vm_page_unlock_queues();
 			if (object != tobject)
 				VM_OBJECT_UNLOCK(object);
-			VM_OBJECT_UNLOCK(tobject);
-			msleep(m, &vm_page_queue_mtx, PDROP | PVM, "madvpo", 0);
+			msleep(m, VM_OBJECT_MTX(tobject), PDROP | PVM, "madvpo", 0);
 			VM_OBJECT_LOCK(object);
   			goto relookup;
 		}
@@ -1342,9 +1342,9 @@ vm_object_split(vm_map_entry_t entry)
 		 */
 		if ((m->flags & PG_BUSY) || m->busy) {
 			vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
-			VM_OBJECT_UNLOCK(orig_object);
+			vm_page_unlock_queues();
 			VM_OBJECT_UNLOCK(new_object);
-			msleep(m, &vm_page_queue_mtx, PDROP | PVM, "spltwt", 0);
+			msleep(m, VM_OBJECT_MTX(orig_object), PDROP | PVM, "spltwt", 0);
 			VM_OBJECT_LOCK(new_object);
 			VM_OBJECT_LOCK(orig_object);
 			vm_page_lock_queues();
@@ -1478,9 +1478,9 @@ vm_object_backing_scan(vm_object_t object, int op)
 					vm_page_lock_queues();
 					vm_page_flag_set(p,
 					    PG_WANTED | PG_REFERENCED);
-					VM_OBJECT_UNLOCK(backing_object);
+					vm_page_unlock_queues();
 					VM_OBJECT_UNLOCK(object);
-					msleep(p, &vm_page_queue_mtx,
+					msleep(p, VM_OBJECT_MTX(backing_object),
 					    PDROP | PVM, "vmocol", 0);
 					VM_OBJECT_LOCK(object);
 					VM_OBJECT_LOCK(backing_object);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index af77ada..5905ee1 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -495,12 +495,13 @@ vm_page_free_zero(vm_page_t m)
 int
 vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
 {
-	vm_object_t object;
 
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
 	if ((m->flags & PG_BUSY) || (also_m_busy && m->busy)) {
 		vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
+		vm_page_unlock_queues();
+
 		/*
 		 * It's possible that while we sleep, the page will get
 		 * unbusied and freed.  If we are holding the object
@@ -508,10 +509,7 @@ vm_page_sleep_if_busy(vm_page_t m, int also_m_busy, const char *msg)
 		 * such that even if m->object changes, we can re-lock
 		 * it.
 		 */
-		object = m->object;
-		VM_OBJECT_UNLOCK(object);
-		msleep(m, &vm_page_queue_mtx, PDROP | PVM, msg, 0);
-		VM_OBJECT_LOCK(object);
+		msleep(m, VM_OBJECT_MTX(m->object), PVM, msg, 0);
 		return (TRUE);
 	}
 	return (FALSE);
@@ -1482,9 +1480,8 @@ retrylookup:
 		vm_page_lock_queues();
 		if (m->busy || (m->flags & PG_BUSY)) {
 			vm_page_flag_set(m, PG_WANTED | PG_REFERENCED);
-			VM_OBJECT_UNLOCK(object);
-			msleep(m, &vm_page_queue_mtx, PDROP | PVM, "pgrbwt", 0);
-			VM_OBJECT_LOCK(object);
+			vm_page_unlock_queues();
+			msleep(m, VM_OBJECT_MTX(m->object), PVM, "pgrbwt", 0);
 			if ((allocflags & VM_ALLOC_RETRY) == 0)
 				return (NULL);
 			goto retrylookup;