In the common case, pmap_enter_quick() completes without sleeping.

In such cases, the busying of the page and the unlocking of the
containing object by vm_map_pmap_enter() and vm_fault_prefault() is
unnecessary overhead.  To eliminate this overhead, this change
modifies pmap_enter_quick() so that it expects the object to be locked
on entry and it assumes the responsibility for busying the page and
unlocking the object if it must sleep.  Note: alpha, amd64, i386 and
ia64 are the only implementations optimized by this change; arm,
powerpc, and sparc64 still conservatively busy the page and unlock the
object within every pmap_enter_quick() call.

Additionally, this change is the first case where we synchronize
access to the page's PG_BUSY flag and busy field using the containing
object's lock rather than the global page queues lock.  (Modifications
to the page's PG_BUSY flag and busy field have asserted both locks for
several weeks, enabling an incremental transition.)

11 files changed, 107 insertions, 34 deletions

diff --git a/sys/alpha/alpha/pmap.c b/sys/alpha/alpha/pmap.c
index 789a93a..41a5558 100644
--- a/sys/alpha/alpha/pmap.c
+++ b/sys/alpha/alpha/pmap.c
@@ -153,6 +153,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
+#include <sys/malloc.h>
 #include <sys/proc.h>
 #include <sys/msgbuf.h>
 #include <sys/vmmeter.h>
@@ -334,7 +335,7 @@ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va,
 
 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va);
 
-static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex);
+static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags);
 static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m);
 static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t);
 #ifdef SMP
@@ -1038,24 +1039,28 @@ pmap_pinit(pmap)
  * mapped correctly.
  */
 static vm_page_t
-_pmap_allocpte(pmap, ptepindex)
-	pmap_t	pmap;
-	unsigned ptepindex;
+_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags)
 {
 	pt_entry_t* pte;
 	vm_offset_t ptepa;
 	vm_page_t m;
 
+	KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
+	    (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
+	    ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK"));
+
 	/*
 	 * Find or fabricate a new pagetable page
 	 */
 	if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ |
 	    VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) {
-		PMAP_UNLOCK(pmap);
-		vm_page_unlock_queues();
-		VM_WAIT;
-		vm_page_lock_queues();
-		PMAP_LOCK(pmap);
+		if (flags & M_WAITOK) {
+			PMAP_UNLOCK(pmap);
+			vm_page_unlock_queues();
+			VM_WAIT;
+			vm_page_lock_queues();
+			PMAP_LOCK(pmap);
+		}
 
 		/*
 		 * Indicate the need to retry.  While waiting, the page table
@@ -1082,7 +1087,8 @@ _pmap_allocpte(pmap, ptepindex)
 		pt_entry_t* l1pte = &pmap->pm_lev1[l1index];
 		pt_entry_t* l2map;
 		if (!pmap_pte_v(l1pte)) {
-			if (_pmap_allocpte(pmap, NUSERLEV3MAPS + l1index) == NULL) {
+			if (_pmap_allocpte(pmap, NUSERLEV3MAPS + l1index,
+			    flags) == NULL) {
 				--m->wire_count;
 				vm_page_free(m);
 				return (NULL);
@@ -1146,7 +1152,7 @@ retry:
 		 * Here if the pte page isn't mapped, or if it has been
 		 * deallocated.
 		 */
-		m = _pmap_allocpte(pmap, ptepindex);
+		m = _pmap_allocpte(pmap, ptepindex, M_WAITOK);
 		if (m == NULL)
 			goto retry;
 	}
@@ -1842,9 +1848,20 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte)
 				}
 				mpte->wire_count++;
 			} else {
-				mpte = _pmap_allocpte(pmap, ptepindex);
-				if (mpte == NULL)
+				mpte = _pmap_allocpte(pmap, ptepindex,
+				    M_NOWAIT);
+				if (mpte == NULL) {
+					PMAP_UNLOCK(pmap);
+					vm_page_busy(m);
+					vm_page_unlock_queues();
+					VM_OBJECT_UNLOCK(m->object);
+					VM_WAIT;
+					VM_OBJECT_LOCK(m->object);
+					vm_page_lock_queues();
+					vm_page_wakeup(m);
+					PMAP_LOCK(pmap);
 					goto retry;
+				}
 			}
 		}
 	} else {
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index a19c721..e089d62 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -2053,9 +2053,19 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte)
 				mpte->wire_count++;
 			} else {
 				mpte = _pmap_allocpte(pmap, ptepindex,
-				    M_WAITOK);
-				if (mpte == NULL)
+				    M_NOWAIT);
+				if (mpte == NULL) {
+					PMAP_UNLOCK(pmap);
+					vm_page_busy(m);
+					vm_page_unlock_queues();
+					VM_OBJECT_UNLOCK(m->object);
+					VM_WAIT;
+					VM_OBJECT_LOCK(m->object);
+					vm_page_lock_queues();
+					vm_page_wakeup(m);
+					PMAP_LOCK(pmap);
 					goto retry;
+				}
 			}
 		}
 	} else {
diff --git a/sys/arm/arm/pmap.c b/sys/arm/arm/pmap.c
index f9dca09..9159787 100644
--- a/sys/arm/arm/pmap.c
+++ b/sys/arm/arm/pmap.c
@@ -3417,8 +3417,19 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 vm_page_t
 pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte)
 {
+
+	vm_page_lock_queues();
+	vm_page_busy(m);
+	vm_page_unlock_queues();
+	VM_OBJECT_UNLOCK(m->object);
+	mtx_lock(&Giant);
 	pmap_enter(pmap, va, m, VM_PROT_READ|VM_PROT_EXECUTE, FALSE);
 	pmap_dcache_wbinv_all(pmap); /* XXX: shouldn't be needed */
+	mtx_unlock(&Giant);
+	VM_OBJECT_LOCK(m->object);
+	vm_page_lock_queues();
+	vm_page_wakeup(m);
+	vm_page_unlock_queues();
 	return (NULL);
 }
 
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index d16bf02..67ac6f6 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -2094,9 +2094,19 @@ retry:
 				mpte->wire_count++;
 			} else {
 				mpte = _pmap_allocpte(pmap, ptepindex,
-				    M_WAITOK);
-				if (mpte == NULL)
+				    M_NOWAIT);
+				if (mpte == NULL) {
+					PMAP_UNLOCK(pmap);
+					vm_page_busy(m);
+					vm_page_unlock_queues();
+					VM_OBJECT_UNLOCK(m->object);
+					VM_WAIT;
+					VM_OBJECT_LOCK(m->object);
+					vm_page_lock_queues();
+					vm_page_wakeup(m);
+					PMAP_LOCK(pmap);
 					goto retry;
+				}
 			}
 		}
 	} else {
diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c
index da1bcf1..a2feed7 100644
--- a/sys/ia64/ia64/pmap.c
+++ b/sys/ia64/ia64/pmap.c
@@ -1631,9 +1631,13 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte)
 	while ((pte = pmap_find_pte(va)) == NULL) {
 		pmap_install(oldpmap);
 		PMAP_UNLOCK(pmap);
+		vm_page_busy(m);
 		vm_page_unlock_queues();
+		VM_OBJECT_UNLOCK(m->object);
 		VM_WAIT;
+		VM_OBJECT_LOCK(m->object);
 		vm_page_lock_queues();
+		vm_page_wakeup(m);
 		PMAP_LOCK(pmap);
 		oldpmap = pmap_install(pmap);
 	}
diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c
index 5398d61..136e412 100644
--- a/sys/powerpc/aim/mmu_oea.c
+++ b/sys/powerpc/aim/mmu_oea.c
@@ -1048,9 +1048,17 @@ vm_page_t
 pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte)
 {
 
+	vm_page_lock_queues();
+	vm_page_busy(m);
+	vm_page_unlock_queues();
+	VM_OBJECT_UNLOCK(m->object);
 	mtx_lock(&Giant);
 	pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE);
 	mtx_unlock(&Giant);
+	VM_OBJECT_LOCK(m->object);
+	vm_page_lock_queues();
+	vm_page_wakeup(m);
+	vm_page_unlock_queues();
 	return (NULL);
 }
 
diff --git a/sys/powerpc/powerpc/mmu_oea.c b/sys/powerpc/powerpc/mmu_oea.c
index 5398d61..136e412 100644
--- a/sys/powerpc/powerpc/mmu_oea.c
+++ b/sys/powerpc/powerpc/mmu_oea.c
@@ -1048,9 +1048,17 @@ vm_page_t
 pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte)
 {
 
+	vm_page_lock_queues();
+	vm_page_busy(m);
+	vm_page_unlock_queues();
+	VM_OBJECT_UNLOCK(m->object);
 	mtx_lock(&Giant);
 	pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE);
 	mtx_unlock(&Giant);
+	VM_OBJECT_LOCK(m->object);
+	vm_page_lock_queues();
+	vm_page_wakeup(m);
+	vm_page_unlock_queues();
 	return (NULL);
 }
 
diff --git a/sys/powerpc/powerpc/pmap.c b/sys/powerpc/powerpc/pmap.c
index 5398d61..136e412 100644
--- a/sys/powerpc/powerpc/pmap.c
+++ b/sys/powerpc/powerpc/pmap.c
@@ -1048,9 +1048,17 @@ vm_page_t
 pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte)
 {
 
+	vm_page_lock_queues();
+	vm_page_busy(m);
+	vm_page_unlock_queues();
+	VM_OBJECT_UNLOCK(m->object);
 	mtx_lock(&Giant);
 	pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE);
 	mtx_unlock(&Giant);
+	VM_OBJECT_LOCK(m->object);
+	vm_page_lock_queues();
+	vm_page_wakeup(m);
+	vm_page_unlock_queues();
 	return (NULL);
 }
 
diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c
index 18cd0f9..fa964ab 100644
--- a/sys/sparc64/sparc64/pmap.c
+++ b/sys/sparc64/sparc64/pmap.c
@@ -1401,9 +1401,17 @@ vm_page_t
 pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte)
 {
 
+	vm_page_lock_queues();
+	vm_page_busy(m);
+	vm_page_unlock_queues();
+	VM_OBJECT_UNLOCK(m->object);
 	mtx_lock(&Giant);
 	pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE);
 	mtx_unlock(&Giant);
+	VM_OBJECT_LOCK(m->object);
+	vm_page_lock_queues();
+	vm_page_wakeup(m);
+	vm_page_unlock_queues();
 	return (NULL);
 }
 
diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c
index 4bd82f4..6fac17c 100644
--- a/sys/vm/vm_fault.c
+++ b/sys/vm/vm_fault.c
@@ -973,23 +973,17 @@ vm_fault_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry)
 			VM_OBJECT_UNLOCK(lobject);
 			break;
 		}
-		vm_page_lock_queues();
 		if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) &&
 			(m->busy == 0) &&
 		    (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
 
 			if ((m->queue - m->pc) == PQ_CACHE) {
+				vm_page_lock_queues();
 				vm_page_deactivate(m);
+				vm_page_unlock_queues();
 			}
-			vm_page_busy(m);
-			vm_page_unlock_queues();
-			VM_OBJECT_UNLOCK(lobject);
 			mpte = pmap_enter_quick(pmap, addr, m, mpte);
-			VM_OBJECT_LOCK(lobject);
-			vm_page_lock_queues();
-			vm_page_wakeup(m);
 		}
-		vm_page_unlock_queues();
 		VM_OBJECT_UNLOCK(lobject);
 	}
 }
diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index 3788a4f..46ed849 100644
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -1417,22 +1417,17 @@ vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot,
 		    cnt.v_free_count < cnt.v_free_reserved) {
 			break;
 		}
-		vm_page_lock_queues();
 		if ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL &&
 		    (p->busy == 0) &&
 		    (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) {
-			if ((p->queue - p->pc) == PQ_CACHE)
+			if ((p->queue - p->pc) == PQ_CACHE) {
+				vm_page_lock_queues();
 				vm_page_deactivate(p);
-			vm_page_busy(p);
-			vm_page_unlock_queues();
-			VM_OBJECT_UNLOCK(object);
+				vm_page_unlock_queues();
+			}
 			mpte = pmap_enter_quick(map->pmap,
 				addr + ptoa(tmpidx), p, mpte);
-			VM_OBJECT_LOCK(object);
-			vm_page_lock_queues();
-			vm_page_wakeup(p);
 		}
-		vm_page_unlock_queues();
 	}
 unlock_return:
 	VM_OBJECT_UNLOCK(object);