Pages that do belong to an object and page queue can now be freed without

holding the page queues lock.  Thus, the page table pages released by
pmap_remove() and pmap_remove_pages() can be freed after the page queues
lock is released.

Approved by:	re (kensmith)

2 files changed, 16 insertions, 20 deletions

diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index e8cdba8..d70fd4b 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -210,7 +210,8 @@ static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq,
 		vm_offset_t sva, pd_entry_t ptepde, vm_page_t *free);
-static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde);
+static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde,
+    vm_page_t *free);
 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
 		vm_offset_t va);
 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
@@ -1895,10 +1896,9 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va,
  * Remove a single page from a process address space
  */
 static void
-pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde)
+pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, vm_page_t *free)
 {
 	pt_entry_t *pte;
-	vm_page_t free = NULL;
 
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	if ((*pde & PG_V) == 0)
@@ -1906,9 +1906,8 @@ pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde)
 	pte = pmap_pde_to_pte(pde, va);
 	if ((*pte & PG_V) == 0)
 		return;
-	pmap_remove_pte(pmap, pte, va, *pde, &free);
+	pmap_remove_pte(pmap, pte, va, *pde, free);
 	pmap_invalidate_page(pmap, va);
-	pmap_free_zero_pages(free);
 }
 
 /*
@@ -1947,7 +1946,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 	if (sva + PAGE_SIZE == eva) {
 		pde = pmap_pde(pmap, sva);
 		if (pde && (*pde & PG_PS) == 0) {
-			pmap_remove_page(pmap, sva, pde);
+			pmap_remove_page(pmap, sva, pde, &free);
 			goto out;
 		}
 	}
@@ -2018,12 +2017,11 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 		}
 	}
 out:
-	if (anyvalid) {
+	if (anyvalid)
 		pmap_invalidate_all(pmap);
-		pmap_free_zero_pages(free);
-	}
 	vm_page_unlock_queues();	
 	PMAP_UNLOCK(pmap);
+	pmap_free_zero_pages(free);
 }
 
 /*
@@ -3003,9 +3001,9 @@ pmap_remove_pages(pmap_t pmap)
 		}
 	}
 	pmap_invalidate_all(pmap);
-	pmap_free_zero_pages(free);
 	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
+	pmap_free_zero_pages(free);
 }
 
 /*
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index 01b2422..420936d 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -271,7 +271,8 @@ static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
     vm_page_t *free);
-static void pmap_remove_page(struct pmap *pmap, vm_offset_t va);
+static void pmap_remove_page(struct pmap *pmap, vm_offset_t va,
+    vm_page_t *free);
 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
 					vm_offset_t va);
 static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
@@ -1987,19 +1988,17 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, vm_page_t *free)
  * Remove a single page from a process address space
  */
 static void
-pmap_remove_page(pmap_t pmap, vm_offset_t va)
+pmap_remove_page(pmap_t pmap, vm_offset_t va, vm_page_t *free)
 {
 	pt_entry_t *pte;
-	vm_page_t free = NULL;
 
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)
 		return;
-	pmap_remove_pte(pmap, pte, va, &free);
+	pmap_remove_pte(pmap, pte, va, free);
 	pmap_invalidate_page(pmap, va);
-	pmap_free_zero_pages(free);
 }
 
 /*
@@ -2036,7 +2035,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 	 */
 	if ((sva + PAGE_SIZE == eva) && 
 	    ((pmap->pm_pdir[(sva >> PDRSHIFT)] & PG_PS) == 0)) {
-		pmap_remove_page(pmap, sva);
+		pmap_remove_page(pmap, sva, &free);
 		goto out;
 	}
 
@@ -2095,12 +2094,11 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 	}
 out:
 	sched_unpin();
-	if (anyvalid) {
+	if (anyvalid)
 		pmap_invalidate_all(pmap);
-		pmap_free_zero_pages(free);
-	}
 	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
+	pmap_free_zero_pages(free);
 }
 
 /*
@@ -3119,9 +3117,9 @@ pmap_remove_pages(pmap_t pmap)
 	}
 	sched_unpin();
 	pmap_invalidate_all(pmap);
-	pmap_free_zero_pages(free);
 	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
+	pmap_free_zero_pages(free);
 }
 
 /*