Modify TLB invalidation handling.

Reviewed by:	alc@, peter@
MFC after:	1 week

2 files changed, 149 insertions, 60 deletions

diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 3cbf418..8fb06e1 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -209,7 +209,7 @@ static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try);
 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
 static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq,
-		vm_offset_t sva, pd_entry_t ptepde);
+		vm_offset_t sva, pd_entry_t ptepde, vm_page_t *free);
 static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde);
 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
 		vm_offset_t va);
@@ -221,8 +221,9 @@ static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags);
 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
 
 static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags);
-static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m);
-static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t);
+static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m,
+                vm_page_t* free);
+static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *);
 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
 
 CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t));
@@ -1065,24 +1066,36 @@ pmap_qremove(vm_offset_t sva, int count)
 /***************************************************
  * Page table page management routines.....
  ***************************************************/
+static PMAP_INLINE void
+pmap_free_zero_pages(vm_page_t free)
+{
+	vm_page_t m;
+
+	while (free != NULL) {
+		m = free;
+		free = m->right;
+		vm_page_free_zero(m);
+	}
+}
 
 /*
  * This routine unholds page table pages, and if the hold count
  * drops to zero, then it decrements the wire count.
  */
 static PMAP_INLINE int
-pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
+pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *free)
 {
 
 	--m->wire_count;
 	if (m->wire_count == 0)
-		return _pmap_unwire_pte_hold(pmap, va, m);
+		return _pmap_unwire_pte_hold(pmap, va, m, free);
 	else
 		return 0;
 }
 
 static int 
-_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
+_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, 
+    vm_page_t *free)
 {
 	vm_offset_t pteva;
 
@@ -1114,14 +1127,14 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
 		vm_page_t pdpg;
 
 		pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME);
-		pmap_unwire_pte_hold(pmap, va, pdpg);
+		pmap_unwire_pte_hold(pmap, va, pdpg, free);
 	}
 	if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) {
 		/* We just released a PD, unhold the matching PDP */
 		vm_page_t pdppg;
 
 		pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME);
-		pmap_unwire_pte_hold(pmap, va, pdppg);
+		pmap_unwire_pte_hold(pmap, va, pdppg, free);
 	}
 
 	/*
@@ -1130,7 +1143,13 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
 	 */
 	pmap_invalidate_page(pmap, pteva);
 
-	vm_page_free_zero(m);
+	/* 
+	 * Put page on a list so that it is released after
+	 * *ALL* TLB shootdown is done
+	 */
+	m->right = *free;
+	*free = m;
+	
 	atomic_subtract_int(&cnt.v_wire_count, 1);
 	return 1;
 }
@@ -1140,7 +1159,7 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m)
  * conditionally free the page, and manage the hold/wire counts.
  */
 static int
-pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde)
+pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, vm_page_t *free)
 {
 	vm_page_t mpte;
 
@@ -1148,7 +1167,7 @@ pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde)
 		return 0;
 	KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0"));
 	mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
-	return pmap_unwire_pte_hold(pmap, va, mpte);
+	return pmap_unwire_pte_hold(pmap, va, mpte, free);
 }
 
 void
@@ -1366,7 +1385,7 @@ pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags)
 {
 	vm_pindex_t ptepindex;
 	pd_entry_t *pd;
-	vm_page_t m;
+	vm_page_t m, free;
 
 	KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT ||
 	    (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK,
@@ -1390,8 +1409,10 @@ retry:
 		*pd = 0;
 		pd = 0;
 		pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
-		pmap_unuse_pt(pmap, va, *pmap_pdpe(pmap, va));
+		free = NULL;
+		pmap_unuse_pt(pmap, va, *pmap_pdpe(pmap, va), &free);
 		pmap_invalidate_all(kernel_pmap);
+		pmap_free_zero_pages(free);
 	}
 
 	/*
@@ -1609,7 +1630,7 @@ pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq)
 	pt_entry_t *pte, tpte;
 	pv_entry_t next_pv, pv;
 	vm_offset_t va;
-	vm_page_t m;
+	vm_page_t m, free;
 
 	TAILQ_FOREACH(m, &vpq->pl, pageq) {
 		if (m->hold_count || m->busy)
@@ -1635,12 +1656,14 @@ pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq)
 				    va, tpte));
 				vm_page_dirty(m);
 			}
+			free = NULL;
+			pmap_unuse_pt(pmap, va, ptepde, &free);
 			pmap_invalidate_page(pmap, va);
+			pmap_free_zero_pages(free);
 			TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 			if (TAILQ_EMPTY(&m->md.pv_list))
 				vm_page_flag_clear(m, PG_WRITEABLE);
 			m->md.pv_list_count--;
-			pmap_unuse_pt(pmap, va, ptepde);
 			free_pv_entry(pmap, pv);
 			if (pmap != locked_pmap)
 				PMAP_UNLOCK(pmap);
@@ -1833,7 +1856,8 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
  * pmap_remove_pte: do the things to unmap a page in a process
  */
 static int
-pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, pd_entry_t ptepde)
+pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, 
+    pd_entry_t ptepde, vm_page_t *free)
 {
 	pt_entry_t oldpte;
 	vm_page_t m;
@@ -1861,7 +1885,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, pd_entry_t ptepde)
 			vm_page_flag_set(m, PG_REFERENCED);
 		pmap_remove_entry(pmap, m, va);
 	}
-	return (pmap_unuse_pt(pmap, va, ptepde));
+	return (pmap_unuse_pt(pmap, va, ptepde, free));
 }
 
 /*
@@ -1871,6 +1895,7 @@ static void
 pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde)
 {
 	pt_entry_t *pte;
+	vm_page_t free = NULL;
 
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	if ((*pde & PG_V) == 0)
@@ -1878,8 +1903,9 @@ pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde)
 	pte = pmap_pde_to_pte(pde, va);
 	if ((*pte & PG_V) == 0)
 		return;
-	pmap_remove_pte(pmap, pte, va, *pde);
+	pmap_remove_pte(pmap, pte, va, *pde, &free);
 	pmap_invalidate_page(pmap, va);
+	pmap_free_zero_pages(free);
 }
 
 /*
@@ -1896,6 +1922,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 	pdp_entry_t *pdpe;
 	pd_entry_t ptpaddr, *pde;
 	pt_entry_t *pte;
+	vm_page_t free = NULL;
 	int anyvalid;
 
 	/*
@@ -1959,7 +1986,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 		if ((ptpaddr & PG_PS) != 0) {
 			*pde = 0;
 			pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE;
-			pmap_unuse_pt(pmap, sva, *pdpe);
+			pmap_unuse_pt(pmap, sva, *pdpe, &free);
 			anyvalid = 1;
 			continue;
 		}
@@ -1983,14 +2010,16 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 			 */
 			if ((*pte & PG_G) == 0)
 				anyvalid = 1;
-			if (pmap_remove_pte(pmap, pte, sva, ptpaddr))
+			if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free))
 				break;
 		}
 	}
 out:
-	vm_page_unlock_queues();
-	if (anyvalid)
+	if (anyvalid) {
 		pmap_invalidate_all(pmap);
+		pmap_free_zero_pages(free);
+	}
+	vm_page_unlock_queues();	
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2014,6 +2043,7 @@ pmap_remove_all(vm_page_t m)
 	pmap_t pmap;
 	pt_entry_t *pte, tpte;
 	pd_entry_t ptepde;
+	vm_page_t free;
 
 #if defined(PMAP_DIAGNOSTIC)
 	/*
@@ -2045,10 +2075,12 @@ pmap_remove_all(vm_page_t m)
 			    pv->pv_va, tpte));
 			vm_page_dirty(m);
 		}
+		free = NULL;
+		pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free);
 		pmap_invalidate_page(pmap, pv->pv_va);
+		pmap_free_zero_pages(free);
 		TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 		m->md.pv_list_count--;
-		pmap_unuse_pt(pmap, pv->pv_va, ptepde);
 		free_pv_entry(pmap, pv);
 		PMAP_UNLOCK(pmap);
 	}
@@ -2161,9 +2193,9 @@ retry:
 			}
 		}
 	}
-	vm_page_unlock_queues();
 	if (anychanged)
 		pmap_invalidate_all(pmap);
+	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2413,6 +2445,7 @@ static vm_page_t
 pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
     vm_prot_t prot, vm_page_t mpte)
 {
+	vm_page_t free;
 	pt_entry_t *pte;
 	vm_paddr_t pa;
 
@@ -2471,7 +2504,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	pte = vtopte(va);
 	if (*pte) {
 		if (mpte != NULL) {
-			pmap_unwire_pte_hold(pmap, va, mpte);
+			mpte->wire_count--;
 			mpte = NULL;
 		}
 		return (mpte);
@@ -2483,7 +2516,11 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 &&
 	    !pmap_try_insert_pv_entry(pmap, va, m)) {
 		if (mpte != NULL) {
-			pmap_unwire_pte_hold(pmap, va, mpte);
+			free = NULL;
+			if (pmap_unwire_pte_hold(pmap, va, mpte, &free)) {
+				pmap_invalidate_page(pmap, va);
+				pmap_free_zero_pages(free);
+			}
 			mpte = NULL;
 		}
 		return (mpte);
@@ -2657,6 +2694,7 @@ void
 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 	  vm_offset_t src_addr)
 {
+	vm_page_t   free;
 	vm_offset_t addr;
 	vm_offset_t end_addr = src_addr + len;
 	vm_offset_t va_next;
@@ -2716,7 +2754,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 				dst_pmap->pm_stats.resident_count +=
 				    NBPDR / PAGE_SIZE;
 			} else
-				pmap_unwire_pte_hold(dst_pmap, addr, dstmpde);
+				dstmpde->wire_count--;
 			continue;
 		}
 
@@ -2753,9 +2791,15 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 					*dst_pte = ptetemp & ~(PG_W | PG_M |
 					    PG_A);
 					dst_pmap->pm_stats.resident_count++;
-	 			} else
-					pmap_unwire_pte_hold(dst_pmap, addr,
-					    dstmpte);
+	 			} else {
+					free = NULL;
+					if (pmap_unwire_pte_hold(dst_pmap,
+					    addr, dstmpte, &free)) {
+					    	pmap_invalidate_page(dst_pmap,
+					 	    addr);
+				    	    	pmap_free_zero_pages(free);
+					}
+				}
 				if (dstmpte->wire_count >= srcmpte->wire_count)
 					break;
 			}
@@ -2866,7 +2910,7 @@ void
 pmap_remove_pages(pmap_t pmap)
 {
 	pt_entry_t *pte, tpte;
-	vm_page_t m;
+	vm_page_t m, free = NULL;
 	pv_entry_t pv;
 	struct pv_chunk *pc, *npc;
 	int field, idx;
@@ -2939,7 +2983,7 @@ pmap_remove_pages(pmap_t pmap)
 				if (TAILQ_EMPTY(&m->md.pv_list))
 					vm_page_flag_clear(m, PG_WRITEABLE);
 				pmap_unuse_pt(pmap, pv->pv_va,
-				    *vtopde(pv->pv_va));
+				    *vtopde(pv->pv_va), &free);
 			}
 		}
 		if (allfree) {
@@ -2952,8 +2996,9 @@ pmap_remove_pages(pmap_t pmap)
 			vm_page_free(m);
 		}
 	}
-	vm_page_unlock_queues();
 	pmap_invalidate_all(pmap);
+	pmap_free_zero_pages(free);
+	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index e3201af..01b2422 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -269,7 +269,8 @@ static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try);
 
 static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va,
     vm_page_t m, vm_prot_t prot, vm_page_t mpte);
-static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva);
+static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva,
+    vm_page_t *free);
 static void pmap_remove_page(struct pmap *pmap, vm_offset_t va);
 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m,
 					vm_offset_t va);
@@ -280,10 +281,10 @@ static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va,
 static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags);
 
 static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags);
-static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m);
+static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free);
 static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va);
 static void pmap_pte_release(pt_entry_t *pte);
-static int pmap_unuse_pt(pmap_t, vm_offset_t);
+static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t *);
 static vm_offset_t pmap_kmem_choose(vm_offset_t addr);
 #ifdef PAE
 static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait);
@@ -1129,24 +1130,35 @@ pmap_qremove(vm_offset_t sva, int count)
 /***************************************************
  * Page table page management routines.....
  ***************************************************/
+static PMAP_INLINE void
+pmap_free_zero_pages(vm_page_t free)
+{
+	vm_page_t m;
+
+	while (free != NULL) {
+		m = free;
+		free = m->right;
+		vm_page_free_zero(m);
+	}
+}
 
 /*
  * This routine unholds page table pages, and if the hold count
  * drops to zero, then it decrements the wire count.
  */
 static PMAP_INLINE int
-pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
+pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free)
 {
 
 	--m->wire_count;
 	if (m->wire_count == 0)
-		return _pmap_unwire_pte_hold(pmap, m);
+		return _pmap_unwire_pte_hold(pmap, m, free);
 	else
 		return 0;
 }
 
 static int 
-_pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
+_pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free)
 {
 	vm_offset_t pteva;
 
@@ -1156,6 +1168,8 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
 	pmap->pm_pdir[m->pindex] = 0;
 	--pmap->pm_stats.resident_count;
 
+	atomic_subtract_int(&cnt.v_wire_count, 1);
+
 	/*
 	 * Do an invltlb to make the invalidated mapping
 	 * take effect immediately.
@@ -1163,8 +1177,13 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
 	pteva = VM_MAXUSER_ADDRESS + i386_ptob(m->pindex);
 	pmap_invalidate_page(pmap, pteva);
 
-	vm_page_free_zero(m);
-	atomic_subtract_int(&cnt.v_wire_count, 1);
+	/* 
+	 * Put page on a list so that it is released after
+	 * *ALL* TLB shootdown is done
+	 */
+	m->right = *free;
+	*free = m;
+
 	return 1;
 }
 
@@ -1173,7 +1192,7 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m)
  * conditionally free the page, and manage the hold/wire counts.
  */
 static int
-pmap_unuse_pt(pmap_t pmap, vm_offset_t va)
+pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t *free)
 {
 	pd_entry_t ptepde;
 	vm_page_t mpte;
@@ -1182,7 +1201,7 @@ pmap_unuse_pt(pmap_t pmap, vm_offset_t va)
 		return 0;
 	ptepde = *pmap_pde(pmap, va);
 	mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME);
-	return pmap_unwire_pte_hold(pmap, mpte);
+	return pmap_unwire_pte_hold(pmap, mpte, free);
 }
 
 void
@@ -1683,7 +1702,7 @@ pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq)
 	pt_entry_t *pte, tpte;
 	pv_entry_t next_pv, pv;
 	vm_offset_t va;
-	vm_page_t m;
+	vm_page_t m, free;
 
 	sched_pin();
 	TAILQ_FOREACH(m, &vpq->pl, pageq) {
@@ -1710,12 +1729,14 @@ pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq)
 				    va, (uintmax_t)tpte));
 				vm_page_dirty(m);
 			}
+			free = NULL;
+			pmap_unuse_pt(pmap, va, &free);
 			pmap_invalidate_page(pmap, va);
+			pmap_free_zero_pages(free);
 			TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 			if (TAILQ_EMPTY(&m->md.pv_list))
 				vm_page_flag_clear(m, PG_WRITEABLE);
 			m->md.pv_list_count--;
-			pmap_unuse_pt(pmap, va);
 			free_pv_entry(pmap, pv);
 			if (pmap != locked_pmap)
 				PMAP_UNLOCK(pmap);
@@ -1930,7 +1951,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m)
  * pmap_remove_pte: do the things to unmap a page in a process
  */
 static int
-pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va)
+pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, vm_page_t *free)
 {
 	pt_entry_t oldpte;
 	vm_page_t m;
@@ -1959,7 +1980,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va)
 			vm_page_flag_set(m, PG_REFERENCED);
 		pmap_remove_entry(pmap, m, va);
 	}
-	return (pmap_unuse_pt(pmap, va));
+	return (pmap_unuse_pt(pmap, va, free));
 }
 
 /*
@@ -1969,14 +1990,16 @@ static void
 pmap_remove_page(pmap_t pmap, vm_offset_t va)
 {
 	pt_entry_t *pte;
+	vm_page_t free = NULL;
 
 	mtx_assert(&vm_page_queue_mtx, MA_OWNED);
 	KASSERT(curthread->td_pinned > 0, ("curthread not pinned"));
 	PMAP_LOCK_ASSERT(pmap, MA_OWNED);
 	if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0)
 		return;
-	pmap_remove_pte(pmap, pte, va);
+	pmap_remove_pte(pmap, pte, va, &free);
 	pmap_invalidate_page(pmap, va);
+	pmap_free_zero_pages(free);
 }
 
 /*
@@ -1991,6 +2014,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 	vm_offset_t pdnxt;
 	pd_entry_t ptpaddr;
 	pt_entry_t *pte;
+	vm_page_t free = NULL;
 	int anyvalid;
 
 	/*
@@ -2065,15 +2089,17 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 			 */
 			if ((*pte & PG_G) == 0)
 				anyvalid = 1;
-			if (pmap_remove_pte(pmap, pte, sva))
+			if (pmap_remove_pte(pmap, pte, sva, &free))
 				break;
 		}
 	}
 out:
 	sched_unpin();
-	vm_page_unlock_queues();
-	if (anyvalid)
+	if (anyvalid) {
 		pmap_invalidate_all(pmap);
+		pmap_free_zero_pages(free);
+	}
+	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2096,6 +2122,7 @@ pmap_remove_all(vm_page_t m)
 	pv_entry_t pv;
 	pmap_t pmap;
 	pt_entry_t *pte, tpte;
+	vm_page_t free;
 
 #if defined(PMAP_DIAGNOSTIC)
 	/*
@@ -2128,10 +2155,12 @@ pmap_remove_all(vm_page_t m)
 			    pv->pv_va, (uintmax_t)tpte));
 			vm_page_dirty(m);
 		}
+		free = NULL;
+		pmap_unuse_pt(pmap, pv->pv_va, &free);
 		pmap_invalidate_page(pmap, pv->pv_va);
+		pmap_free_zero_pages(free);
 		TAILQ_REMOVE(&m->md.pv_list, pv, pv_list);
 		m->md.pv_list_count--;
-		pmap_unuse_pt(pmap, pv->pv_va);
 		free_pv_entry(pmap, pv);
 		PMAP_UNLOCK(pmap);
 	}
@@ -2254,9 +2283,9 @@ retry:
 		}
 	}
 	sched_unpin();
-	vm_page_unlock_queues();
 	if (anychanged)
 		pmap_invalidate_all(pmap);
+	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }
 
@@ -2515,6 +2544,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 {
 	pt_entry_t *pte;
 	vm_paddr_t pa;
+	vm_page_t free;
 
 	KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva ||
 	    (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0,
@@ -2571,7 +2601,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	pte = vtopte(va);
 	if (*pte) {
 		if (mpte != NULL) {
-			pmap_unwire_pte_hold(pmap, mpte);
+			mpte->wire_count--;
 			mpte = NULL;
 		}
 		return (mpte);
@@ -2583,7 +2613,12 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m,
 	if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 &&
 	    !pmap_try_insert_pv_entry(pmap, va, m)) {
 		if (mpte != NULL) {
-			pmap_unwire_pte_hold(pmap, mpte);
+			free = NULL;
+			if (pmap_unwire_pte_hold(pmap, mpte, &free)) {
+				pmap_invalidate_page(pmap, va);
+				pmap_free_zero_pages(free);
+			}
+			
 			mpte = NULL;
 		}
 		return (mpte);
@@ -2740,6 +2775,7 @@ void
 pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 	  vm_offset_t src_addr)
 {
+	vm_page_t   free;
 	vm_offset_t addr;
 	vm_offset_t end_addr = src_addr + len;
 	vm_offset_t pdnxt;
@@ -2816,8 +2852,15 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len,
 					*dst_pte = ptetemp & ~(PG_W | PG_M |
 					    PG_A);
 					dst_pmap->pm_stats.resident_count++;
-	 			} else
-					pmap_unwire_pte_hold(dst_pmap, dstmpte);
+	 			} else {
+					free = NULL;
+					if (pmap_unwire_pte_hold( dst_pmap,
+					    dstmpte, &free)) {
+						pmap_invalidate_page(dst_pmap,
+						    addr);
+						pmap_free_zero_pages(free);
+					}
+				}
 				if (dstmpte->wire_count >= srcmpte->wire_count)
 					break;
 			}
@@ -2985,7 +3028,7 @@ void
 pmap_remove_pages(pmap_t pmap)
 {
 	pt_entry_t *pte, tpte;
-	vm_page_t m;
+	vm_page_t m, free = NULL;
 	pv_entry_t pv;
 	struct pv_chunk *pc, *npc;
 	int field, idx;
@@ -3059,7 +3102,7 @@ pmap_remove_pages(pmap_t pmap)
 				if (TAILQ_EMPTY(&m->md.pv_list))
 					vm_page_flag_clear(m, PG_WRITEABLE);
 
-				pmap_unuse_pt(pmap, pv->pv_va);
+				pmap_unuse_pt(pmap, pv->pv_va, &free);
 			}
 		}
 		if (allfree) {
@@ -3075,8 +3118,9 @@ pmap_remove_pages(pmap_t pmap)
 		}
 	}
 	sched_unpin();
-	vm_page_unlock_queues();
 	pmap_invalidate_all(pmap);
+	pmap_free_zero_pages(free);
+	vm_page_unlock_queues();
 	PMAP_UNLOCK(pmap);
 }