diff options
author | ups <ups@FreeBSD.org> | 2007-04-21 14:17:30 +0000 |
---|---|---|
committer | ups <ups@FreeBSD.org> | 2007-04-21 14:17:30 +0000 |
commit | cca3de2c55be08432928aafb01996eb30f5ebcf0 (patch) | |
tree | de644e12ea0f2b54d7f7d8ffc66aa4f7bae06286 | |
parent | 2cd524ebede2b227bc0180dda7bd0d3ec7b21972 (diff) | |
download | FreeBSD-src-cca3de2c55be08432928aafb01996eb30f5ebcf0.zip FreeBSD-src-cca3de2c55be08432928aafb01996eb30f5ebcf0.tar.gz |
Modify TLB invalidation handling.
Reviewed by: alc@, peter@
MFC after: 1 week
-rw-r--r-- | sys/amd64/amd64/pmap.c | 111 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 98 |
2 files changed, 149 insertions, 60 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 3cbf418..8fb06e1 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -209,7 +209,7 @@ static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try); static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte); static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, - vm_offset_t sva, pd_entry_t ptepde); + vm_offset_t sva, pd_entry_t ptepde, vm_page_t *free); static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde); static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va); @@ -221,8 +221,9 @@ static vm_page_t pmap_allocpde(pmap_t pmap, vm_offset_t va, int flags); static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); static vm_page_t _pmap_allocpte(pmap_t pmap, vm_pindex_t ptepindex, int flags); -static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m); -static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t); +static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, + vm_page_t* free); +static int pmap_unuse_pt(pmap_t, vm_offset_t, pd_entry_t, vm_page_t *); static vm_offset_t pmap_kmem_choose(vm_offset_t addr); CTASSERT(1 << PDESHIFT == sizeof(pd_entry_t)); @@ -1065,24 +1066,36 @@ pmap_qremove(vm_offset_t sva, int count) /*************************************************** * Page table page management routines..... ***************************************************/ +static PMAP_INLINE void +pmap_free_zero_pages(vm_page_t free) +{ + vm_page_t m; + + while (free != NULL) { + m = free; + free = m->right; + vm_page_free_zero(m); + } +} /* * This routine unholds page table pages, and if the hold count * drops to zero, then it decrements the wire count. */ static PMAP_INLINE int -pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m) +pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t *free) { --m->wire_count; if (m->wire_count == 0) - return _pmap_unwire_pte_hold(pmap, va, m); + return _pmap_unwire_pte_hold(pmap, va, m, free); else return 0; } static int -_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m) +_pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m, + vm_page_t *free) { vm_offset_t pteva; @@ -1114,14 +1127,14 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m) vm_page_t pdpg; pdpg = PHYS_TO_VM_PAGE(*pmap_pdpe(pmap, va) & PG_FRAME); - pmap_unwire_pte_hold(pmap, va, pdpg); + pmap_unwire_pte_hold(pmap, va, pdpg, free); } if (m->pindex >= NUPDE && m->pindex < (NUPDE + NUPDPE)) { /* We just released a PD, unhold the matching PDP */ vm_page_t pdppg; pdppg = PHYS_TO_VM_PAGE(*pmap_pml4e(pmap, va) & PG_FRAME); - pmap_unwire_pte_hold(pmap, va, pdppg); + pmap_unwire_pte_hold(pmap, va, pdppg, free); } /* @@ -1130,7 +1143,13 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m) */ pmap_invalidate_page(pmap, pteva); - vm_page_free_zero(m); + /* + * Put page on a list so that it is released after + * *ALL* TLB shootdown is done + */ + m->right = *free; + *free = m; + atomic_subtract_int(&cnt.v_wire_count, 1); return 1; } @@ -1140,7 +1159,7 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m) * conditionally free the page, and manage the hold/wire counts. */ static int -pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde) +pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde, vm_page_t *free) { vm_page_t mpte; @@ -1148,7 +1167,7 @@ pmap_unuse_pt(pmap_t pmap, vm_offset_t va, pd_entry_t ptepde) return 0; KASSERT(ptepde != 0, ("pmap_unuse_pt: ptepde != 0")); mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); - return pmap_unwire_pte_hold(pmap, va, mpte); + return pmap_unwire_pte_hold(pmap, va, mpte, free); } void @@ -1366,7 +1385,7 @@ pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags) { vm_pindex_t ptepindex; pd_entry_t *pd; - vm_page_t m; + vm_page_t m, free; KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, @@ -1390,8 +1409,10 @@ retry: *pd = 0; pd = 0; pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; - pmap_unuse_pt(pmap, va, *pmap_pdpe(pmap, va)); + free = NULL; + pmap_unuse_pt(pmap, va, *pmap_pdpe(pmap, va), &free); pmap_invalidate_all(kernel_pmap); + pmap_free_zero_pages(free); } /* @@ -1609,7 +1630,7 @@ pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq) pt_entry_t *pte, tpte; pv_entry_t next_pv, pv; vm_offset_t va; - vm_page_t m; + vm_page_t m, free; TAILQ_FOREACH(m, &vpq->pl, pageq) { if (m->hold_count || m->busy) @@ -1635,12 +1656,14 @@ pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq) va, tpte)); vm_page_dirty(m); } + free = NULL; + pmap_unuse_pt(pmap, va, ptepde, &free); pmap_invalidate_page(pmap, va); + pmap_free_zero_pages(free); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_flag_clear(m, PG_WRITEABLE); m->md.pv_list_count--; - pmap_unuse_pt(pmap, va, ptepde); free_pv_entry(pmap, pv); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); @@ -1833,7 +1856,8 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) * pmap_remove_pte: do the things to unmap a page in a process */ static int -pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, pd_entry_t ptepde) +pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, + pd_entry_t ptepde, vm_page_t *free) { pt_entry_t oldpte; vm_page_t m; @@ -1861,7 +1885,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, pd_entry_t ptepde) vm_page_flag_set(m, PG_REFERENCED); pmap_remove_entry(pmap, m, va); } - return (pmap_unuse_pt(pmap, va, ptepde)); + return (pmap_unuse_pt(pmap, va, ptepde, free)); } /* @@ -1871,6 +1895,7 @@ static void pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde) { pt_entry_t *pte; + vm_page_t free = NULL; PMAP_LOCK_ASSERT(pmap, MA_OWNED); if ((*pde & PG_V) == 0) @@ -1878,8 +1903,9 @@ pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde) pte = pmap_pde_to_pte(pde, va); if ((*pte & PG_V) == 0) return; - pmap_remove_pte(pmap, pte, va, *pde); + pmap_remove_pte(pmap, pte, va, *pde, &free); pmap_invalidate_page(pmap, va); + pmap_free_zero_pages(free); } /* @@ -1896,6 +1922,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) pdp_entry_t *pdpe; pd_entry_t ptpaddr, *pde; pt_entry_t *pte; + vm_page_t free = NULL; int anyvalid; /* @@ -1959,7 +1986,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) if ((ptpaddr & PG_PS) != 0) { *pde = 0; pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; - pmap_unuse_pt(pmap, sva, *pdpe); + pmap_unuse_pt(pmap, sva, *pdpe, &free); anyvalid = 1; continue; } @@ -1983,14 +2010,16 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) */ if ((*pte & PG_G) == 0) anyvalid = 1; - if (pmap_remove_pte(pmap, pte, sva, ptpaddr)) + if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free)) break; } } out: - vm_page_unlock_queues(); - if (anyvalid) + if (anyvalid) { pmap_invalidate_all(pmap); + pmap_free_zero_pages(free); + } + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } @@ -2014,6 +2043,7 @@ pmap_remove_all(vm_page_t m) pmap_t pmap; pt_entry_t *pte, tpte; pd_entry_t ptepde; + vm_page_t free; #if defined(PMAP_DIAGNOSTIC) /* @@ -2045,10 +2075,12 @@ pmap_remove_all(vm_page_t m) pv->pv_va, tpte)); vm_page_dirty(m); } + free = NULL; + pmap_unuse_pt(pmap, pv->pv_va, ptepde, &free); pmap_invalidate_page(pmap, pv->pv_va); + pmap_free_zero_pages(free); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); m->md.pv_list_count--; - pmap_unuse_pt(pmap, pv->pv_va, ptepde); free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } @@ -2161,9 +2193,9 @@ retry: } } } - vm_page_unlock_queues(); if (anychanged) pmap_invalidate_all(pmap); + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } @@ -2413,6 +2445,7 @@ static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte) { + vm_page_t free; pt_entry_t *pte; vm_paddr_t pa; @@ -2471,7 +2504,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, pte = vtopte(va); if (*pte) { if (mpte != NULL) { - pmap_unwire_pte_hold(pmap, va, mpte); + mpte->wire_count--; mpte = NULL; } return (mpte); @@ -2483,7 +2516,11 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 && !pmap_try_insert_pv_entry(pmap, va, m)) { if (mpte != NULL) { - pmap_unwire_pte_hold(pmap, va, mpte); + free = NULL; + if (pmap_unwire_pte_hold(pmap, va, mpte, &free)) { + pmap_invalidate_page(pmap, va); + pmap_free_zero_pages(free); + } mpte = NULL; } return (mpte); @@ -2657,6 +2694,7 @@ void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { + vm_page_t free; vm_offset_t addr; vm_offset_t end_addr = src_addr + len; vm_offset_t va_next; @@ -2716,7 +2754,7 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, dst_pmap->pm_stats.resident_count += NBPDR / PAGE_SIZE; } else - pmap_unwire_pte_hold(dst_pmap, addr, dstmpde); + dstmpde->wire_count--; continue; } @@ -2753,9 +2791,15 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, *dst_pte = ptetemp & ~(PG_W | PG_M | PG_A); dst_pmap->pm_stats.resident_count++; - } else - pmap_unwire_pte_hold(dst_pmap, addr, - dstmpte); + } else { + free = NULL; + if (pmap_unwire_pte_hold(dst_pmap, + addr, dstmpte, &free)) { + pmap_invalidate_page(dst_pmap, + addr); + pmap_free_zero_pages(free); + } + } if (dstmpte->wire_count >= srcmpte->wire_count) break; } @@ -2866,7 +2910,7 @@ void pmap_remove_pages(pmap_t pmap) { pt_entry_t *pte, tpte; - vm_page_t m; + vm_page_t m, free = NULL; pv_entry_t pv; struct pv_chunk *pc, *npc; int field, idx; @@ -2939,7 +2983,7 @@ pmap_remove_pages(pmap_t pmap) if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_flag_clear(m, PG_WRITEABLE); pmap_unuse_pt(pmap, pv->pv_va, - *vtopde(pv->pv_va)); + *vtopde(pv->pv_va), &free); } } if (allfree) { @@ -2952,8 +2996,9 @@ pmap_remove_pages(pmap_t pmap) vm_page_free(m); } } - vm_page_unlock_queues(); pmap_invalidate_all(pmap); + pmap_free_zero_pages(free); + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index e3201af..01b2422 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -269,7 +269,8 @@ static pv_entry_t get_pv_entry(pmap_t locked_pmap, int try); static vm_page_t pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t mpte); -static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva); +static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t sva, + vm_page_t *free); static void pmap_remove_page(struct pmap *pmap, vm_offset_t va); static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va); @@ -280,10 +281,10 @@ static boolean_t pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va, int flags); static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags); -static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m); +static int _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free); static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va); static void pmap_pte_release(pt_entry_t *pte); -static int pmap_unuse_pt(pmap_t, vm_offset_t); +static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t *); static vm_offset_t pmap_kmem_choose(vm_offset_t addr); #ifdef PAE static void *pmap_pdpt_allocf(uma_zone_t zone, int bytes, u_int8_t *flags, int wait); @@ -1129,24 +1130,35 @@ pmap_qremove(vm_offset_t sva, int count) /*************************************************** * Page table page management routines..... ***************************************************/ +static PMAP_INLINE void +pmap_free_zero_pages(vm_page_t free) +{ + vm_page_t m; + + while (free != NULL) { + m = free; + free = m->right; + vm_page_free_zero(m); + } +} /* * This routine unholds page table pages, and if the hold count * drops to zero, then it decrements the wire count. */ static PMAP_INLINE int -pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) +pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free) { --m->wire_count; if (m->wire_count == 0) - return _pmap_unwire_pte_hold(pmap, m); + return _pmap_unwire_pte_hold(pmap, m, free); else return 0; } static int -_pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) +_pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m, vm_page_t *free) { vm_offset_t pteva; @@ -1156,6 +1168,8 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) pmap->pm_pdir[m->pindex] = 0; --pmap->pm_stats.resident_count; + atomic_subtract_int(&cnt.v_wire_count, 1); + /* * Do an invltlb to make the invalidated mapping * take effect immediately. @@ -1163,8 +1177,13 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) pteva = VM_MAXUSER_ADDRESS + i386_ptob(m->pindex); pmap_invalidate_page(pmap, pteva); - vm_page_free_zero(m); - atomic_subtract_int(&cnt.v_wire_count, 1); + /* + * Put page on a list so that it is released after + * *ALL* TLB shootdown is done + */ + m->right = *free; + *free = m; + return 1; } @@ -1173,7 +1192,7 @@ _pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) * conditionally free the page, and manage the hold/wire counts. */ static int -pmap_unuse_pt(pmap_t pmap, vm_offset_t va) +pmap_unuse_pt(pmap_t pmap, vm_offset_t va, vm_page_t *free) { pd_entry_t ptepde; vm_page_t mpte; @@ -1182,7 +1201,7 @@ pmap_unuse_pt(pmap_t pmap, vm_offset_t va) return 0; ptepde = *pmap_pde(pmap, va); mpte = PHYS_TO_VM_PAGE(ptepde & PG_FRAME); - return pmap_unwire_pte_hold(pmap, mpte); + return pmap_unwire_pte_hold(pmap, mpte, free); } void @@ -1683,7 +1702,7 @@ pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq) pt_entry_t *pte, tpte; pv_entry_t next_pv, pv; vm_offset_t va; - vm_page_t m; + vm_page_t m, free; sched_pin(); TAILQ_FOREACH(m, &vpq->pl, pageq) { @@ -1710,12 +1729,14 @@ pmap_collect(pmap_t locked_pmap, struct vpgqueues *vpq) va, (uintmax_t)tpte)); vm_page_dirty(m); } + free = NULL; + pmap_unuse_pt(pmap, va, &free); pmap_invalidate_page(pmap, va); + pmap_free_zero_pages(free); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_flag_clear(m, PG_WRITEABLE); m->md.pv_list_count--; - pmap_unuse_pt(pmap, va); free_pv_entry(pmap, pv); if (pmap != locked_pmap) PMAP_UNLOCK(pmap); @@ -1930,7 +1951,7 @@ pmap_try_insert_pv_entry(pmap_t pmap, vm_offset_t va, vm_page_t m) * pmap_remove_pte: do the things to unmap a page in a process */ static int -pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va) +pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, vm_page_t *free) { pt_entry_t oldpte; vm_page_t m; @@ -1959,7 +1980,7 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va) vm_page_flag_set(m, PG_REFERENCED); pmap_remove_entry(pmap, m, va); } - return (pmap_unuse_pt(pmap, va)); + return (pmap_unuse_pt(pmap, va, free)); } /* @@ -1969,14 +1990,16 @@ static void pmap_remove_page(pmap_t pmap, vm_offset_t va) { pt_entry_t *pte; + vm_page_t free = NULL; mtx_assert(&vm_page_queue_mtx, MA_OWNED); KASSERT(curthread->td_pinned > 0, ("curthread not pinned")); PMAP_LOCK_ASSERT(pmap, MA_OWNED); if ((pte = pmap_pte_quick(pmap, va)) == NULL || *pte == 0) return; - pmap_remove_pte(pmap, pte, va); + pmap_remove_pte(pmap, pte, va, &free); pmap_invalidate_page(pmap, va); + pmap_free_zero_pages(free); } /* @@ -1991,6 +2014,7 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) vm_offset_t pdnxt; pd_entry_t ptpaddr; pt_entry_t *pte; + vm_page_t free = NULL; int anyvalid; /* @@ -2065,15 +2089,17 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) */ if ((*pte & PG_G) == 0) anyvalid = 1; - if (pmap_remove_pte(pmap, pte, sva)) + if (pmap_remove_pte(pmap, pte, sva, &free)) break; } } out: sched_unpin(); - vm_page_unlock_queues(); - if (anyvalid) + if (anyvalid) { pmap_invalidate_all(pmap); + pmap_free_zero_pages(free); + } + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } @@ -2096,6 +2122,7 @@ pmap_remove_all(vm_page_t m) pv_entry_t pv; pmap_t pmap; pt_entry_t *pte, tpte; + vm_page_t free; #if defined(PMAP_DIAGNOSTIC) /* @@ -2128,10 +2155,12 @@ pmap_remove_all(vm_page_t m) pv->pv_va, (uintmax_t)tpte)); vm_page_dirty(m); } + free = NULL; + pmap_unuse_pt(pmap, pv->pv_va, &free); pmap_invalidate_page(pmap, pv->pv_va); + pmap_free_zero_pages(free); TAILQ_REMOVE(&m->md.pv_list, pv, pv_list); m->md.pv_list_count--; - pmap_unuse_pt(pmap, pv->pv_va); free_pv_entry(pmap, pv); PMAP_UNLOCK(pmap); } @@ -2254,9 +2283,9 @@ retry: } } sched_unpin(); - vm_page_unlock_queues(); if (anychanged) pmap_invalidate_all(pmap); + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } @@ -2515,6 +2544,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, { pt_entry_t *pte; vm_paddr_t pa; + vm_page_t free; KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva || (m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0, @@ -2571,7 +2601,7 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, pte = vtopte(va); if (*pte) { if (mpte != NULL) { - pmap_unwire_pte_hold(pmap, mpte); + mpte->wire_count--; mpte = NULL; } return (mpte); @@ -2583,7 +2613,12 @@ pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0 && !pmap_try_insert_pv_entry(pmap, va, m)) { if (mpte != NULL) { - pmap_unwire_pte_hold(pmap, mpte); + free = NULL; + if (pmap_unwire_pte_hold(pmap, mpte, &free)) { + pmap_invalidate_page(pmap, va); + pmap_free_zero_pages(free); + } + mpte = NULL; } return (mpte); @@ -2740,6 +2775,7 @@ void pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, vm_offset_t src_addr) { + vm_page_t free; vm_offset_t addr; vm_offset_t end_addr = src_addr + len; vm_offset_t pdnxt; @@ -2816,8 +2852,15 @@ pmap_copy(pmap_t dst_pmap, pmap_t src_pmap, vm_offset_t dst_addr, vm_size_t len, *dst_pte = ptetemp & ~(PG_W | PG_M | PG_A); dst_pmap->pm_stats.resident_count++; - } else - pmap_unwire_pte_hold(dst_pmap, dstmpte); + } else { + free = NULL; + if (pmap_unwire_pte_hold( dst_pmap, + dstmpte, &free)) { + pmap_invalidate_page(dst_pmap, + addr); + pmap_free_zero_pages(free); + } + } if (dstmpte->wire_count >= srcmpte->wire_count) break; } @@ -2985,7 +3028,7 @@ void pmap_remove_pages(pmap_t pmap) { pt_entry_t *pte, tpte; - vm_page_t m; + vm_page_t m, free = NULL; pv_entry_t pv; struct pv_chunk *pc, *npc; int field, idx; @@ -3059,7 +3102,7 @@ pmap_remove_pages(pmap_t pmap) if (TAILQ_EMPTY(&m->md.pv_list)) vm_page_flag_clear(m, PG_WRITEABLE); - pmap_unuse_pt(pmap, pv->pv_va); + pmap_unuse_pt(pmap, pv->pv_va, &free); } } if (allfree) { @@ -3075,8 +3118,9 @@ pmap_remove_pages(pmap_t pmap) } } sched_unpin(); - vm_page_unlock_queues(); pmap_invalidate_all(pmap); + pmap_free_zero_pages(free); + vm_page_unlock_queues(); PMAP_UNLOCK(pmap); } |