diff options
author | alc <alc@FreeBSD.org> | 2010-06-10 16:56:35 +0000 |
---|---|---|
committer | alc <alc@FreeBSD.org> | 2010-06-10 16:56:35 +0000 |
commit | 7c212e010d5269026628a1e2e686c89679c23af8 (patch) | |
tree | 2e32fd87e6b7bde5898f2e0cb503e19a50ea5f7b /sys | |
parent | c2cccc78fe9cc4c3e971806f8c5ce77d16a4bd39 (diff) | |
download | FreeBSD-src-7c212e010d5269026628a1e2e686c89679c23af8.zip FreeBSD-src-7c212e010d5269026628a1e2e686c89679c23af8.tar.gz |
Reduce the scope of the page queues lock and the number of
PG_REFERENCED changes in vm_pageout_object_deactivate_pages().
Simplify this function's inner loop using TAILQ_FOREACH(), and shorten
some of its overly long lines. Update a stale comment.
Assert that PG_REFERENCED may be cleared only if the object containing
the page is locked. Add a comment documenting this.
Assert that a caller to vm_page_requeue() holds the page queues lock,
and assert that the page is on a page queue.
Push down the page queues lock into pmap_ts_referenced() and
pmap_page_exists_quick(). (As of now, there are no longer any pmap
functions that expect to be called with the page queues lock held.)
Neither pmap_ts_referenced() nor pmap_page_exists_quick() should ever
be passed an unmanaged page. Assert this rather than returning "0"
and "FALSE" respectively.
ARM:
Simplify pmap_page_exists_quick() by switching to TAILQ_FOREACH().
Push down the page queues lock inside of pmap_clearbit(), simplifying
pmap_clear_modify(), pmap_clear_reference(), and pmap_remove_write().
Additionally, this allows for avoiding the acquisition of the page
queues lock in some cases.
PowerPC/AIM:
moea*_page_exits_quick() and moea*_page_wired_mappings() will never be
called before pmap initialization is complete. Therefore, the check
for moea_initialized can be eliminated.
Push down the page queues lock inside of moea*_clear_bit(),
simplifying moea*_clear_modify() and moea*_clear_reference().
The last parameter to moea*_clear_bit() is never used. Eliminate it.
PowerPC/BookE:
Simplify mmu_booke_page_exists_quick()'s control flow.
Reviewed by: kib@
Diffstat (limited to 'sys')
-rw-r--r-- | sys/amd64/amd64/pmap.c | 33 | ||||
-rw-r--r-- | sys/arm/arm/pmap.c | 38 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 32 | ||||
-rw-r--r-- | sys/i386/xen/pmap.c | 22 | ||||
-rw-r--r-- | sys/ia64/ia64/pmap.c | 28 | ||||
-rw-r--r-- | sys/mips/mips/pmap.c | 25 | ||||
-rw-r--r-- | sys/powerpc/aim/mmu_oea.c | 52 | ||||
-rw-r--r-- | sys/powerpc/aim/mmu_oea64.c | 54 | ||||
-rw-r--r-- | sys/powerpc/booke/pmap.c | 34 | ||||
-rw-r--r-- | sys/sparc64/sparc64/pmap.c | 24 | ||||
-rw-r--r-- | sys/sun4v/sun4v/pmap.c | 24 | ||||
-rw-r--r-- | sys/vm/vm_page.c | 16 | ||||
-rw-r--r-- | sys/vm/vm_page.h | 3 | ||||
-rw-r--r-- | sys/vm/vm_pageout.c | 68 |
14 files changed, 227 insertions, 226 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index e138cbc..d85c3fa 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -3899,30 +3899,35 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) struct md_page *pvh; pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return (FALSE); - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { - return (TRUE); + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - if (loops < 16) { + if (!rv && loops < 16) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { - if (PV_PMAP(pv) == pmap) - return (TRUE); + if (PV_PMAP(pv) == pmap) { + rv = TRUE; + break; + } loops++; if (loops >= 16) break; } } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -4335,10 +4340,10 @@ pmap_ts_referenced(vm_page_t m) vm_offset_t va; int rtval = 0; - if (m->flags & PG_FICTITIOUS) - return (rtval); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + vm_page_lock_queues(); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) { pmap = PV_PMAP(pv); PMAP_LOCK(pmap); @@ -4362,7 +4367,7 @@ pmap_ts_referenced(vm_page_t m) rtval++; if (rtval > 4) { PMAP_UNLOCK(pmap); - return (rtval); + goto out; } } } @@ -4391,6 +4396,8 @@ pmap_ts_referenced(vm_page_t m) PMAP_UNLOCK(pmap); } while ((pv = pvn) != NULL && pv != pvf); } +out: + vm_page_unlock_queues(); return (rtval); } diff --git a/sys/arm/arm/pmap.c b/sys/arm/arm/pmap.c index a375987..06a4abe 100644 --- a/sys/arm/arm/pmap.c +++ b/sys/arm/arm/pmap.c @@ -1423,7 +1423,7 @@ pmap_clearbit(struct vm_page *pg, u_int maskbits) u_int oflags; int count = 0; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_queues(); if (maskbits & PVF_WRITE) maskbits |= PVF_MOD; @@ -1433,6 +1433,7 @@ pmap_clearbit(struct vm_page *pg, u_int maskbits) pg->md.pvh_attrs &= ~(maskbits & (PVF_MOD | PVF_REF)); if (TAILQ_EMPTY(&pg->md.pv_list)) { + vm_page_unlock_queues(); return (0); } @@ -1568,6 +1569,7 @@ pmap_clearbit(struct vm_page *pg, u_int maskbits) if (maskbits & PVF_WRITE) vm_page_flag_clear(pg, PG_WRITEABLE); + vm_page_unlock_queues(); return (count); } @@ -4417,24 +4419,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return (FALSE); - - /* - * Not found, check current mappings returning immediately - */ - for (pv = TAILQ_FIRST(&m->md.pv_list); - pv; - pv = TAILQ_NEXT(pv, pv_list)) { + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); + TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { - return (TRUE); + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -4469,8 +4470,8 @@ int pmap_ts_referenced(vm_page_t m) { - if (m->flags & PG_FICTITIOUS) - return (0); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); return (pmap_clearbit(m, PVF_REF)); } @@ -4508,10 +4509,8 @@ pmap_clear_modify(vm_page_t m) */ if ((m->flags & PG_WRITEABLE) == 0) return; - vm_page_lock_queues(); if (m->md.pvh_attrs & PVF_MOD) pmap_clearbit(m, PVF_MOD); - vm_page_unlock_queues(); } @@ -4541,10 +4540,8 @@ pmap_clear_reference(vm_page_t m) KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("pmap_clear_reference: page %p is not managed", m)); - vm_page_lock_queues(); if (m->md.pvh_attrs & PVF_REF) pmap_clearbit(m, PVF_REF); - vm_page_unlock_queues(); } @@ -4565,11 +4562,8 @@ pmap_remove_write(vm_page_t m) */ VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); if ((m->oflags & VPO_BUSY) != 0 || - (m->flags & PG_WRITEABLE) != 0) { - vm_page_lock_queues(); + (m->flags & PG_WRITEABLE) != 0) pmap_clearbit(m, PVF_WRITE); - vm_page_unlock_queues(); - } } diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index 68c5edc..b740834 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -4061,30 +4061,35 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) struct md_page *pvh; pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return (FALSE); - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { - return (TRUE); + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - if (loops < 16) { + if (!rv && loops < 16) { pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); TAILQ_FOREACH(pv, &pvh->pv_list, pv_list) { - if (PV_PMAP(pv) == pmap) - return (TRUE); + if (PV_PMAP(pv) == pmap) { + rv = TRUE; + break; + } loops++; if (loops >= 16) break; } } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -4512,11 +4517,11 @@ pmap_ts_referenced(vm_page_t m) vm_offset_t va; int rtval = 0; - if (m->flags & PG_FICTITIOUS) - return (rtval); - sched_pin(); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); pvh = pa_to_pvh(VM_PAGE_TO_PHYS(m)); + vm_page_lock_queues(); + sched_pin(); TAILQ_FOREACH_SAFE(pv, &pvh->pv_list, pv_list, pvn) { va = pv->pv_va; pmap = PV_PMAP(pv); @@ -4571,6 +4576,7 @@ pmap_ts_referenced(vm_page_t m) } out: sched_unpin(); + vm_page_unlock_queues(); return (rtval); } diff --git a/sys/i386/xen/pmap.c b/sys/i386/xen/pmap.c index 1a3a9d5..d4e02d9 100644 --- a/sys/i386/xen/pmap.c +++ b/sys/i386/xen/pmap.c @@ -3449,20 +3449,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return (FALSE); - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (PV_PMAP(pv) == pmap) { - return TRUE; + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -3839,10 +3842,10 @@ pmap_ts_referenced(vm_page_t m) pt_entry_t *pte; int rtval = 0; - if (m->flags & PG_FICTITIOUS) - return (rtval); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); + vm_page_lock_queues(); sched_pin(); - mtx_assert(&vm_page_queue_mtx, MA_OWNED); if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pvf = pv; do { @@ -3867,6 +3870,7 @@ pmap_ts_referenced(vm_page_t m) PT_SET_MA(PADDR1, 0); sched_unpin(); + vm_page_unlock_queues(); return (rtval); } diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 373d019..56e8a42 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -1837,23 +1837,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return FALSE; - - /* - * Not found, check current mappings returning immediately if found. - */ - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { - return TRUE; + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -1949,9 +1949,9 @@ pmap_ts_referenced(vm_page_t m) pv_entry_t pv; int count = 0; - if (m->flags & PG_FICTITIOUS) - return 0; - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { PMAP_LOCK(pv->pv_pmap); oldpmap = pmap_switch(pv->pv_pmap); @@ -1965,8 +1965,8 @@ pmap_ts_referenced(vm_page_t m) pmap_switch(oldpmap); PMAP_UNLOCK(pv->pv_pmap); } - - return count; + vm_page_unlock_queues(); + return (count); } /* diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c index 87c6678..a55814b 100644 --- a/sys/mips/mips/pmap.c +++ b/sys/mips/mips/pmap.c @@ -2348,20 +2348,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return FALSE; - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { - return TRUE; + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -2594,14 +2597,16 @@ pmap_remove_write(vm_page_t m) int pmap_ts_referenced(vm_page_t m) { - if (m->flags & PG_FICTITIOUS) - return (0); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); if (m->md.pv_flags & PV_TABLE_REF) { + vm_page_lock_queues(); m->md.pv_flags &= ~PV_TABLE_REF; - return 1; + vm_page_unlock_queues(); + return (1); } - return 0; + return (0); } /* diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index 3d7845d..b2663be 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -286,7 +286,7 @@ static void moea_enter_locked(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); static void moea_syncicache(vm_offset_t, vm_size_t); static boolean_t moea_query_bit(vm_page_t, int); -static u_int moea_clear_bit(vm_page_t, int, int *); +static u_int moea_clear_bit(vm_page_t, int); static void moea_kremove(mmu_t, vm_offset_t); int moea_pte_spill(vm_offset_t); @@ -1315,9 +1315,7 @@ moea_clear_reference(mmu_t mmu, vm_page_t m) KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea_clear_reference: page %p is not managed", m)); - vm_page_lock_queues(); - moea_clear_bit(m, PTE_REF, NULL); - vm_page_unlock_queues(); + moea_clear_bit(m, PTE_REF); } void @@ -1337,9 +1335,7 @@ moea_clear_modify(mmu_t mmu, vm_page_t m) */ if ((m->flags & PG_WRITEABLE) == 0) return; - vm_page_lock_queues(); - moea_clear_bit(m, PTE_CHG, NULL); - vm_page_unlock_queues(); + moea_clear_bit(m, PTE_CHG); } /* @@ -1409,14 +1405,10 @@ moea_remove_write(mmu_t mmu, vm_page_t m) boolean_t moea_ts_referenced(mmu_t mmu, vm_page_t m) { - int count; - - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (0); - count = moea_clear_bit(m, PTE_REF, NULL); - - return (count); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("moea_ts_referenced: page %p is not managed", m)); + return (moea_clear_bit(m, PTE_REF)); } /* @@ -1531,19 +1523,23 @@ moea_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m) { int loops; struct pvo_entry *pvo; + boolean_t rv; - if (!moea_initialized || (m->flags & PG_FICTITIOUS)) - return FALSE; - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("moea_page_exists_quick: page %p is not managed", m)); loops = 0; + rv = FALSE; + vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { - if (pvo->pvo_pmap == pmap) - return (TRUE); + if (pvo->pvo_pmap == pmap) { + rv = TRUE; + break; + } if (++loops >= 16) break; } - - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -1557,7 +1553,7 @@ moea_page_wired_mappings(mmu_t mmu, vm_page_t m) int count; count = 0; - if (!moea_initialized || (m->flags & PG_FICTITIOUS) != 0) + if ((m->flags & PG_FICTITIOUS) != 0) return (count); vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) @@ -2315,17 +2311,17 @@ moea_query_bit(vm_page_t m, int ptebit) } static u_int -moea_clear_bit(vm_page_t m, int ptebit, int *origbit) +moea_clear_bit(vm_page_t m, int ptebit) { u_int count; struct pvo_entry *pvo; struct pte *pt; - int rv; + + vm_page_lock_queues(); /* * Clear the cached value. */ - rv = moea_attr_fetch(m); moea_attr_clear(m, ptebit); /* @@ -2353,15 +2349,11 @@ moea_clear_bit(vm_page_t m, int ptebit, int *origbit) } mtx_unlock(&moea_table_mutex); } - rv |= pvo->pvo_pte.pte.pte_lo; pvo->pvo_pte.pte.pte_lo &= ~ptebit; MOEA_PVO_CHECK(pvo); /* sanity check */ } - if (origbit != NULL) { - *origbit = rv; - } - + vm_page_unlock_queues(); return (count); } diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c index 8ff27d0..62d85ce 100644 --- a/sys/powerpc/aim/mmu_oea64.c +++ b/sys/powerpc/aim/mmu_oea64.c @@ -358,7 +358,7 @@ static void moea64_bridge_cpu_bootstrap(mmu_t, int ap); static void moea64_enter_locked(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t); static boolean_t moea64_query_bit(vm_page_t, u_int64_t); -static u_int moea64_clear_bit(vm_page_t, u_int64_t, u_int64_t *); +static u_int moea64_clear_bit(vm_page_t, u_int64_t); static void moea64_kremove(mmu_t, vm_offset_t); static void moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz); @@ -1510,9 +1510,7 @@ moea64_clear_reference(mmu_t mmu, vm_page_t m) KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, ("moea64_clear_reference: page %p is not managed", m)); - vm_page_lock_queues(); - moea64_clear_bit(m, LPTE_REF, NULL); - vm_page_unlock_queues(); + moea64_clear_bit(m, LPTE_REF); } void @@ -1532,9 +1530,7 @@ moea64_clear_modify(mmu_t mmu, vm_page_t m) */ if ((m->flags & PG_WRITEABLE) == 0) return; - vm_page_lock_queues(); - moea64_clear_bit(m, LPTE_CHG, NULL); - vm_page_unlock_queues(); + moea64_clear_bit(m, LPTE_CHG); } /* @@ -1605,14 +1601,10 @@ moea64_remove_write(mmu_t mmu, vm_page_t m) boolean_t moea64_ts_referenced(mmu_t mmu, vm_page_t m) { - int count; - - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (0); - count = moea64_clear_bit(m, LPTE_REF, NULL); - - return (count); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("moea64_ts_referenced: page %p is not managed", m)); + return (moea64_clear_bit(m, LPTE_REF)); } /* @@ -1721,21 +1713,23 @@ moea64_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m) { int loops; struct pvo_entry *pvo; + boolean_t rv; - if (!moea64_initialized || (m->flags & PG_FICTITIOUS)) - return FALSE; - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("moea64_page_exists_quick: page %p is not managed", m)); loops = 0; + rv = FALSE; + vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) { - if (pvo->pvo_pmap == pmap) - return (TRUE); + if (pvo->pvo_pmap == pmap) { + rv = TRUE; + break; + } if (++loops >= 16) break; } - - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -1749,7 +1743,7 @@ moea64_page_wired_mappings(mmu_t mmu, vm_page_t m) int count; count = 0; - if (!moea64_initialized || (m->flags & PG_FICTITIOUS) != 0) + if ((m->flags & PG_FICTITIOUS) != 0) return (count); vm_page_lock_queues(); LIST_FOREACH(pvo, vm_page_to_pvoh(m), pvo_vlink) @@ -2445,19 +2439,17 @@ moea64_query_bit(vm_page_t m, u_int64_t ptebit) } static u_int -moea64_clear_bit(vm_page_t m, u_int64_t ptebit, u_int64_t *origbit) +moea64_clear_bit(vm_page_t m, u_int64_t ptebit) { u_int count; struct pvo_entry *pvo; struct lpte *pt; - uint64_t rv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_queues(); /* * Clear the cached value. */ - rv = moea64_attr_fetch(m); moea64_attr_clear(m, ptebit); /* @@ -2486,16 +2478,12 @@ moea64_clear_bit(vm_page_t m, u_int64_t ptebit, u_int64_t *origbit) moea64_pte_clear(pt, pvo->pvo_pmap, PVO_VADDR(pvo), ptebit); } } - rv |= pvo->pvo_pte.lpte.pte_lo; pvo->pvo_pte.lpte.pte_lo &= ~ptebit; MOEA_PVO_CHECK(pvo); /* sanity check */ UNLOCK_TABLE(); } - if (origbit != NULL) { - *origbit = rv; - } - + vm_page_unlock_queues(); return (count); } diff --git a/sys/powerpc/booke/pmap.c b/sys/powerpc/booke/pmap.c index 4bc5e93..2d368b4 100644 --- a/sys/powerpc/booke/pmap.c +++ b/sys/powerpc/booke/pmap.c @@ -2293,17 +2293,14 @@ mmu_booke_ts_referenced(mmu_t mmu, vm_page_t m) pv_entry_t pv; int count; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (0); - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("mmu_booke_ts_referenced: page %p is not managed", m)); count = 0; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { PMAP_LOCK(pv->pv_pmap); - if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL) { - if (!PTE_ISVALID(pte)) - goto make_sure_to_unlock; - + if ((pte = pte_find(mmu, pv->pv_pmap, pv->pv_va)) != NULL && + PTE_ISVALID(pte)) { if (PTE_ISREFERENCED(pte)) { mtx_lock_spin(&tlbivax_mutex); tlb_miss_lock(); @@ -2320,9 +2317,9 @@ mmu_booke_ts_referenced(mmu_t mmu, vm_page_t m) } } } -make_sure_to_unlock: PMAP_UNLOCK(pv->pv_pmap); } + vm_page_unlock_queues(); return (count); } @@ -2394,20 +2391,23 @@ mmu_booke_page_exists_quick(mmu_t mmu, pmap_t pmap, vm_page_t m) { pv_entry_t pv; int loops; + boolean_t rv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (FALSE); - + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("mmu_booke_page_exists_quick: page %p is not managed", m)); loops = 0; + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_link) { - if (pv->pv_pmap == pmap) - return (TRUE); - + if (pv->pv_pmap == pmap) { + rv = TRUE; + break; + } if (++loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index 7769bdc..736602c 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -1789,20 +1789,25 @@ pmap_page_exists_quick(pmap_t pm, vm_page_t m) { struct tte *tp; int loops; + boolean_t rv; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (FALSE); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); loops = 0; + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(tp, &m->md.tte_list, tte_link) { if ((tp->tte_data & TD_PV) == 0) continue; - if (TTE_GET_PMAP(tp) == pm) - return (TRUE); + if (TTE_GET_PMAP(tp) == pm) { + rv = TRUE; + break; + } if (++loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -1878,10 +1883,10 @@ pmap_ts_referenced(vm_page_t m) u_long data; int count; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); - if ((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) != 0) - return (0); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); count = 0; + vm_page_lock_queues(); if ((tp = TAILQ_FIRST(&m->md.tte_list)) != NULL) { tpf = tp; do { @@ -1895,6 +1900,7 @@ pmap_ts_referenced(vm_page_t m) break; } while ((tp = tpn) != NULL && tp != tpf); } + vm_page_unlock_queues(); return (count); } diff --git a/sys/sun4v/sun4v/pmap.c b/sys/sun4v/sun4v/pmap.c index 1448e6e..4da1dea 100644 --- a/sys/sun4v/sun4v/pmap.c +++ b/sys/sun4v/sun4v/pmap.c @@ -1738,20 +1738,23 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) { pv_entry_t pv; int loops = 0; + boolean_t rv; - if (m->flags & PG_FICTITIOUS) - return FALSE; - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_page_exists_quick: page %p is not managed", m)); + rv = FALSE; + vm_page_lock_queues(); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { if (pv->pv_pmap == pmap) { - return TRUE; + rv = TRUE; + break; } loops++; if (loops >= 16) break; } - return (FALSE); + vm_page_unlock_queues(); + return (rv); } /* @@ -2309,17 +2312,15 @@ pmap_tte_hash_resize(pmap_t pmap) int pmap_ts_referenced(vm_page_t m) { - int rv; pv_entry_t pv, pvf, pvn; pmap_t pmap; tte_t otte_data; + KASSERT((m->flags & (PG_FICTITIOUS | PG_UNMANAGED)) == 0, + ("pmap_ts_referenced: page %p is not managed", m)); rv = 0; - if (m->flags & PG_FICTITIOUS) - return (rv); - - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_queues(); if ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { pvf = pv; @@ -2347,6 +2348,7 @@ pmap_ts_referenced(vm_page_t m) PMAP_UNLOCK(pmap); } while ((pv = pvn) != NULL && pv != pvf); } + vm_page_unlock_queues(); return (rv); } diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 9d66599..1ec86af 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -502,6 +502,8 @@ vm_page_flag_clear(vm_page_t m, unsigned short bits) { mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT((bits & PG_REFERENCED) == 0 || VM_OBJECT_LOCKED(m->object), + ("PG_REFERENCED and !VM_OBJECT_LOCKED")); m->flags &= ~bits; } @@ -1333,8 +1335,7 @@ vm_waitpfault(void) /* * vm_page_requeue: * - * If the given page is contained within a page queue, move it to the tail - * of that queue. + * Move the given page to the tail of its present page queue. * * The page queues must be locked. */ @@ -1344,11 +1345,12 @@ vm_page_requeue(vm_page_t m) int queue = VM_PAGE_GETQUEUE(m); struct vpgqueues *vpq; - if (queue != PQ_NONE) { - vpq = &vm_page_queues[queue]; - TAILQ_REMOVE(&vpq->pl, m, pageq); - TAILQ_INSERT_TAIL(&vpq->pl, m, pageq); - } + mtx_assert(&vm_page_queue_mtx, MA_OWNED); + KASSERT(queue != PQ_NONE, + ("vm_page_requeue: page %p is not queued", m)); + vpq = &vm_page_queues[queue]; + TAILQ_REMOVE(&vpq->pl, m, pageq); + TAILQ_INSERT_TAIL(&vpq->pl, m, pageq); } /* diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index e8f68d2..3133ee4 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -219,6 +219,9 @@ extern struct vpglocks pa_lock[]; * pte mappings, nor can they be removed from their objects via * the object, and such pages are also not on any PQ queue. * + * PG_REFERENCED may be cleared only if the object containing the page is + * locked. + * * PG_WRITEABLE is set exclusively on managed pages by pmap_enter(). When it * does so, the page must be VPO_BUSY. */ diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 3158a7d..bff803d 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -547,21 +547,17 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags) /* * vm_pageout_object_deactivate_pages * - * deactivate enough pages to satisfy the inactive target - * requirements or if vm_page_proc_limit is set, then - * deactivate all of the pages in the object and its - * backing_objects. + * Deactivate enough pages to satisfy the inactive target + * requirements. * * The object and map must be locked. */ static void -vm_pageout_object_deactivate_pages(pmap, first_object, desired) - pmap_t pmap; - vm_object_t first_object; - long desired; +vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object, + long desired) { vm_object_t backing_object, object; - vm_page_t p, next; + vm_page_t p; int actcount, remove_mode; VM_OBJECT_LOCK_ASSERT(first_object, MA_OWNED); @@ -579,61 +575,57 @@ vm_pageout_object_deactivate_pages(pmap, first_object, desired) if (object->shadow_count > 1) remove_mode = 1; /* - * scan the objects entire memory queue + * Scan the object's entire memory queue. */ - p = TAILQ_FIRST(&object->memq); - while (p != NULL) { + TAILQ_FOREACH(p, &object->memq, listq) { if (pmap_resident_count(pmap) <= desired) goto unlock_return; - next = TAILQ_NEXT(p, listq); - if ((p->oflags & VPO_BUSY) != 0 || p->busy != 0) { - p = next; + if ((p->oflags & VPO_BUSY) != 0 || p->busy != 0) continue; - } + PCPU_INC(cnt.v_pdpages); vm_page_lock(p); - vm_page_lock_queues(); - cnt.v_pdpages++; - if (p->wire_count != 0 || - p->hold_count != 0 || + if (p->wire_count != 0 || p->hold_count != 0 || !pmap_page_exists_quick(pmap, p)) { - vm_page_unlock_queues(); vm_page_unlock(p); - p = next; continue; } actcount = pmap_ts_referenced(p); - if (actcount) { - vm_page_flag_set(p, PG_REFERENCED); - } else if (p->flags & PG_REFERENCED) { - actcount = 1; + if ((p->flags & PG_REFERENCED) != 0) { + if (actcount == 0) + actcount = 1; + vm_page_lock_queues(); + vm_page_flag_clear(p, PG_REFERENCED); + vm_page_unlock_queues(); } - if ((p->queue != PQ_ACTIVE) && - (p->flags & PG_REFERENCED)) { + if (p->queue != PQ_ACTIVE && actcount != 0) { vm_page_activate(p); p->act_count += actcount; - vm_page_flag_clear(p, PG_REFERENCED); } else if (p->queue == PQ_ACTIVE) { - if ((p->flags & PG_REFERENCED) == 0) { - p->act_count -= min(p->act_count, ACT_DECLINE); - if (!remove_mode && (vm_pageout_algorithm || (p->act_count == 0))) { + if (actcount == 0) { + p->act_count -= min(p->act_count, + ACT_DECLINE); + if (!remove_mode && + (vm_pageout_algorithm || + p->act_count == 0)) { pmap_remove_all(p); vm_page_deactivate(p); } else { + vm_page_lock_queues(); vm_page_requeue(p); + vm_page_unlock_queues(); } } else { vm_page_activate(p); - vm_page_flag_clear(p, PG_REFERENCED); - if (p->act_count < (ACT_MAX - ACT_ADVANCE)) + if (p->act_count < ACT_MAX - + ACT_ADVANCE) p->act_count += ACT_ADVANCE; + vm_page_lock_queues(); vm_page_requeue(p); + vm_page_unlock_queues(); } - } else if (p->queue == PQ_INACTIVE) { + } else if (p->queue == PQ_INACTIVE) pmap_remove_all(p); - } - vm_page_unlock_queues(); vm_page_unlock(p); - p = next; } if ((backing_object = object->backing_object) == NULL) goto unlock_return; |