diff options
author | markj <markj@FreeBSD.org> | 2017-02-03 00:58:08 +0000 |
---|---|---|
committer | markj <markj@FreeBSD.org> | 2017-02-03 00:58:08 +0000 |
commit | 0f4a2a370df2931f2522aa4b0fe8bab484e1e9d9 (patch) | |
tree | 950f1d7822fc6bc416945a34dd378d1d16bce73a /sys | |
parent | 255e9c9c24c418684d57fcc94a27f982286ca903 (diff) | |
download | FreeBSD-src-0f4a2a370df2931f2522aa4b0fe8bab484e1e9d9.zip FreeBSD-src-0f4a2a370df2931f2522aa4b0fe8bab484e1e9d9.tar.gz |
MFC r311902:
Coalesce TLB shootdowns of global PTEs in pmap_advise() on x86.
Diffstat (limited to 'sys')
-rw-r--r-- | sys/amd64/amd64/pmap.c | 26 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 25 |
2 files changed, 35 insertions, 16 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index ae525da..d7c721c 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -6054,7 +6054,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) pdp_entry_t *pdpe; pd_entry_t oldpde, *pde; pt_entry_t *pte, PG_A, PG_G, PG_M, PG_RW, PG_V; - vm_offset_t va_next; + vm_offset_t va, va_next; vm_page_t m; boolean_t anychanged; @@ -6134,11 +6134,11 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) } if (va_next > eva) va_next = eva; + va = va_next; for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, sva += PAGE_SIZE) { - if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | - PG_V)) - continue; + if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) + goto maybe_invlrng; else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { if (advice == MADV_DONTNEED) { /* @@ -6153,12 +6153,22 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) } else if ((*pte & PG_A) != 0) atomic_clear_long(pte, PG_A); else - continue; - if ((*pte & PG_G) != 0) - pmap_invalidate_page(pmap, sva); - else + goto maybe_invlrng; + + if ((*pte & PG_G) != 0) { + if (va == va_next) + va = sva; + } else anychanged = TRUE; + continue; +maybe_invlrng: + if (va != va_next) { + pmap_invalidate_range(pmap, va, sva); + va = va_next; + } } + if (va != va_next) + pmap_invalidate_range(pmap, va, sva); } if (anychanged) pmap_invalidate_all(pmap); diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index d717876..a4a2169 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -4959,7 +4959,7 @@ pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice) { pd_entry_t oldpde, *pde; pt_entry_t *pte; - vm_offset_t pdnxt; + vm_offset_t va, pdnxt; vm_page_t m; boolean_t anychanged, pv_lists_locked; @@ -5020,11 +5020,11 @@ resume: } if (pdnxt > eva) pdnxt = eva; + va = pdnxt; for (pte = pmap_pte_quick(pmap, sva); sva != pdnxt; pte++, sva += PAGE_SIZE) { - if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | - PG_V)) - continue; + if ((*pte & (PG_MANAGED | PG_V)) != (PG_MANAGED | PG_V)) + goto maybe_invlrng; else if ((*pte & (PG_M | PG_RW)) == (PG_M | PG_RW)) { if (advice == MADV_DONTNEED) { /* @@ -5039,12 +5039,21 @@ resume: } else if ((*pte & PG_A) != 0) atomic_clear_int((u_int *)pte, PG_A); else - continue; - if ((*pte & PG_G) != 0) - pmap_invalidate_page(pmap, sva); - else + goto maybe_invlrng; + if ((*pte & PG_G) != 0) { + if (va == pdnxt) + va = sva; + } else anychanged = TRUE; + continue; +maybe_invlrng: + if (va != pdnxt) { + pmap_invalidate_range(pmap, va, sva); + va = pdnxt; + } } + if (va != pdnxt) + pmap_invalidate_range(pmap, va, sva); } if (anychanged) pmap_invalidate_all(pmap); |