diff options
author | alc <alc@FreeBSD.org> | 2010-07-15 16:25:51 +0000 |
---|---|---|
committer | alc <alc@FreeBSD.org> | 2010-07-15 16:25:51 +0000 |
commit | 377e76ab8340f9043bd8916b5d175cd01d14e9c4 (patch) | |
tree | 0cdbd89ffa1b03c976c09ac3d65aef9f4a896b12 /sys/amd64 | |
parent | d4cce5f67be8873f101ffcc70a71feee8aebf77b (diff) | |
download | FreeBSD-src-377e76ab8340f9043bd8916b5d175cd01d14e9c4.zip FreeBSD-src-377e76ab8340f9043bd8916b5d175cd01d14e9c4.tar.gz |
Optimize pmap_remove()'s handling of PG_G mappings. Specifically,
instead of calling pmap_invalidate_page() for each PG_G mapping, call
pmap_invalidate_range() for each range of PG_G mappings. In addition,
eliminate a redundant call to pmap_invalidate_page(). Both
pmap_remove_pte() and pmap_remove_page() called pmap_invalidate_page()
when the mapping had the PG_G attribute. Now, only pmap_remove_page()
calls pmap_invalidate_page(). Altogether, these changes eliminate 53%
of the TLB shootdowns for a "buildworld" on a ZFS file system. On
FFS, the reduction is 3%.
MFC after: 6 weeks
Diffstat (limited to 'sys/amd64')
-rw-r--r-- | sys/amd64/amd64/pmap.c | 29 |
1 files changed, 15 insertions, 14 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 74731ab..79e3d67 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -2602,12 +2602,6 @@ pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, vm_offset_t va, oldpte = pte_load_clear(ptq); if (oldpte & PG_W) pmap->pm_stats.wired_count -= 1; - /* - * Machines that don't support invlpg, also don't support - * PG_G. - */ - if (oldpte & PG_G) - pmap_invalidate_page(kernel_pmap, va); pmap_resident_count_dec(pmap, 1); if (oldpte & PG_MANAGED) { m = PHYS_TO_VM_PAGE(oldpte & PG_FRAME); @@ -2647,7 +2641,7 @@ pmap_remove_page(pmap_t pmap, vm_offset_t va, pd_entry_t *pde, vm_page_t *free) void pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) { - vm_offset_t va_next; + vm_offset_t va, va_next; pml4_entry_t *pml4e; pdp_entry_t *pdpe; pd_entry_t ptpaddr, *pde; @@ -2748,20 +2742,27 @@ pmap_remove(pmap_t pmap, vm_offset_t sva, vm_offset_t eva) if (va_next > eva) va_next = eva; + va = va_next; for (pte = pmap_pde_to_pte(pde, sva); sva != va_next; pte++, sva += PAGE_SIZE) { - if (*pte == 0) + if (*pte == 0) { + if (va != va_next) { + pmap_invalidate_range(pmap, va, sva); + va = va_next; + } continue; - - /* - * The TLB entry for a PG_G mapping is invalidated - * by pmap_remove_pte(). - */ + } if ((*pte & PG_G) == 0) anyvalid = 1; - if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free)) + else if (va == va_next) + va = sva; + if (pmap_remove_pte(pmap, pte, sva, ptpaddr, &free)) { + sva += PAGE_SIZE; break; + } } + if (va != va_next) + pmap_invalidate_range(pmap, va, sva); } out: if (anyvalid) |