diff options
author | alc <alc@FreeBSD.org> | 2008-07-31 22:45:28 +0000 |
---|---|---|
committer | alc <alc@FreeBSD.org> | 2008-07-31 22:45:28 +0000 |
commit | ba620c543424ed214f813a9ea9d4a23bc6479d71 (patch) | |
tree | c88987876fd63060b42406f14f99b6de4ddecf9a /sys/amd64 | |
parent | ed77206d40f8acb7ba29f965d63786510eaf660f (diff) | |
download | FreeBSD-src-ba620c543424ed214f813a9ea9d4a23bc6479d71.zip FreeBSD-src-ba620c543424ed214f813a9ea9d4a23bc6479d71.tar.gz |
Enhance pmap_change_attr(). Specifically, avoid 2MB page demotions, cache
mode changes, and cache and TLB invalidation when some or all of the
specified range is already mapped with the specified cache mode.
Submitted by: Magesh Dhasayyan
Diffstat (limited to 'sys/amd64')
-rw-r--r-- | sys/amd64/amd64/pmap.c | 71 | ||||
-rw-r--r-- | sys/amd64/include/pmap.h | 4 |
2 files changed, 54 insertions, 21 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index ef5839c..ccf8080 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -230,9 +230,11 @@ static void pmap_fill_ptp(pt_entry_t *firstpte, pt_entry_t newpte); static void pmap_insert_pt_page(pmap_t pmap, vm_page_t mpte); static boolean_t pmap_is_modified_pvh(struct md_page *pvh); static vm_page_t pmap_lookup_pt_page(pmap_t pmap, vm_offset_t va); +static void pmap_pde_attr(pd_entry_t *pde, int cache_bits); static void pmap_promote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va); static boolean_t pmap_protect_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t sva, vm_prot_t prot); +static void pmap_pte_attr(pt_entry_t *pte, int cache_bits); static int pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, vm_page_t *free); static int pmap_remove_pte(pmap_t pmap, pt_entry_t *ptq, @@ -4250,27 +4252,24 @@ pmap_clear_reference(vm_page_t m) /* Adjust the cache mode for a 4KB page mapped via a PTE. */ static __inline void -pmap_pte_attr(vm_offset_t va, int mode) +pmap_pte_attr(pt_entry_t *pte, int cache_bits) { - pt_entry_t *pte; u_int opte, npte; - pte = vtopte(va); - /* * The cache mode bits are all in the low 32-bits of the * PTE, so we can just spin on updating the low 32-bits. */ do { opte = *(u_int *)pte; - npte = opte & ~(PG_PTE_PAT | PG_NC_PCD | PG_NC_PWT); - npte |= pmap_cache_bits(mode, 0); + npte = opte & ~PG_PTE_CACHE; + npte |= cache_bits; } while (npte != opte && !atomic_cmpset_int((u_int *)pte, opte, npte)); } /* Adjust the cache mode for a 2MB page mapped via a PDE. */ static __inline void -pmap_pde_attr(pd_entry_t *pde, int mode) +pmap_pde_attr(pd_entry_t *pde, int cache_bits) { u_int opde, npde; @@ -4280,8 +4279,8 @@ pmap_pde_attr(pd_entry_t *pde, int mode) */ do { opde = *(u_int *)pde; - npde = opde & ~(PG_PDE_PAT | PG_NC_PCD | PG_NC_PWT); - npde |= pmap_cache_bits(mode, 1); + npde = opde & ~PG_PDE_CACHE; + npde |= cache_bits; } while (npde != opde && !atomic_cmpset_int((u_int *)pde, opde, npde)); } @@ -4357,6 +4356,8 @@ pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) pdp_entry_t *pdpe; pd_entry_t *pde; pt_entry_t *pte; + int cache_bits_pte, cache_bits_pde; + boolean_t changed; base = trunc_page(va); offset = va & PAGE_MASK; @@ -4369,6 +4370,9 @@ pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) if (base < DMAP_MIN_ADDRESS) return (EINVAL); + cache_bits_pde = cache_bits_pte = -1; + changed = FALSE; + /* * Pages that aren't mapped aren't supported. Also break down 2MB pages * into 4KB pages if required. @@ -4387,6 +4391,18 @@ pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) } if (*pde & PG_PS) { /* + * If the current 2MB page already has the required + * memory type, then we need not demote this page. Just + * increment tmpva to the next 2MB page frame. + */ + if (cache_bits_pde < 0) + cache_bits_pde = pmap_cache_bits(mode, 1); + if ((*pde & PG_PDE_CACHE) == cache_bits_pde) { + tmpva = trunc_2mpage(tmpva) + NBPDR; + continue; + } + + /* * If the current offset aligns with a 2MB page frame * and there is at least 2MB left within the range, then * we need not break down this page into 4KB pages. @@ -4412,27 +4428,40 @@ pmap_change_attr(vm_offset_t va, vm_size_t size, int mode) /* * Ok, all the pages exist, so run through them updating their - * cache mode. + * cache mode if required. */ - for (tmpva = base; size > 0; ) { + for (tmpva = base; tmpva < base + size; ) { pde = pmap_pde(kernel_pmap, tmpva); if (*pde & PG_PS) { - pmap_pde_attr(pde, mode); - tmpva += NBPDR; - size -= NBPDR; + if (cache_bits_pde < 0) + cache_bits_pde = pmap_cache_bits(mode, 1); + if ((*pde & PG_PDE_CACHE) != cache_bits_pde) { + pmap_pde_attr(pde, cache_bits_pde); + if (!changed) + changed = TRUE; + } + tmpva = trunc_2mpage(tmpva) + NBPDR; } else { - pmap_pte_attr(tmpva, mode); + if (cache_bits_pte < 0) + cache_bits_pte = pmap_cache_bits(mode, 0); + pte = vtopte(tmpva); + if ((*pte & PG_PTE_CACHE) != cache_bits_pte) { + pmap_pte_attr(pte, cache_bits_pte); + if (!changed) + changed = TRUE; + } tmpva += PAGE_SIZE; - size -= PAGE_SIZE; } } /* - * Flush CPU caches to make sure any data isn't cached that shouldn't - * be, etc. - */ - pmap_invalidate_range(kernel_pmap, base, tmpva); - pmap_invalidate_cache(); + * Flush CPU caches if required to make sure any data isn't cached that + * shouldn't be, etc. + */ + if (changed) { + pmap_invalidate_range(kernel_pmap, base, tmpva); + pmap_invalidate_cache(); + } return (0); } diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index 332c281..ce3bf99 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -75,6 +75,10 @@ #define PG_PROT (PG_RW|PG_U) /* all protection bits . */ #define PG_N (PG_NC_PWT|PG_NC_PCD) /* Non-cacheable */ +/* Page level cache control fields used to determine the PAT type */ +#define PG_PDE_CACHE (PG_PDE_PAT | PG_NC_PWT | PG_NC_PCD) +#define PG_PTE_CACHE (PG_PTE_PAT | PG_NC_PWT | PG_NC_PCD) + /* * Promotion to a 2MB (PDE) page mapping requires that the corresponding 4KB * (PTE) page mappings have identical settings for the following fields: |