diff options
Diffstat (limited to 'sys/arm')
-rw-r--r-- | sys/arm/arm/pmap-v6.c | 522 | ||||
-rw-r--r-- | sys/arm/include/pmap.h | 27 |
2 files changed, 470 insertions, 79 deletions
diff --git a/sys/arm/arm/pmap-v6.c b/sys/arm/arm/pmap-v6.c index a1351ed..1174660 100644 --- a/sys/arm/arm/pmap-v6.c +++ b/sys/arm/arm/pmap-v6.c @@ -141,6 +141,7 @@ /* Include header files */ #include "opt_vm.h" +#include "opt_pmap.h" #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); @@ -158,6 +159,7 @@ __FBSDID("$FreeBSD$"); #include <sys/rwlock.h> #include <sys/smp.h> #include <sys/sched.h> +#include <sys/sysctl.h> #include <vm/vm.h> #include <vm/vm_param.h> @@ -193,6 +195,12 @@ int pmap_debug_level = 0; #define PMAP_INLINE __inline #endif /* PMAP_DEBUG */ +#ifdef PV_STATS +#define PV_STAT(x) do { x ; } while (0) +#else +#define PV_STAT(x) do { } while (0) +#endif + #ifdef ARM_L2_PIPT #define pmap_l2cache_wbinv_range(va, pa, size) cpu_l2cache_wbinv_range((pa), (size)) #define pmap_l2cache_inv_range(va, pa, size) cpu_l2cache_inv_range((pa), (size)) @@ -206,8 +214,11 @@ extern struct pv_addr systempage; /* * Internal function prototypes */ -static void pmap_free_pv_entry (pv_entry_t); -static pv_entry_t pmap_get_pv_entry(void); + +static void pmap_free_pv_chunk(struct pv_chunk *pc); +static void pmap_free_pv_entry(pmap_t pmap, pv_entry_t pv); +static pv_entry_t pmap_get_pv_entry(pmap_t pmap, boolean_t try); +static vm_page_t pmap_pv_reclaim(pmap_t locked_pmap); static void pmap_enter_locked(pmap_t, vm_offset_t, vm_page_t, vm_prot_t, boolean_t, int); @@ -386,13 +397,73 @@ int pmap_needs_pte_sync; #define pmap_is_current(pm) ((pm) == pmap_kernel() || \ curproc->p_vmspace->vm_map.pmap == (pm)) -static uma_zone_t pvzone = NULL; + +/* + * Data for the pv entry allocation mechanism + */ +static TAILQ_HEAD(pch, pv_chunk) pv_chunks = TAILQ_HEAD_INITIALIZER(pv_chunks); +static int pv_entry_count, pv_entry_max, pv_entry_high_water; +static int shpgperproc = PMAP_SHPGPERPROC; + +struct pv_chunk *pv_chunkbase; /* KVA block for pv_chunks */ +int pv_maxchunks; /* How many chunks we have KVA for */ +vm_offset_t pv_vafree; /* Freelist stored in the PTE */ + +static __inline struct pv_chunk * +pv_to_chunk(pv_entry_t pv) +{ + + return ((struct pv_chunk *)((uintptr_t)pv & ~(uintptr_t)PAGE_MASK)); +} + +#define PV_PMAP(pv) (pv_to_chunk(pv)->pc_pmap) + +CTASSERT(sizeof(struct pv_chunk) == PAGE_SIZE); +CTASSERT(_NPCM == 8); +CTASSERT(_NPCPV == 252); + +#define PC_FREE0_6 0xfffffffful /* Free values for index 0 through 6 */ +#define PC_FREE7 0x0ffffffful /* Free values for index 7 */ + +static const uint32_t pc_freemask[_NPCM] = { + PC_FREE0_6, PC_FREE0_6, PC_FREE0_6, + PC_FREE0_6, PC_FREE0_6, PC_FREE0_6, + PC_FREE0_6, PC_FREE7 +}; + +static SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); + +SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_count, CTLFLAG_RD, &pv_entry_count, 0, + "Current number of pv entries"); + +#ifdef PV_STATS +static int pc_chunk_count, pc_chunk_allocs, pc_chunk_frees, pc_chunk_tryfail; + +SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_count, CTLFLAG_RD, &pc_chunk_count, 0, + "Current number of pv entry chunks"); +SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_allocs, CTLFLAG_RD, &pc_chunk_allocs, 0, + "Current number of pv entry chunks allocated"); +SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_frees, CTLFLAG_RD, &pc_chunk_frees, 0, + "Current number of pv entry chunks frees"); +SYSCTL_INT(_vm_pmap, OID_AUTO, pc_chunk_tryfail, CTLFLAG_RD, &pc_chunk_tryfail, 0, + "Number of times tried to get a chunk page but failed."); + +static long pv_entry_frees, pv_entry_allocs; +static int pv_entry_spare; + +SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_frees, CTLFLAG_RD, &pv_entry_frees, 0, + "Current number of pv entry frees"); +SYSCTL_LONG(_vm_pmap, OID_AUTO, pv_entry_allocs, CTLFLAG_RD, &pv_entry_allocs, 0, + "Current number of pv entry allocs"); +SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_spare, CTLFLAG_RD, &pv_entry_spare, 0, + "Current number of spare pv entries"); +#endif + uma_zone_t l2zone; static uma_zone_t l2table_zone; static vm_offset_t pmap_kernel_l2dtable_kva; static vm_offset_t pmap_kernel_l2ptp_kva; static vm_paddr_t pmap_kernel_l2ptp_phys; -static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; static struct rwlock pvh_global_lock; int l1_mem_types[] = { @@ -846,7 +917,7 @@ pmap_clearbit(struct vm_page *pg, u_int maskbits) */ TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) { va = pv->pv_va; - pm = pv->pv_pmap; + pm = PV_PMAP(pv); oflags = pv->pv_flags; pv->pv_flags &= ~maskbits; @@ -923,12 +994,10 @@ pmap_enter_pv(struct vm_page *pg, struct pv_entry *pve, pmap_t pm, rw_assert(&pvh_global_lock, RA_WLOCKED); PMAP_ASSERT_LOCKED(pm); - pve->pv_pmap = pm; pve->pv_va = va; pve->pv_flags = flags; TAILQ_INSERT_HEAD(&pg->md.pv_list, pve, pv_list); - TAILQ_INSERT_HEAD(&pm->pm_pvlist, pve, pv_plist); pg->md.pvh_attrs |= flags & (PVF_REF | PVF_MOD); if (pve->pv_flags & PVF_WIRED) ++pm->pm_stats.wired_count; @@ -948,7 +1017,7 @@ pmap_find_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) rw_assert(&pvh_global_lock, RA_WLOCKED); TAILQ_FOREACH(pv, &pg->md.pv_list, pv_list) - if (pm == pv->pv_pmap && va == pv->pv_va) + if (pm == PV_PMAP(pv) && va == pv->pv_va) break; return (pv); } @@ -1014,7 +1083,6 @@ pmap_nuke_pv(struct vm_page *pg, pmap_t pm, struct pv_entry *pve) PMAP_ASSERT_LOCKED(pm); TAILQ_REMOVE(&pg->md.pv_list, pve, pv_list); - TAILQ_REMOVE(&pm->pm_pvlist, pve, pv_plist); if (pve->pv_flags & PVF_WIRED) --pm->pm_stats.wired_count; @@ -1047,7 +1115,7 @@ pmap_remove_pv(struct vm_page *pg, pmap_t pm, vm_offset_t va) pve = TAILQ_FIRST(&pg->md.pv_list); while (pve) { - if (pve->pv_pmap == pm && pve->pv_va == va) { /* match? */ + if (PV_PMAP(pve) == pm && pve->pv_va == va) { /* match? */ pmap_nuke_pv(pg, pm, pve); break; } @@ -1142,6 +1210,48 @@ pmap_page_init(vm_page_t m) m->md.pv_memattr = VM_MEMATTR_DEFAULT; } +static vm_offset_t +pmap_ptelist_alloc(vm_offset_t *head) +{ + pt_entry_t *pte; + vm_offset_t va; + + va = *head; + if (va == 0) + return (va); /* Out of memory */ + pte = vtopte(va); + *head = *pte; + if ((*head & L2_TYPE_MASK) != L2_TYPE_INV) + panic("%s: va is not L2_TYPE_INV!", __func__); + *pte = 0; + return (va); +} + +static void +pmap_ptelist_free(vm_offset_t *head, vm_offset_t va) +{ + pt_entry_t *pte; + + if ((va & L2_TYPE_MASK) != L2_TYPE_INV) + panic("%s: freeing va that is not L2_TYPE INV!", __func__); + pte = vtopte(va); + *pte = *head; /* virtual! L2_TYPE is L2_TYPE_INV though */ + *head = va; +} + +static void +pmap_ptelist_init(vm_offset_t *head, void *base, int npages) +{ + int i; + vm_offset_t va; + + *head = 0; + for (i = npages - 1; i >= 0; i--) { + va = (vm_offset_t)base + i * PAGE_SIZE; + pmap_ptelist_free(head, va); + } +} + /* * Initialize the pmap module. * Called by vm_init, to initialize any structures that the pmap @@ -1150,7 +1260,6 @@ pmap_page_init(vm_page_t m) void pmap_init(void) { - int shpgperproc = PMAP_SHPGPERPROC; PDEBUG(1, printf("pmap_init: phys_start = %08x\n", PHYSADDR)); @@ -1160,21 +1269,35 @@ pmap_init(void) NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); /* - * Initialize the PV entry allocator. + * Initialize the address space for the pv chunks. */ - pvzone = uma_zcreate("PV ENTRY", sizeof (struct pv_entry), NULL, NULL, - NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM | UMA_ZONE_NOFREE); + TUNABLE_INT_FETCH("vm.pmap.shpgperproc", &shpgperproc); pv_entry_max = shpgperproc * maxproc + cnt.v_page_count; - uma_zone_reserve_kva(pvzone, pv_entry_max); + TUNABLE_INT_FETCH("vm.pmap.pv_entries", &pv_entry_max); + pv_entry_max = roundup(pv_entry_max, _NPCPV); pv_entry_high_water = 9 * (pv_entry_max / 10); + pv_maxchunks = MAX(pv_entry_max / _NPCPV, maxproc); + pv_chunkbase = (struct pv_chunk *)kmem_alloc_nofault(kernel_map, + PAGE_SIZE * pv_maxchunks); + + if (pv_chunkbase == NULL) + panic("pmap_init: not enough kvm for pv chunks"); + + pmap_ptelist_init(&pv_vafree, pv_chunkbase, pv_maxchunks); + /* * Now it is safe to enable pv_table recording. */ PDEBUG(1, printf("pmap_init: done!\n")); } +SYSCTL_INT(_vm_pmap, OID_AUTO, pv_entry_max, CTLFLAG_RD, &pv_entry_max, 0, + "Max number of PV entries"); +SYSCTL_INT(_vm_pmap, OID_AUTO, shpgperproc, CTLFLAG_RD, &shpgperproc, 0, + "Page share factor per proc"); + int pmap_fault_fixup(pmap_t pm, vm_offset_t va, vm_prot_t ftype, int user) { @@ -1656,7 +1779,7 @@ pmap_bootstrap(vm_offset_t firstaddr, struct pv_addr *l1pt) PMAP_LOCK_INIT(kernel_pmap); CPU_FILL(&kernel_pmap->pm_active); kernel_pmap->pm_domain = PMAP_DOMAIN_KERNEL; - TAILQ_INIT(&kernel_pmap->pm_pvlist); + TAILQ_INIT(&kernel_pmap->pm_pvchunk); /* * Initialize the global pv list lock. @@ -1924,38 +2047,61 @@ pmap_growkernel(vm_offset_t addr) void pmap_remove_pages(pmap_t pmap) { - struct pv_entry *pv, *npv; - struct l2_bucket *l2b = NULL; - vm_page_t m; - pt_entry_t *pt; - - rw_wlock(&pvh_global_lock); - PMAP_LOCK(pmap); - for (pv = TAILQ_FIRST(&pmap->pm_pvlist); pv; pv = npv) { - if (pv->pv_flags & PVF_WIRED) { - /* Cannot remove wired pages now. */ - npv = TAILQ_NEXT(pv, pv_plist); - continue; + struct pv_entry *pv; + struct l2_bucket *l2b = NULL; + vm_page_t m; + pt_entry_t *pt; + struct pv_chunk *pc, *npc; + uint32_t inuse, bitmask; + int allfree, bit, field, idx; + + rw_wlock(&pvh_global_lock); + PMAP_LOCK(pmap); + + TAILQ_FOREACH_SAFE(pc, &pmap->pm_pvchunk, pc_list, npc) { + allfree = 1; + for (field = 0; field < _NPCM; field++) { + inuse = ~pc->pc_map[field] & pc_freemask[field]; + while (inuse != 0) { + bit = ffs(inuse) - 1; + bitmask = 1ul << bit; + idx = field * sizeof(inuse) * NBBY + bit; + pv = &pc->pc_pventry[idx]; + inuse &= ~bitmask; + if (pv->pv_flags & PVF_WIRED) { + /* Cannot remove wired pages now. */ + allfree = 0; + continue; + } + l2b = pmap_get_l2_bucket(pmap, pv->pv_va); + KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); + pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; + m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK); + KASSERT((vm_offset_t)m >= KERNBASE, ("Trying to access non-existent page va %x pte %x", pv->pv_va, *pt)); + *pt = 0; + PTE_SYNC(pt); + + /* Mark free */ + PV_STAT(pv_entry_frees++); + PV_STAT(pv_entry_spare++); + pv_entry_count--; + pmap->pm_stats.resident_count--; + pc->pc_map[field] |= bitmask; + pmap_nuke_pv(m, pmap, pv); + pmap_free_l2_bucket(pmap, l2b, 1); + } } - pmap->pm_stats.resident_count--; - l2b = pmap_get_l2_bucket(pmap, pv->pv_va); - KASSERT(l2b != NULL, ("No L2 bucket in pmap_remove_pages")); - pt = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; - m = PHYS_TO_VM_PAGE(*pt & L2_ADDR_MASK); - KASSERT((vm_offset_t)m >= KERNBASE, ("Trying to access non-existent page va %x pte %x", pv->pv_va, *pt)); - *pt = 0; - PTE_SYNC(pt); - npv = TAILQ_NEXT(pv, pv_plist); - pmap_nuke_pv(m, pmap, pv); - if (TAILQ_EMPTY(&m->md.pv_list)) - vm_page_aflag_clear(m, PGA_WRITEABLE); - pmap_free_pv_entry(pv); - pmap_free_l2_bucket(pmap, l2b, 1); + if (allfree) { + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + pmap_free_pv_chunk(pc); + } + } - rw_wunlock(&pvh_global_lock); - cpu_tlb_flushID(); - cpu_cpwait(); - PMAP_UNLOCK(pmap); + + rw_wunlock(&pvh_global_lock); + cpu_tlb_flushID(); + cpu_cpwait(); + PMAP_UNLOCK(pmap); } @@ -2306,6 +2452,7 @@ void pmap_remove_all(vm_page_t m) { pv_entry_t pv; + pmap_t pmap; pt_entry_t *ptep; struct l2_bucket *l2b; boolean_t flush = FALSE; @@ -2320,25 +2467,26 @@ pmap_remove_all(vm_page_t m) rw_wlock(&pvh_global_lock); curpm = vmspace_pmap(curproc->p_vmspace); while ((pv = TAILQ_FIRST(&m->md.pv_list)) != NULL) { - if (flush == FALSE && (pv->pv_pmap == curpm || - pv->pv_pmap == pmap_kernel())) + pmap = PV_PMAP(pv); + if (flush == FALSE && (pmap == curpm || + pmap == pmap_kernel())) flush = TRUE; - PMAP_LOCK(pv->pv_pmap); - l2b = pmap_get_l2_bucket(pv->pv_pmap, pv->pv_va); + PMAP_LOCK(pmap); + l2b = pmap_get_l2_bucket(pmap, pv->pv_va); KASSERT(l2b != NULL, ("No l2 bucket")); ptep = &l2b->l2b_kva[l2pte_index(pv->pv_va)]; if (L2_S_WRITABLE(*ptep)) vm_page_dirty(m); *ptep = 0; - if (pmap_is_current(pv->pv_pmap)) + if (pmap_is_current(pmap)) PTE_SYNC(ptep); - pmap_free_l2_bucket(pv->pv_pmap, l2b, 1); - pv->pv_pmap->pm_stats.resident_count--; + pmap_free_l2_bucket(pmap, l2b, 1); + pmap->pm_stats.resident_count--; flags |= pv->pv_flags; - pmap_nuke_pv(m, pv->pv_pmap, pv); - PMAP_UNLOCK(pv->pv_pmap); - pmap_free_pv_entry(pv); + pmap_nuke_pv(m, pmap, pv); + pmap_free_pv_entry(pmap, pv); + PMAP_UNLOCK(pmap); } m->md.pvh_attrs &= ~(PVF_MOD | PVF_REF); @@ -2694,15 +2842,13 @@ do_l2b_alloc: if ((pve = pmap_remove_pv(opg, pmap, va))) { oflags = pve->pv_flags; - if (m && ((m->oflags & VPO_UNMANAGED))) { - pmap_free_pv_entry(pve); - pve = NULL; - } + if (m && ((m->oflags & VPO_UNMANAGED))) + pmap_free_pv_entry(pmap, pve); } } if ((m && !(m->oflags & VPO_UNMANAGED))) { - if ((!pve) && (pve = pmap_get_pv_entry()) == NULL) + if ((!pve) && (pve = pmap_get_pv_entry(pmap, FALSE)) == NULL) panic("pmap_enter: no pv entries"); KASSERT(va < kmi.clean_sva || va >= kmi.clean_eva, @@ -3024,7 +3170,7 @@ pmap_pinit(pmap_t pmap) CPU_ZERO(&pmap->pm_active); - TAILQ_INIT(&pmap->pm_pvlist); + TAILQ_INIT(&pmap->pm_pvchunk); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); pmap->pm_stats.resident_count = 1; if (vector_page < KERNBASE) { @@ -3040,31 +3186,253 @@ pmap_pinit(pmap_t pmap) * page management routines. ***************************************************/ +/* + * We are in a serious low memory condition. Resort to + * drastic measures to free some pages so we can allocate + * another pv entry chunk. + */ +static vm_page_t +pmap_pv_reclaim(pmap_t locked_pmap) +{ + struct pch newtail; + struct pv_chunk *pc; + struct l2_bucket *l2b = NULL; + pmap_t pmap; + pt_entry_t *pt; + pv_entry_t pv; + vm_offset_t va; + vm_page_t free, m, m_pc; + uint32_t inuse; + int bit, field, freed, idx; + + PMAP_ASSERT_LOCKED(locked_pmap); + pmap = NULL; + free = m_pc = NULL; + TAILQ_INIT(&newtail); + while ((pc = TAILQ_FIRST(&pv_chunks)) != NULL && (pv_vafree == 0 || + free == NULL)) { + TAILQ_REMOVE(&pv_chunks, pc, pc_lru); + if (pmap != pc->pc_pmap) { + if (pmap != NULL) { + cpu_tlb_flushID(); + cpu_cpwait(); + if (pmap != locked_pmap) + PMAP_UNLOCK(pmap); + } + pmap = pc->pc_pmap; + /* Avoid deadlock and lock recursion. */ + if (pmap > locked_pmap) + PMAP_LOCK(pmap); + else if (pmap != locked_pmap && !PMAP_TRYLOCK(pmap)) { + pmap = NULL; + TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); + continue; + } + } + + /* + * Destroy every non-wired, 4 KB page mapping in the chunk. + */ + freed = 0; + for (field = 0; field < _NPCM; field++) { + for (inuse = ~pc->pc_map[field] & pc_freemask[field]; + inuse != 0; inuse &= ~(1UL << bit)) { + bit = ffs(inuse) - 1; + idx = field * sizeof(inuse) * NBBY + bit; + pv = &pc->pc_pventry[idx]; + if (pv->pv_flags & PVF_WIRED) + continue; + + va = pv->pv_va; + l2b = pmap_get_l2_bucket(pmap, va); + KASSERT(l2b != NULL, ("No l2 bucket")); + pt = &l2b->l2b_kva[l2pte_index(va)]; + m = PHYS_TO_VM_PAGE(l2pte_pa(*pt)); + KASSERT((vm_offset_t)m >= KERNBASE, + ("Trying to access non-existent page " + "va %x pte %x in %s", va, *pt)); + *pt = 0; + PTE_SYNC(pt); + pmap_nuke_pv(m, pmap, pv); + pc->pc_map[field] |= 1UL << bit; + freed++; + } + } + if (freed == 0) { + TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); + continue; + } + /* Every freed mapping is for a 4 KB page. */ + pmap->pm_stats.resident_count -= freed; + PV_STAT(pv_entry_frees += freed); + PV_STAT(pv_entry_spare += freed); + pv_entry_count -= freed; + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + for (field = 0; field < _NPCM; field++) + if (pc->pc_map[field] != pc_freemask[field]) { + TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, + pc_list); + TAILQ_INSERT_TAIL(&newtail, pc, pc_lru); + + /* + * One freed pv entry in locked_pmap is + * sufficient. + */ + if (pmap == locked_pmap) + goto out; + break; + } + if (field == _NPCM) { + PV_STAT(pv_entry_spare -= _NPCPV); + PV_STAT(pc_chunk_count--); + PV_STAT(pc_chunk_frees++); + /* Entire chunk is free; return it. */ + m_pc = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc)); + pmap_qremove((vm_offset_t)pc, 1); + pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc); + break; + } + } +out: + TAILQ_CONCAT(&pv_chunks, &newtail, pc_lru); + if (pmap != NULL) { + cpu_tlb_flushID(); + cpu_cpwait(); + if (pmap != locked_pmap) + PMAP_UNLOCK(pmap); + } + return (m_pc); +} + +/* + * free the pv_entry back to the free list + */ static void -pmap_free_pv_entry(pv_entry_t pv) +pmap_free_pv_entry(pmap_t pmap, pv_entry_t pv) { + struct pv_chunk *pc; + int bit, field, idx; + + rw_assert(&pvh_global_lock, RA_WLOCKED); + PMAP_ASSERT_LOCKED(pmap); + PV_STAT(pv_entry_frees++); + PV_STAT(pv_entry_spare++); pv_entry_count--; - uma_zfree(pvzone, pv); + pc = pv_to_chunk(pv); + idx = pv - &pc->pc_pventry[0]; + field = idx / (sizeof(u_long) * NBBY); + bit = idx % (sizeof(u_long) * NBBY); + pc->pc_map[field] |= 1ul << bit; + for (idx = 0; idx < _NPCM; idx++) + if (pc->pc_map[idx] != pc_freemask[idx]) { + /* + * 98% of the time, pc is already at the head of the + * list. If it isn't already, move it to the head. + */ + if (__predict_false(TAILQ_FIRST(&pmap->pm_pvchunk) != + pc)) { + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, + pc_list); + } + return; + } + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + pmap_free_pv_chunk(pc); } +static void +pmap_free_pv_chunk(struct pv_chunk *pc) +{ + vm_page_t m; + + TAILQ_REMOVE(&pv_chunks, pc, pc_lru); + PV_STAT(pv_entry_spare -= _NPCPV); + PV_STAT(pc_chunk_count--); + PV_STAT(pc_chunk_frees++); + /* entire chunk is free, return it */ + m = PHYS_TO_VM_PAGE(pmap_kextract((vm_offset_t)pc)); + pmap_qremove((vm_offset_t)pc, 1); + vm_page_unwire(m, 0); + vm_page_free(m); + pmap_ptelist_free(&pv_vafree, (vm_offset_t)pc); + +} -/* - * get a new pv_entry, allocating a block from the system - * when needed. - * the memory allocation is performed bypassing the malloc code - * because of the possibility of allocations at interrupt time. - */ static pv_entry_t -pmap_get_pv_entry(void) +pmap_get_pv_entry(pmap_t pmap, boolean_t try) { - pv_entry_t ret_value; + static const struct timeval printinterval = { 60, 0 }; + static struct timeval lastprint; + struct pv_chunk *pc; + pv_entry_t pv; + vm_page_t m; + int bit, field, idx; + rw_assert(&pvh_global_lock, RA_WLOCKED); + PMAP_ASSERT_LOCKED(pmap); + PV_STAT(pv_entry_allocs++); pv_entry_count++; + if (pv_entry_count > pv_entry_high_water) - pagedaemon_wakeup(); - ret_value = uma_zalloc(pvzone, M_NOWAIT); - return ret_value; + if (ratecheck(&lastprint, &printinterval)) + printf("%s: Approaching the limit on PV entries.\n", + __func__); +retry: + pc = TAILQ_FIRST(&pmap->pm_pvchunk); + if (pc != NULL) { + for (field = 0; field < _NPCM; field++) { + if (pc->pc_map[field]) { + bit = ffs(pc->pc_map[field]) - 1; + break; + } + } + if (field < _NPCM) { + idx = field * sizeof(pc->pc_map[field]) * NBBY + bit; + pv = &pc->pc_pventry[idx]; + pc->pc_map[field] &= ~(1ul << bit); + /* If this was the last item, move it to tail */ + for (field = 0; field < _NPCM; field++) + if (pc->pc_map[field] != 0) { + PV_STAT(pv_entry_spare--); + return (pv); /* not full, return */ + } + TAILQ_REMOVE(&pmap->pm_pvchunk, pc, pc_list); + TAILQ_INSERT_TAIL(&pmap->pm_pvchunk, pc, pc_list); + PV_STAT(pv_entry_spare--); + return (pv); + } + } + /* + * Access to the ptelist "pv_vafree" is synchronized by the pvh + * global lock. If "pv_vafree" is currently non-empty, it will + * remain non-empty until pmap_ptelist_alloc() completes. + */ + if (pv_vafree == 0 || (m = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | + VM_ALLOC_NOOBJ | VM_ALLOC_WIRED)) == NULL) { + if (try) { + pv_entry_count--; + PV_STAT(pc_chunk_tryfail++); + return (NULL); + } + m = pmap_pv_reclaim(pmap); + if (m == NULL) + goto retry; + } + PV_STAT(pc_chunk_count++); + PV_STAT(pc_chunk_allocs++); + pc = (struct pv_chunk *)pmap_ptelist_alloc(&pv_vafree); + pmap_qenter((vm_offset_t)pc, &m, 1); + pc->pc_pmap = pmap; + pc->pc_map[0] = pc_freemask[0] & ~1ul; /* preallocated bit 0 */ + for (field = 1; field < _NPCM; field++) + pc->pc_map[field] = pc_freemask[field]; + TAILQ_INSERT_TAIL(&pv_chunks, pc, pc_lru); + pv = &pc->pc_pventry[0]; + TAILQ_INSERT_HEAD(&pmap->pm_pvchunk, pc, pc_list); + PV_STAT(pv_entry_spare += _NPCPV - 1); + return (pv); } /* @@ -3142,7 +3510,7 @@ pmap_remove(pmap_t pm, vm_offset_t sva, vm_offset_t eva) if (pve) { is_exec = PV_BEEN_EXECD(pve->pv_flags); is_refd = PV_BEEN_REFD(pve->pv_flags); - pmap_free_pv_entry(pve); + pmap_free_pv_entry(pm, pve); } } @@ -3385,7 +3753,7 @@ pmap_page_exists_quick(pmap_t pmap, vm_page_t m) rv = FALSE; rw_wlock(&pvh_global_lock); TAILQ_FOREACH(pv, &m->md.pv_list, pv_list) { - if (pv->pv_pmap == pmap) { + if (PV_PMAP(pv) == pmap) { rv = TRUE; break; } diff --git a/sys/arm/include/pmap.h b/sys/arm/include/pmap.h index 1b35ab9..3a4726f 100644 --- a/sys/arm/include/pmap.h +++ b/sys/arm/include/pmap.h @@ -116,6 +116,7 @@ struct pv_addr { }; struct pv_entry; +struct pv_chunk; struct md_page { int pvh_attrs; @@ -152,7 +153,11 @@ struct pmap { pd_entry_t *pm_pdir; /* KVA of page directory */ cpuset_t pm_active; /* active on cpus */ struct pmap_statistics pm_stats; /* pmap statictics */ +#if (ARM_MMU_V6 + ARM_MMU_V7) != 0 + TAILQ_HEAD(,pv_chunk) pm_pvchunk; /* list of mappings in pmap */ +#else TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ +#endif }; typedef struct pmap *pmap_t; @@ -180,13 +185,31 @@ extern struct pmap kernel_pmap_store; * mappings of that page. An entry is a pv_entry_t, the list is pv_list. */ typedef struct pv_entry { - pmap_t pv_pmap; /* pmap where mapping lies */ vm_offset_t pv_va; /* virtual address for mapping */ TAILQ_ENTRY(pv_entry) pv_list; - TAILQ_ENTRY(pv_entry) pv_plist; int pv_flags; /* flags (wired, etc...) */ +#if (ARM_MMU_V6 + ARM_MMU_V7) == 0 + pmap_t pv_pmap; /* pmap where mapping lies */ + TAILQ_ENTRY(pv_entry) pv_plist; +#endif } *pv_entry_t; +/* + * pv_entries are allocated in chunks per-process. This avoids the + * need to track per-pmap assignments. + */ +#define _NPCM 8 +#define _NPCPV 252 + +struct pv_chunk { + pmap_t pc_pmap; + TAILQ_ENTRY(pv_chunk) pc_list; + uint32_t pc_map[_NPCM]; /* bitmap; 1 = free */ + uint32_t pc_dummy[3]; /* aligns pv_chunk to 4KB */ + TAILQ_ENTRY(pv_chunk) pc_lru; + struct pv_entry pc_pventry[_NPCPV]; +}; + #ifdef _KERNEL boolean_t pmap_get_pde_pte(pmap_t, vm_offset_t, pd_entry_t **, pt_entry_t **); |