diff options
Diffstat (limited to 'sys/powerpc/aim/mmu_oea64.c')
-rw-r--r-- | sys/powerpc/aim/mmu_oea64.c | 191 |
1 files changed, 41 insertions, 150 deletions
diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c index 624e088..408919d 100644 --- a/sys/powerpc/aim/mmu_oea64.c +++ b/sys/powerpc/aim/mmu_oea64.c @@ -223,8 +223,6 @@ u_int moea64_pteg_mask; * PVO data. */ struct pvo_head *moea64_pvo_table; /* pvo entries by pteg index */ -struct pvo_head moea64_pvo_kunmanaged = /* list of unmanaged pages */ - LIST_HEAD_INITIALIZER(moea64_pvo_kunmanaged); uma_zone_t moea64_upvo_zone; /* zone for pvo entries for unmanaged pages */ uma_zone_t moea64_mpvo_zone; /* zone for pvo entries for managed pages */ @@ -622,8 +620,8 @@ moea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart, pte_lo |= LPTE_G; moea64_pvo_enter(mmup, kernel_pmap, moea64_upvo_zone, - &moea64_pvo_kunmanaged, pa, pa, - pte_lo, PVO_WIRED | PVO_LARGE); + NULL, pa, pa, pte_lo, + PVO_WIRED | PVO_LARGE); } } PMAP_UNLOCK(kernel_pmap); @@ -820,7 +818,7 @@ moea64_mid_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) kernel_pmap->pmap_phys = kernel_pmap; CPU_FILL(&kernel_pmap->pm_active); - LIST_INIT(&kernel_pmap->pmap_pvo); + RB_INIT(&kernel_pmap->pmap_pvo); PMAP_LOCK_INIT(kernel_pmap); @@ -1179,7 +1177,7 @@ moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, int error; if (!moea64_initialized) { - pvo_head = &moea64_pvo_kunmanaged; + pvo_head = NULL; pg = NULL; zone = moea64_upvo_zone; pvo_flags = 0; @@ -1197,7 +1195,7 @@ moea64_enter(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_page_t m, /* XXX change the pvo head for fake pages */ if ((m->oflags & VPO_UNMANAGED) != 0) { pvo_flags &= ~PVO_MANAGED; - pvo_head = &moea64_pvo_kunmanaged; + pvo_head = NULL; zone = moea64_upvo_zone; } @@ -1315,7 +1313,6 @@ moea64_extract(mmu_t mmu, pmap_t pm, vm_offset_t va) struct pvo_entry *pvo; vm_paddr_t pa; - LOCK_TABLE_RD(); PMAP_LOCK(pm); pvo = moea64_pvo_find_va(pm, va); if (pvo == NULL) @@ -1323,7 +1320,6 @@ moea64_extract(mmu_t mmu, pmap_t pm, vm_offset_t va) else pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va - PVO_VADDR(pvo)); - UNLOCK_TABLE_RD(); PMAP_UNLOCK(pm); return (pa); } @@ -1333,37 +1329,6 @@ moea64_extract(mmu_t mmu, pmap_t pm, vm_offset_t va) * pmap and virtual address pair if that mapping permits the given * protection. */ - -extern int pa_tryrelock_restart; - -static int -vm_page_pa_tryrelock_moea64(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) -{ - /* - * This is a duplicate of vm_page_pa_tryrelock(), but with proper - * handling of the table lock - */ - vm_paddr_t lockpa; - - lockpa = *locked; - *locked = pa; - if (lockpa) { - PA_LOCK_ASSERT(lockpa, MA_OWNED); - if (PA_LOCKPTR(pa) == PA_LOCKPTR(lockpa)) - return (0); - PA_UNLOCK(lockpa); - } - if (PA_TRYLOCK(pa)) - return (0); - UNLOCK_TABLE_RD(); - PMAP_UNLOCK(pmap); - atomic_add_int(&pa_tryrelock_restart, 1); - PA_LOCK(pa); - LOCK_TABLE_RD(); - PMAP_LOCK(pmap); - return (EAGAIN); -} - vm_page_t moea64_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot) { @@ -1373,21 +1338,19 @@ moea64_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot) m = NULL; pa = 0; - LOCK_TABLE_RD(); PMAP_LOCK(pmap); retry: pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF); if (pvo != NULL && (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) && ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == LPTE_RW || (prot & VM_PROT_WRITE) == 0)) { - if (vm_page_pa_tryrelock_moea64(pmap, + if (vm_page_pa_tryrelock(pmap, pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN, &pa)) goto retry; m = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); vm_page_hold(m); } PA_UNLOCK_COND(pa); - UNLOCK_TABLE_RD(); PMAP_UNLOCK(pmap); return (m); } @@ -1435,8 +1398,7 @@ moea64_uma_page_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) PMAP_LOCK(kernel_pmap); moea64_pvo_enter(installed_mmu, kernel_pmap, moea64_upvo_zone, - &moea64_pvo_kunmanaged, va, VM_PAGE_TO_PHYS(m), LPTE_M, - PVO_WIRED | PVO_BOOTSTRAP); + NULL, va, VM_PAGE_TO_PHYS(m), LPTE_M, PVO_WIRED | PVO_BOOTSTRAP); if (needed_lock) PMAP_UNLOCK(kernel_pmap); @@ -1510,12 +1472,10 @@ moea64_is_prefaultable(mmu_t mmu, pmap_t pmap, vm_offset_t va) struct pvo_entry *pvo; boolean_t rv; - LOCK_TABLE_RD(); PMAP_LOCK(pmap); pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF); rv = pvo == NULL || (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0; PMAP_UNLOCK(pmap); - UNLOCK_TABLE_RD(); return (rv); } @@ -1671,7 +1631,7 @@ moea64_kenter_attr(mmu_t mmu, vm_offset_t va, vm_offset_t pa, vm_memattr_t ma) LOCK_TABLE_WR(); PMAP_LOCK(kernel_pmap); error = moea64_pvo_enter(mmu, kernel_pmap, moea64_upvo_zone, - &moea64_pvo_kunmanaged, va, pa, pte_lo, PVO_WIRED); + NULL, va, pa, pte_lo, PVO_WIRED); PMAP_UNLOCK(kernel_pmap); UNLOCK_TABLE_WR(); @@ -1704,13 +1664,11 @@ moea64_kextract(mmu_t mmu, vm_offset_t va) if (va < VM_MIN_KERNEL_ADDRESS) return (va); - LOCK_TABLE_RD(); PMAP_LOCK(kernel_pmap); pvo = moea64_pvo_find_va(kernel_pmap, va); KASSERT(pvo != NULL, ("moea64_kextract: no addr found for %#" PRIxPTR, va)); pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va - PVO_VADDR(pvo)); - UNLOCK_TABLE_RD(); PMAP_UNLOCK(kernel_pmap); return (pa); } @@ -1857,7 +1815,7 @@ void moea64_pinit(mmu_t mmu, pmap_t pmap) { PMAP_LOCK_INIT(pmap); - LIST_INIT(&pmap->pmap_pvo); + RB_INIT(&pmap->pmap_pvo); pmap->pm_slb_tree_root = slb_alloc_tree(); pmap->pm_slb = slb_alloc_user_cache(); @@ -1871,7 +1829,7 @@ moea64_pinit(mmu_t mmu, pmap_t pmap) uint32_t hash; PMAP_LOCK_INIT(pmap); - LIST_INIT(&pmap->pmap_pvo); + RB_INIT(&pmap->pmap_pvo); if (pmap_bootstrapped) pmap->pmap_phys = (pmap_t)moea64_kextract(mmu, @@ -1967,7 +1925,7 @@ void moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) { - struct pvo_entry *pvo, *tpvo; + struct pvo_entry *pvo, *tpvo, key; CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva, eva, prot); @@ -1982,26 +1940,11 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva, LOCK_TABLE_RD(); PMAP_LOCK(pm); - if ((eva - sva)/PAGE_SIZE < pm->pm_stats.resident_count) { - while (sva < eva) { - #ifdef __powerpc64__ - if (pm != kernel_pmap && - user_va_to_slb_entry(pm, sva) == NULL) { - sva = roundup2(sva + 1, SEGMENT_LENGTH); - continue; - } - #endif - pvo = moea64_pvo_find_va(pm, sva); - if (pvo != NULL) - moea64_pvo_protect(mmu, pm, pvo, prot); - sva += PAGE_SIZE; - } - } else { - LIST_FOREACH_SAFE(pvo, &pm->pmap_pvo, pvo_plink, tpvo) { - if (PVO_VADDR(pvo) < sva || PVO_VADDR(pvo) >= eva) - continue; - moea64_pvo_protect(mmu, pm, pvo, prot); - } + key.pvo_vaddr = sva; + for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); + pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { + tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); + moea64_pvo_protect(mmu, pm, pvo, prot); } UNLOCK_TABLE_RD(); PMAP_UNLOCK(pm); @@ -2080,7 +2023,7 @@ moea64_remove_pages(mmu_t mmu, pmap_t pm) LOCK_TABLE_WR(); PMAP_LOCK(pm); - LIST_FOREACH_SAFE(pvo, &pm->pmap_pvo, pvo_plink, tpvo) { + RB_FOREACH_SAFE(pvo, pvo_tree, &pm->pmap_pvo, tpvo) { if (!(pvo->pvo_vaddr & PVO_WIRED)) moea64_pvo_remove(mmu, pvo); } @@ -2094,7 +2037,7 @@ moea64_remove_pages(mmu_t mmu, pmap_t pm) void moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva) { - struct pvo_entry *pvo, *tpvo; + struct pvo_entry *pvo, *tpvo, key; /* * Perform an unsynchronized read. This is, however, safe. @@ -2104,26 +2047,11 @@ moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva) LOCK_TABLE_WR(); PMAP_LOCK(pm); - if ((eva - sva)/PAGE_SIZE < pm->pm_stats.resident_count) { - while (sva < eva) { - #ifdef __powerpc64__ - if (pm != kernel_pmap && - user_va_to_slb_entry(pm, sva) == NULL) { - sva = roundup2(sva + 1, SEGMENT_LENGTH); - continue; - } - #endif - pvo = moea64_pvo_find_va(pm, sva); - if (pvo != NULL) - moea64_pvo_remove(mmu, pvo); - sva += PAGE_SIZE; - } - } else { - LIST_FOREACH_SAFE(pvo, &pm->pmap_pvo, pvo_plink, tpvo) { - if (PVO_VADDR(pvo) < sva || PVO_VADDR(pvo) >= eva) - continue; - moea64_pvo_remove(mmu, pvo); - } + key.pvo_vaddr = sva; + for (pvo = RB_NFIND(pvo_tree, &pm->pmap_pvo, &key); + pvo != NULL && PVO_VADDR(pvo) < eva; pvo = tpvo) { + tpvo = RB_NEXT(pvo_tree, &pm->pmap_pvo, pvo); + moea64_pvo_remove(mmu, pvo); } UNLOCK_TABLE_WR(); PMAP_UNLOCK(pm); @@ -2275,12 +2203,6 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, uma_zone_t zone, moea64_bpvo_pool_index++; bootstrap = 1; } else { - /* - * Note: drop the table lock around the UMA allocation in - * case the UMA allocator needs to manipulate the page - * table. The mapping we are working with is already - * protected by the PMAP lock. - */ pvo = uma_zalloc(zone, M_NOWAIT); } @@ -2297,7 +2219,7 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, uma_zone_t zone, if (flags & PVO_WIRED) pvo->pvo_vaddr |= PVO_WIRED; - if (pvo_head != &moea64_pvo_kunmanaged) + if (pvo_head != NULL) pvo->pvo_vaddr |= PVO_MANAGED; if (bootstrap) pvo->pvo_vaddr |= PVO_BOOTSTRAP; @@ -2310,15 +2232,17 @@ moea64_pvo_enter(mmu_t mmu, pmap_t pm, uma_zone_t zone, /* * Add to pmap list */ - LIST_INSERT_HEAD(&pm->pmap_pvo, pvo, pvo_plink); + RB_INSERT(pvo_tree, &pm->pmap_pvo, pvo); /* * Remember if the list was empty and therefore will be the first * item. */ - if (LIST_FIRST(pvo_head) == NULL) - first = 1; - LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); + if (pvo_head != NULL) { + if (LIST_FIRST(pvo_head) == NULL) + first = 1; + LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink); + } if (pvo->pvo_vaddr & PVO_WIRED) { pvo->pvo_pte.lpte.pte_hi |= LPTE_WIRED; @@ -2381,10 +2305,9 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo) pvo->pvo_pmap->pm_stats.wired_count--; /* - * Remove this PVO from the PV and pmap lists. + * Remove this PVO from the pmap list. */ - LIST_REMOVE(pvo, pvo_vlink); - LIST_REMOVE(pvo, pvo_plink); + RB_REMOVE(pvo_tree, &pvo->pvo_pmap->pmap_pvo, pvo); /* * Remove this from the overflow list and return it to the pool @@ -2398,6 +2321,7 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo) pg = PHYS_TO_VM_PAGE(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN); if ((pvo->pvo_vaddr & PVO_MANAGED) == PVO_MANAGED && pg != NULL) { + LIST_REMOVE(pvo, pvo_vlink); if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) { if (pvo->pvo_pte.lpte.pte_lo & LPTE_CHG) vm_page_dirty(pg); @@ -2421,41 +2345,10 @@ moea64_pvo_remove(mmu_t mmu, struct pvo_entry *pvo) static struct pvo_entry * moea64_pvo_find_va(pmap_t pm, vm_offset_t va) { - struct pvo_entry *pvo; - int ptegidx; - uint64_t vsid; - #ifdef __powerpc64__ - uint64_t slbv; - - if (pm == kernel_pmap) { - slbv = kernel_va_to_slbv(va); - } else { - struct slb *slb; - slb = user_va_to_slb_entry(pm, va); - /* The page is not mapped if the segment isn't */ - if (slb == NULL) - return NULL; - slbv = slb->slbv; - } + struct pvo_entry key; - vsid = (slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT; - if (slbv & SLBV_L) - va &= ~moea64_large_page_mask; - else - va &= ~ADDR_POFF; - ptegidx = va_to_pteg(vsid, va, slbv & SLBV_L); - #else - va &= ~ADDR_POFF; - vsid = va_to_vsid(pm, va); - ptegidx = va_to_pteg(vsid, va, 0); - #endif - - LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) { - if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) - break; - } - - return (pvo); + key.pvo_vaddr = va & ~ADDR_POFF; + return (RB_FIND(pvo_tree, &pm->pmap_pvo, &key)); } static boolean_t @@ -2549,21 +2442,21 @@ moea64_clear_bit(mmu_t mmu, vm_page_t m, u_int64_t ptebit) boolean_t moea64_dev_direct_mapped(mmu_t mmu, vm_offset_t pa, vm_size_t size) { - struct pvo_entry *pvo; + struct pvo_entry *pvo, key; vm_offset_t ppa; int error = 0; - LOCK_TABLE_RD(); PMAP_LOCK(kernel_pmap); - for (ppa = pa & ~ADDR_POFF; ppa < pa + size; ppa += PAGE_SIZE) { - pvo = moea64_pvo_find_va(kernel_pmap, ppa); + key.pvo_vaddr = ppa = pa & ~ADDR_POFF; + for (pvo = RB_FIND(pvo_tree, &kernel_pmap->pmap_pvo, &key); + ppa < pa + size; ppa += PAGE_SIZE, + pvo = RB_NEXT(pvo_tree, &kernel_pmap->pmap_pvo, pvo)) { if (pvo == NULL || (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) != ppa) { error = EFAULT; break; } } - UNLOCK_TABLE_RD(); PMAP_UNLOCK(kernel_pmap); return (error); @@ -2626,7 +2519,6 @@ moea64_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz) vm_paddr_t pa; vm_size_t len; - LOCK_TABLE_RD(); PMAP_LOCK(pm); while (sz > 0) { lim = round_page(va); @@ -2640,6 +2532,5 @@ moea64_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz) va += len; sz -= len; } - UNLOCK_TABLE_RD(); PMAP_UNLOCK(pm); } |