diff options
-rw-r--r-- | sys/amd64/amd64/pmap.c | 140 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 140 |
2 files changed, 114 insertions, 166 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index d30be9c..ae6c091 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -214,7 +214,6 @@ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va); -static int pmap_release_free_page(pmap_t pmap, vm_page_t p); static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex); static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va); static vm_page_t pmap_page_lookup(vm_object_t object, vm_pindex_t pindex); @@ -1295,7 +1294,9 @@ void pmap_pinit(pmap) register struct pmap *pmap; { - vm_page_t ptdpg; + vm_page_t ptdpg[NPGPTD]; + vm_offset_t pa; + int i; /* * No need to allocate page table space yet but we do need a valid @@ -1313,18 +1314,24 @@ pmap_pinit(pmap) NPGPTD); /* - * allocate the page directory page + * allocate the page directory page(s) */ - ptdpg = vm_page_grab(pmap->pm_pteobj, PTDPTDI, - VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); - vm_page_lock_queues(); - vm_page_flag_clear(ptdpg, PG_BUSY); - ptdpg->valid = VM_PAGE_BITS_ALL; - vm_page_unlock_queues(); + for (i = 0; i < NPGPTD; i++) { + ptdpg[i] = vm_page_grab(pmap->pm_pteobj, PTDPTDI + i, + VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED | + VM_ALLOC_ZERO); + vm_page_lock_queues(); + vm_page_flag_clear(ptdpg[i], PG_BUSY); + ptdpg[i]->valid = VM_PAGE_BITS_ALL; + vm_page_unlock_queues(); + } - pmap_qenter((vm_offset_t)pmap->pm_pdir, &ptdpg, NPGPTD); - if ((ptdpg->flags & PG_ZERO) == 0) - bzero(pmap->pm_pdir, PAGE_SIZE); + pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD); + + for (i = 0; i < NPGPTD; i++) { + if ((ptdpg[i]->flags & PG_ZERO) == 0) + bzero(pmap->pm_pdir + (i * NPDEPG), PAGE_SIZE); + } mtx_lock_spin(&allpmaps_lock); LIST_INSERT_HEAD(&allpmaps, pmap, pm_list); @@ -1336,9 +1343,11 @@ pmap_pinit(pmap) pmap->pm_pdir[MPPTDI] = PTD[MPPTDI]; #endif - /* install self-referential address mapping entry */ - pmap->pm_pdir[PTDPTDI] = - VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW | PG_A | PG_M; + /* install self-referential address mapping entry(s) */ + for (i = 0; i < NPGPTD; i++) { + pa = VM_PAGE_TO_PHYS(ptdpg[i]); + pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M; + } pmap->pm_active = 0; TAILQ_INIT(&pmap->pm_pvlist); @@ -1358,50 +1367,6 @@ pmap_pinit2(pmap) /* XXX: Remove this stub when no longer called */ } -static int -pmap_release_free_page(pmap_t pmap, vm_page_t p) -{ - pd_entry_t *pde = pmap->pm_pdir; - - /* - * This code optimizes the case of freeing non-busy - * page-table pages. Those pages are zero now, and - * might as well be placed directly into the zero queue. - */ - vm_page_lock_queues(); - if (vm_page_sleep_if_busy(p, FALSE, "pmaprl")) - return (0); - vm_page_busy(p); - - /* - * Remove the page table page from the processes address space. - */ - pde[p->pindex] = 0; - pmap->pm_stats.resident_count--; - - if (p->hold_count) { - panic("pmap_release: freeing held page table page"); - } - /* - * Page directory pages need to have the kernel - * stuff cleared, so they can go into the zero queue also. - */ - if (p->pindex == PTDPTDI) { - bzero(pde + KPTDI, nkpt * sizeof(pd_entry_t)); -#ifdef SMP - pde[MPPTDI] = 0; -#endif - pde[APTDPTDI] = 0; - pmap_kremove((vm_offset_t) pmap->pm_pdir); - } - - p->wire_count--; - atomic_subtract_int(&cnt.v_wire_count, 1); - vm_page_free_zero(p); - vm_page_unlock_queues(); - return 1; -} - /* * this routine is called if the page table page is not * mapped correctly. @@ -1525,36 +1490,45 @@ pmap_allocpte(pmap_t pmap, vm_offset_t va) void pmap_release(pmap_t pmap) { - vm_page_t p,n,ptdpg; - vm_object_t object = pmap->pm_pteobj; - int curgeneration; + vm_object_t object; + vm_page_t m; + int i; + + object = pmap->pm_pteobj; + + KASSERT(object->ref_count == 1, + ("pmap_release: pteobj reference count %d != 1", + object->ref_count)); + KASSERT(pmap->pm_stats.resident_count == 0, + ("pmap_release: pmap resident count %ld != 0", + pmap->pm_stats.resident_count)); -#if defined(DIAGNOSTIC) - if (object->ref_count != 1) - panic("pmap_release: pteobj reference count != 1"); -#endif - - ptdpg = NULL; mtx_lock_spin(&allpmaps_lock); LIST_REMOVE(pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); -retry: - curgeneration = object->generation; - for (p = TAILQ_FIRST(&object->memq); p != NULL; p = n) { - n = TAILQ_NEXT(p, listq); - if (p->pindex == PTDPTDI) { - ptdpg = p; - continue; - } - while (1) { - if (!pmap_release_free_page(pmap, p) && - (object->generation != curgeneration)) - goto retry; - } + + bzero(pmap->pm_pdir + KPTDI, nkpt * sizeof(*pmap->pm_pdir)); + for (i = 0; i < NPGPTD; i++) { + pmap->pm_pdir[PTDPTDI + i] = 0; + pmap->pm_pdir[APTDPTDI + i] = 0; } +#ifdef SMP + pmap->pm_pdir[MPPTDI] = 0; +#endif + + pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD); - if (ptdpg && !pmap_release_free_page(pmap, ptdpg)) - goto retry; + vm_page_lock_queues(); + for (i = 0; i < NPGPTD; i++) { + m = TAILQ_FIRST(&object->memq); + m->wire_count--; + atomic_subtract_int(&cnt.v_wire_count, 1); + vm_page_busy(m); + vm_page_free_zero(m); + } + KASSERT(TAILQ_EMPTY(&object->memq), + ("pmap_release: leaking page table pages")); + vm_page_unlock_queues(); } static int diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index d30be9c..ae6c091 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -214,7 +214,6 @@ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va); -static int pmap_release_free_page(pmap_t pmap, vm_page_t p); static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex); static pt_entry_t *pmap_pte_quick(pmap_t pmap, vm_offset_t va); static vm_page_t pmap_page_lookup(vm_object_t object, vm_pindex_t pindex); @@ -1295,7 +1294,9 @@ void pmap_pinit(pmap) register struct pmap *pmap; { - vm_page_t ptdpg; + vm_page_t ptdpg[NPGPTD]; + vm_offset_t pa; + int i; /* * No need to allocate page table space yet but we do need a valid @@ -1313,18 +1314,24 @@ pmap_pinit(pmap) NPGPTD); /* - * allocate the page directory page + * allocate the page directory page(s) */ - ptdpg = vm_page_grab(pmap->pm_pteobj, PTDPTDI, - VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED | VM_ALLOC_ZERO); - vm_page_lock_queues(); - vm_page_flag_clear(ptdpg, PG_BUSY); - ptdpg->valid = VM_PAGE_BITS_ALL; - vm_page_unlock_queues(); + for (i = 0; i < NPGPTD; i++) { + ptdpg[i] = vm_page_grab(pmap->pm_pteobj, PTDPTDI + i, + VM_ALLOC_NORMAL | VM_ALLOC_RETRY | VM_ALLOC_WIRED | + VM_ALLOC_ZERO); + vm_page_lock_queues(); + vm_page_flag_clear(ptdpg[i], PG_BUSY); + ptdpg[i]->valid = VM_PAGE_BITS_ALL; + vm_page_unlock_queues(); + } - pmap_qenter((vm_offset_t)pmap->pm_pdir, &ptdpg, NPGPTD); - if ((ptdpg->flags & PG_ZERO) == 0) - bzero(pmap->pm_pdir, PAGE_SIZE); + pmap_qenter((vm_offset_t)pmap->pm_pdir, ptdpg, NPGPTD); + + for (i = 0; i < NPGPTD; i++) { + if ((ptdpg[i]->flags & PG_ZERO) == 0) + bzero(pmap->pm_pdir + (i * NPDEPG), PAGE_SIZE); + } mtx_lock_spin(&allpmaps_lock); LIST_INSERT_HEAD(&allpmaps, pmap, pm_list); @@ -1336,9 +1343,11 @@ pmap_pinit(pmap) pmap->pm_pdir[MPPTDI] = PTD[MPPTDI]; #endif - /* install self-referential address mapping entry */ - pmap->pm_pdir[PTDPTDI] = - VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW | PG_A | PG_M; + /* install self-referential address mapping entry(s) */ + for (i = 0; i < NPGPTD; i++) { + pa = VM_PAGE_TO_PHYS(ptdpg[i]); + pmap->pm_pdir[PTDPTDI + i] = pa | PG_V | PG_RW | PG_A | PG_M; + } pmap->pm_active = 0; TAILQ_INIT(&pmap->pm_pvlist); @@ -1358,50 +1367,6 @@ pmap_pinit2(pmap) /* XXX: Remove this stub when no longer called */ } -static int -pmap_release_free_page(pmap_t pmap, vm_page_t p) -{ - pd_entry_t *pde = pmap->pm_pdir; - - /* - * This code optimizes the case of freeing non-busy - * page-table pages. Those pages are zero now, and - * might as well be placed directly into the zero queue. - */ - vm_page_lock_queues(); - if (vm_page_sleep_if_busy(p, FALSE, "pmaprl")) - return (0); - vm_page_busy(p); - - /* - * Remove the page table page from the processes address space. - */ - pde[p->pindex] = 0; - pmap->pm_stats.resident_count--; - - if (p->hold_count) { - panic("pmap_release: freeing held page table page"); - } - /* - * Page directory pages need to have the kernel - * stuff cleared, so they can go into the zero queue also. - */ - if (p->pindex == PTDPTDI) { - bzero(pde + KPTDI, nkpt * sizeof(pd_entry_t)); -#ifdef SMP - pde[MPPTDI] = 0; -#endif - pde[APTDPTDI] = 0; - pmap_kremove((vm_offset_t) pmap->pm_pdir); - } - - p->wire_count--; - atomic_subtract_int(&cnt.v_wire_count, 1); - vm_page_free_zero(p); - vm_page_unlock_queues(); - return 1; -} - /* * this routine is called if the page table page is not * mapped correctly. @@ -1525,36 +1490,45 @@ pmap_allocpte(pmap_t pmap, vm_offset_t va) void pmap_release(pmap_t pmap) { - vm_page_t p,n,ptdpg; - vm_object_t object = pmap->pm_pteobj; - int curgeneration; + vm_object_t object; + vm_page_t m; + int i; + + object = pmap->pm_pteobj; + + KASSERT(object->ref_count == 1, + ("pmap_release: pteobj reference count %d != 1", + object->ref_count)); + KASSERT(pmap->pm_stats.resident_count == 0, + ("pmap_release: pmap resident count %ld != 0", + pmap->pm_stats.resident_count)); -#if defined(DIAGNOSTIC) - if (object->ref_count != 1) - panic("pmap_release: pteobj reference count != 1"); -#endif - - ptdpg = NULL; mtx_lock_spin(&allpmaps_lock); LIST_REMOVE(pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); -retry: - curgeneration = object->generation; - for (p = TAILQ_FIRST(&object->memq); p != NULL; p = n) { - n = TAILQ_NEXT(p, listq); - if (p->pindex == PTDPTDI) { - ptdpg = p; - continue; - } - while (1) { - if (!pmap_release_free_page(pmap, p) && - (object->generation != curgeneration)) - goto retry; - } + + bzero(pmap->pm_pdir + KPTDI, nkpt * sizeof(*pmap->pm_pdir)); + for (i = 0; i < NPGPTD; i++) { + pmap->pm_pdir[PTDPTDI + i] = 0; + pmap->pm_pdir[APTDPTDI + i] = 0; } +#ifdef SMP + pmap->pm_pdir[MPPTDI] = 0; +#endif + + pmap_qremove((vm_offset_t)pmap->pm_pdir, NPGPTD); - if (ptdpg && !pmap_release_free_page(pmap, ptdpg)) - goto retry; + vm_page_lock_queues(); + for (i = 0; i < NPGPTD; i++) { + m = TAILQ_FIRST(&object->memq); + m->wire_count--; + atomic_subtract_int(&cnt.v_wire_count, 1); + vm_page_busy(m); + vm_page_free_zero(m); + } + KASSERT(TAILQ_EMPTY(&object->memq), + ("pmap_release: leaking page table pages")); + vm_page_unlock_queues(); } static int |