diff options
author | jchandra <jchandra@FreeBSD.org> | 2010-07-21 09:27:00 +0000 |
---|---|---|
committer | jchandra <jchandra@FreeBSD.org> | 2010-07-21 09:27:00 +0000 |
commit | 10dfd55de4c36ba1bf848c6973ffea562904c58a (patch) | |
tree | 3eec17c60bcda26e4e936937b8f883b4c7a68bb7 /sys/vm | |
parent | fbd45b20c30db99fedc41b36dfb0d325d0c2cec3 (diff) | |
download | FreeBSD-src-10dfd55de4c36ba1bf848c6973ffea562904c58a.zip FreeBSD-src-10dfd55de4c36ba1bf848c6973ffea562904c58a.tar.gz |
Redo the page table page allocation on MIPS, as suggested by
alc@.
The UMA zone based allocation is replaced by a scheme that creates
a new free page list for the KSEG0 region, and a new function
in sys/vm that allocates pages from a specific free page list.
This also fixes a race condition introduced by the UMA based page table
page allocation code. Dropping the page queue and pmap locks before
the call to uma_zfree, and re-acquiring them afterwards will introduce
a race condtion(noted by alc@).
The changes are :
- Revert the earlier changes in MIPS pmap.c that added UMA zone for
page table pages.
- Add a new freelist VM_FREELIST_HIGHMEM to MIPS vmparam.h for memory that
is not directly mapped (in 32bit kernel). Normal page allocations will first
try the HIGHMEM freelist and then the default(direct mapped) freelist.
- Add a new function 'vm_page_t vm_page_alloc_freelist(int flind, int
order, int req)' to vm/vm_page.c to allocate a page from a specified
freelist. The MIPS page table pages will be allocated using this function
from the freelist containing direct mapped pages.
- Move the page initialization code from vm_phys_alloc_contig() to a
new function vm_page_alloc_init(), and use this function to initialize
pages in vm_page_alloc_freelist() too.
- Split the function vm_phys_alloc_pages(int pool, int order) to create
vm_phys_alloc_freelist_pages(int flind, int pool, int order), and use
this function from both vm_page_alloc_freelist() and vm_phys_alloc_pages().
Reviewed by: alc
Diffstat (limited to 'sys/vm')
-rw-r--r-- | sys/vm/vm_page.c | 89 | ||||
-rw-r--r-- | sys/vm/vm_page.h | 3 | ||||
-rw-r--r-- | sys/vm/vm_phys.c | 135 | ||||
-rw-r--r-- | sys/vm/vm_phys.h | 1 |
4 files changed, 154 insertions, 74 deletions
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 348ab99..44b801e 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -1355,6 +1355,95 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) } /* + * Initialize a page that has been freshly dequeued from a freelist. + * The caller has to drop the vnode returned, if it is not NULL. + * + * To be called with vm_page_queue_free_mtx held. + */ +struct vnode * +vm_page_alloc_init(vm_page_t m) +{ + struct vnode *drop; + vm_object_t m_object; + + KASSERT(m->queue == PQ_NONE, + ("vm_page_alloc_init: page %p has unexpected queue %d", + m, m->queue)); + KASSERT(m->wire_count == 0, + ("vm_page_alloc_init: page %p is wired", m)); + KASSERT(m->hold_count == 0, + ("vm_page_alloc_init: page %p is held", m)); + KASSERT(m->busy == 0, + ("vm_page_alloc_init: page %p is busy", m)); + KASSERT(m->dirty == 0, + ("vm_page_alloc_init: page %p is dirty", m)); + KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, + ("vm_page_alloc_init: page %p has unexpected memattr %d", + m, pmap_page_get_memattr(m))); + mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); + drop = NULL; + if ((m->flags & PG_CACHED) != 0) { + m->valid = 0; + m_object = m->object; + vm_page_cache_remove(m); + if (m_object->type == OBJT_VNODE && + m_object->cache == NULL) + drop = m_object->handle; + } else { + KASSERT(VM_PAGE_IS_FREE(m), + ("vm_page_alloc_init: page %p is not free", m)); + KASSERT(m->valid == 0, + ("vm_page_alloc_init: free page %p is valid", m)); + cnt.v_free_count--; + } + if (m->flags & PG_ZERO) + vm_page_zero_count--; + /* Don't clear the PG_ZERO flag; we'll need it later. */ + m->flags = PG_UNMANAGED | (m->flags & PG_ZERO); + m->oflags = 0; + /* Unmanaged pages don't use "act_count". */ + return (drop); +} + +/* + * vm_page_alloc_freelist: + * + * Allocate a page from the specified freelist with specified order. + * Only the ALLOC_CLASS values in req are honored, other request flags + * are ignored. + */ +vm_page_t +vm_page_alloc_freelist(int flind, int order, int req) +{ + struct vnode *drop; + vm_page_t m; + int page_req; + + m = NULL; + page_req = req & VM_ALLOC_CLASS_MASK; + mtx_lock(&vm_page_queue_free_mtx); + /* + * Do not allocate reserved pages unless the req has asked for it. + */ + if (cnt.v_free_count + cnt.v_cache_count > cnt.v_free_reserved || + (page_req == VM_ALLOC_SYSTEM && + cnt.v_free_count + cnt.v_cache_count > cnt.v_interrupt_free_min) || + (page_req == VM_ALLOC_INTERRUPT && + cnt.v_free_count + cnt.v_cache_count > 0)) { + m = vm_phys_alloc_freelist_pages(flind, VM_FREEPOOL_DIRECT, order); + } + if (m == NULL) { + mtx_unlock(&vm_page_queue_free_mtx); + return (NULL); + } + drop = vm_page_alloc_init(m); + mtx_unlock(&vm_page_queue_free_mtx); + if (drop) + vdrop(drop); + return (m); +} + +/* * vm_wait: (also see VM_WAIT macro) * * Block until free pages are available for allocation diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index a877c43..86119a1 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -262,6 +262,7 @@ extern struct vpglocks pa_lock[]; * */ +struct vnode; extern int vm_page_zero_count; extern vm_page_t vm_page_array; /* First resident page in table */ @@ -339,6 +340,8 @@ void vm_pageq_remove(vm_page_t m); void vm_page_activate (vm_page_t); vm_page_t vm_page_alloc (vm_object_t, vm_pindex_t, int); +vm_page_t vm_page_alloc_freelist(int, int, int); +struct vnode *vm_page_alloc_init(vm_page_t); vm_page_t vm_page_grab (vm_object_t, vm_pindex_t, int); void vm_page_cache(vm_page_t); void vm_page_cache_free(vm_object_t, vm_pindex_t, vm_pindex_t); diff --git a/sys/vm/vm_phys.c b/sys/vm/vm_phys.c index a245462..3482337 100644 --- a/sys/vm/vm_phys.c +++ b/sys/vm/vm_phys.c @@ -301,49 +301,67 @@ vm_phys_add_page(vm_paddr_t pa) vm_page_t vm_phys_alloc_pages(int pool, int order) { + vm_page_t m; + int flind; + + for (flind = 0; flind < vm_nfreelists; flind++) { + m = vm_phys_alloc_freelist_pages(flind, pool, order); + if (m != NULL) + return (m); + } + return (NULL); +} + +/* + * Find and dequeue a free page on the given free list, with the + * specified pool and order + */ +vm_page_t +vm_phys_alloc_freelist_pages(int flind, int pool, int order) +{ struct vm_freelist *fl; struct vm_freelist *alt; - int flind, oind, pind; + int oind, pind; vm_page_t m; + KASSERT(flind < VM_NFREELIST, + ("vm_phys_alloc_freelist_pages: freelist %d is out of range", flind)); KASSERT(pool < VM_NFREEPOOL, - ("vm_phys_alloc_pages: pool %d is out of range", pool)); + ("vm_phys_alloc_freelist_pages: pool %d is out of range", pool)); KASSERT(order < VM_NFREEORDER, - ("vm_phys_alloc_pages: order %d is out of range", order)); + ("vm_phys_alloc_freelist_pages: order %d is out of range", order)); mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); - for (flind = 0; flind < vm_nfreelists; flind++) { - fl = vm_phys_free_queues[flind][pool]; - for (oind = order; oind < VM_NFREEORDER; oind++) { - m = TAILQ_FIRST(&fl[oind].pl); + fl = vm_phys_free_queues[flind][pool]; + for (oind = order; oind < VM_NFREEORDER; oind++) { + m = TAILQ_FIRST(&fl[oind].pl); + if (m != NULL) { + TAILQ_REMOVE(&fl[oind].pl, m, pageq); + fl[oind].lcnt--; + m->order = VM_NFREEORDER; + vm_phys_split_pages(m, oind, fl, order); + return (m); + } + } + + /* + * The given pool was empty. Find the largest + * contiguous, power-of-two-sized set of pages in any + * pool. Transfer these pages to the given pool, and + * use them to satisfy the allocation. + */ + for (oind = VM_NFREEORDER - 1; oind >= order; oind--) { + for (pind = 0; pind < VM_NFREEPOOL; pind++) { + alt = vm_phys_free_queues[flind][pind]; + m = TAILQ_FIRST(&alt[oind].pl); if (m != NULL) { - TAILQ_REMOVE(&fl[oind].pl, m, pageq); - fl[oind].lcnt--; + TAILQ_REMOVE(&alt[oind].pl, m, pageq); + alt[oind].lcnt--; m->order = VM_NFREEORDER; + vm_phys_set_pool(pool, m, oind); vm_phys_split_pages(m, oind, fl, order); return (m); } } - - /* - * The given pool was empty. Find the largest - * contiguous, power-of-two-sized set of pages in any - * pool. Transfer these pages to the given pool, and - * use them to satisfy the allocation. - */ - for (oind = VM_NFREEORDER - 1; oind >= order; oind--) { - for (pind = 0; pind < VM_NFREEPOOL; pind++) { - alt = vm_phys_free_queues[flind][pind]; - m = TAILQ_FIRST(&alt[oind].pl); - if (m != NULL) { - TAILQ_REMOVE(&alt[oind].pl, m, pageq); - alt[oind].lcnt--; - m->order = VM_NFREEORDER; - vm_phys_set_pool(pool, m, oind); - vm_phys_split_pages(m, oind, fl, order); - return (m); - } - } - } } return (NULL); } @@ -592,7 +610,7 @@ vm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high, { struct vm_freelist *fl; struct vm_phys_seg *seg; - vm_object_t m_object; + struct vnode *vp; vm_paddr_t pa, pa_last, size; vm_page_t deferred_vdrop_list, m, m_ret; int flind, i, oind, order, pind; @@ -687,50 +705,19 @@ done: vm_phys_split_pages(m_ret, oind, fl, order); for (i = 0; i < npages; i++) { m = &m_ret[i]; - KASSERT(m->queue == PQ_NONE, - ("vm_phys_alloc_contig: page %p has unexpected queue %d", - m, m->queue)); - KASSERT(m->wire_count == 0, - ("vm_phys_alloc_contig: page %p is wired", m)); - KASSERT(m->hold_count == 0, - ("vm_phys_alloc_contig: page %p is held", m)); - KASSERT(m->busy == 0, - ("vm_phys_alloc_contig: page %p is busy", m)); - KASSERT(m->dirty == 0, - ("vm_phys_alloc_contig: page %p is dirty", m)); - KASSERT(pmap_page_get_memattr(m) == VM_MEMATTR_DEFAULT, - ("vm_phys_alloc_contig: page %p has unexpected memattr %d", - m, pmap_page_get_memattr(m))); - if ((m->flags & PG_CACHED) != 0) { - m->valid = 0; - m_object = m->object; - vm_page_cache_remove(m); - if (m_object->type == OBJT_VNODE && - m_object->cache == NULL) { - /* - * Enqueue the vnode for deferred vdrop(). - * - * Unmanaged pages don't use "pageq", so it - * can be safely abused to construct a short- - * lived queue of vnodes. - */ - m->pageq.tqe_prev = m_object->handle; - m->pageq.tqe_next = deferred_vdrop_list; - deferred_vdrop_list = m; - } - } else { - KASSERT(VM_PAGE_IS_FREE(m), - ("vm_phys_alloc_contig: page %p is not free", m)); - KASSERT(m->valid == 0, - ("vm_phys_alloc_contig: free page %p is valid", m)); - cnt.v_free_count--; + vp = vm_page_alloc_init(m); + if (vp != NULL) { + /* + * Enqueue the vnode for deferred vdrop(). + * + * Unmanaged pages don't use "pageq", so it + * can be safely abused to construct a short- + * lived queue of vnodes. + */ + m->pageq.tqe_prev = (void *)vp; + m->pageq.tqe_next = deferred_vdrop_list; + deferred_vdrop_list = m; } - if (m->flags & PG_ZERO) - vm_page_zero_count--; - /* Don't clear the PG_ZERO flag; we'll need it later. */ - m->flags = PG_UNMANAGED | (m->flags & PG_ZERO); - m->oflags = 0; - /* Unmanaged pages don't use "act_count". */ } for (; i < roundup2(npages, 1 << imin(oind, order)); i++) { m = &m_ret[i]; diff --git a/sys/vm/vm_phys.h b/sys/vm/vm_phys.h index 0e012c3..0dbd96a 100644 --- a/sys/vm/vm_phys.h +++ b/sys/vm/vm_phys.h @@ -44,6 +44,7 @@ void vm_phys_add_page(vm_paddr_t pa); vm_page_t vm_phys_alloc_contig(unsigned long npages, vm_paddr_t low, vm_paddr_t high, unsigned long alignment, unsigned long boundary); +vm_page_t vm_phys_alloc_freelist_pages(int flind, int pool, int order); vm_page_t vm_phys_alloc_pages(int pool, int order); vm_paddr_t vm_phys_bootstrap_alloc(vm_size_t size, unsigned long alignment); void vm_phys_free_pages(vm_page_t m, int order); |