diff options
Diffstat (limited to 'sys/vm')
-rw-r--r-- | sys/vm/device_pager.c | 19 | ||||
-rw-r--r-- | sys/vm/sg_pager.c | 9 | ||||
-rw-r--r-- | sys/vm/swap_pager.c | 28 | ||||
-rw-r--r-- | sys/vm/uma_core.c | 2 | ||||
-rw-r--r-- | sys/vm/vm_contig.c | 2 | ||||
-rw-r--r-- | sys/vm/vm_fault.c | 62 | ||||
-rw-r--r-- | sys/vm/vm_glue.c | 14 | ||||
-rw-r--r-- | sys/vm/vm_kern.c | 2 | ||||
-rw-r--r-- | sys/vm/vm_mmap.c | 2 | ||||
-rw-r--r-- | sys/vm/vm_object.c | 89 | ||||
-rw-r--r-- | sys/vm/vm_page.c | 71 | ||||
-rw-r--r-- | sys/vm/vm_page.h | 29 | ||||
-rw-r--r-- | sys/vm/vm_pageout.c | 86 | ||||
-rw-r--r-- | sys/vm/vm_param.h | 8 | ||||
-rw-r--r-- | sys/vm/vnode_pager.c | 85 |
15 files changed, 433 insertions, 75 deletions
diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c index 9002e77..996f740 100644 --- a/sys/vm/device_pager.c +++ b/sys/vm/device_pager.c @@ -251,12 +251,16 @@ dev_pager_getpages(object, m, count, reqpage) VM_OBJECT_LOCK(object); dev_pager_updatefake(page, paddr, memattr); if (count > 1) { - vm_page_lock_queues(); + for (i = 0; i < count; i++) { - if (i != reqpage) + if (i != reqpage) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } } - vm_page_unlock_queues(); } } else { /* @@ -266,10 +270,13 @@ dev_pager_getpages(object, m, count, reqpage) page = dev_pager_getfake(paddr, memattr); VM_OBJECT_LOCK(object); TAILQ_INSERT_TAIL(&object->un_pager.devp.devp_pglist, page, pageq); - vm_page_lock_queues(); - for (i = 0; i < count; i++) + for (i = 0; i < count; i++) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } vm_page_insert(page, object, offset); m[reqpage] = page; } diff --git a/sys/vm/sg_pager.c b/sys/vm/sg_pager.c index a17fe82..845401a 100644 --- a/sys/vm/sg_pager.c +++ b/sys/vm/sg_pager.c @@ -198,10 +198,13 @@ sg_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) TAILQ_INSERT_TAIL(&object->un_pager.sgp.sgp_pglist, page, pageq); /* Free the original pages and insert this fake page into the object. */ - vm_page_lock_queues(); - for (i = 0; i < count; i++) + for (i = 0; i < count; i++) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } vm_page_insert(page, object, offset); m[reqpage] = page; page->valid = VM_PAGE_BITS_ALL; diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index f47719b..6017a52 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -1137,12 +1137,21 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) if (0 < i || j < count) { int k; - vm_page_lock_queues(); - for (k = 0; k < i; ++k) + + for (k = 0; k < i; ++k) { + vm_page_lock(m[k]); + vm_page_lock_queues(); swp_pager_free_nrpage(m[k]); - for (k = j; k < count; ++k) + vm_page_unlock_queues(); + vm_page_unlock(m[k]); + } + for (k = j; k < count; ++k) { + vm_page_lock(m[k]); + vm_page_lock_queues(); swp_pager_free_nrpage(m[k]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[k]); + } } /* @@ -1497,7 +1506,7 @@ swp_pager_async_iodone(struct buf *bp) object = bp->b_pages[0]->object; VM_OBJECT_LOCK(object); } - vm_page_lock_queues(); + /* * cleanup pages. If an error occurs writing to swap, we are in * very serious trouble. If it happens to be a disk error, though, @@ -1509,6 +1518,8 @@ swp_pager_async_iodone(struct buf *bp) for (i = 0; i < bp->b_npages; ++i) { vm_page_t m = bp->b_pages[i]; + vm_page_lock(m); + vm_page_lock_queues(); m->oflags &= ~VPO_SWAPINPROG; if (bp->b_ioflags & BIO_ERROR) { @@ -1605,8 +1616,9 @@ swp_pager_async_iodone(struct buf *bp) if (vm_page_count_severe()) vm_page_try_to_cache(m); } + vm_page_unlock_queues(); + vm_page_unlock(m); } - vm_page_unlock_queues(); /* * adjust pip. NOTE: the original parent may still have its own @@ -1702,10 +1714,12 @@ swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex) m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL|VM_ALLOC_RETRY); if (m->valid == VM_PAGE_BITS_ALL) { vm_object_pip_subtract(object, 1); + vm_page_lock(m); vm_page_lock_queues(); vm_page_activate(m); vm_page_dirty(m); vm_page_unlock_queues(); + vm_page_unlock(m); vm_page_wakeup(m); vm_pager_page_unswapped(m); return; @@ -1714,10 +1728,12 @@ swp_pager_force_pagein(vm_object_t object, vm_pindex_t pindex) if (swap_pager_getpages(object, &m, 1, 0) != VM_PAGER_OK) panic("swap_pager_force_pagein: read from swap failed");/*XXX*/ vm_object_pip_subtract(object, 1); + vm_page_lock(m); vm_page_lock_queues(); vm_page_dirty(m); vm_page_dontneed(m); vm_page_unlock_queues(); + vm_page_unlock(m); vm_page_wakeup(m); vm_pager_page_unswapped(m); } diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index a2d5633..00a8276 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -1022,10 +1022,12 @@ obj_alloc(uma_zone_t zone, int bytes, u_int8_t *flags, int wait) while (pages != startpages) { pages--; p = TAILQ_LAST(&object->memq, pglist); + vm_page_lock(p); vm_page_lock_queues(); vm_page_unwire(p, 0); vm_page_free(p); vm_page_unlock_queues(); + vm_page_unlock(p); } retkva = 0; goto done; diff --git a/sys/vm/vm_contig.c b/sys/vm/vm_contig.c index 78d7e28..7220055 100644 --- a/sys/vm/vm_contig.c +++ b/sys/vm/vm_contig.c @@ -257,9 +257,11 @@ retry: i -= PAGE_SIZE; m = vm_page_lookup(object, OFF_TO_IDX(offset + i)); + vm_page_lock(m); vm_page_lock_queues(); vm_page_free(m); vm_page_unlock_queues(); + vm_page_unlock(m); } VM_OBJECT_UNLOCK(object); vm_map_delete(map, addr, addr + size); diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 058cbb0..de74915 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -137,9 +137,11 @@ release_page(struct faultstate *fs) { vm_page_wakeup(fs->m); + vm_page_lock(fs->m); vm_page_lock_queues(); vm_page_deactivate(fs->m); vm_page_unlock_queues(); + vm_page_unlock(fs->m); fs->m = NULL; } @@ -161,9 +163,11 @@ unlock_and_deallocate(struct faultstate *fs) VM_OBJECT_UNLOCK(fs->object); if (fs->object != fs->first_object) { VM_OBJECT_LOCK(fs->first_object); + vm_page_lock(fs->first_m); vm_page_lock_queues(); vm_page_free(fs->first_m); vm_page_unlock_queues(); + vm_page_unlock(fs->first_m); vm_object_pip_wakeup(fs->first_object); VM_OBJECT_UNLOCK(fs->first_object); fs->first_m = NULL; @@ -305,12 +309,14 @@ RetryFault:; * removes the page from the backing object, * which is not what we want. */ + vm_page_lock(fs.m); vm_page_lock_queues(); if ((fs.m->cow) && (fault_type & VM_PROT_WRITE) && (fs.object == fs.first_object)) { vm_page_cowfault(fs.m); vm_page_unlock_queues(); + vm_page_unlock(fs.m); unlock_and_deallocate(&fs); goto RetryFault; } @@ -333,12 +339,15 @@ RetryFault:; */ if ((fs.m->oflags & VPO_BUSY) || fs.m->busy) { vm_page_unlock_queues(); + vm_page_unlock(fs.m); VM_OBJECT_UNLOCK(fs.object); if (fs.object != fs.first_object) { VM_OBJECT_LOCK(fs.first_object); + vm_page_lock(fs.first_m); vm_page_lock_queues(); vm_page_free(fs.first_m); vm_page_unlock_queues(); + vm_page_unlock(fs.first_m); vm_object_pip_wakeup(fs.first_object); VM_OBJECT_UNLOCK(fs.first_object); fs.first_m = NULL; @@ -358,6 +367,7 @@ RetryFault:; } vm_pageq_remove(fs.m); vm_page_unlock_queues(); + vm_page_unlock(fs.m); /* * Mark page busy for other processes, and the @@ -481,17 +491,25 @@ readrest: continue; if (!are_queues_locked) { are_queues_locked = TRUE; + vm_page_lock(mt); + vm_page_lock_queues(); + } else { + vm_page_unlock_queues(); + vm_page_lock(mt); vm_page_lock_queues(); } if (mt->hold_count || - mt->wire_count) + mt->wire_count) { + vm_page_unlock(mt); continue; + } pmap_remove_all(mt); if (mt->dirty) { vm_page_deactivate(mt); } else { vm_page_cache(mt); } + vm_page_unlock(mt); } if (are_queues_locked) vm_page_unlock_queues(); @@ -623,17 +641,21 @@ vnode_locked: */ if (((fs.map != kernel_map) && (rv == VM_PAGER_ERROR)) || (rv == VM_PAGER_BAD)) { + vm_page_lock(fs.m); vm_page_lock_queues(); vm_page_free(fs.m); vm_page_unlock_queues(); + vm_page_unlock(fs.m); fs.m = NULL; unlock_and_deallocate(&fs); return ((rv == VM_PAGER_ERROR) ? KERN_FAILURE : KERN_PROTECTION_FAILURE); } if (fs.object != fs.first_object) { + vm_page_lock(fs.m); vm_page_lock_queues(); vm_page_free(fs.m); vm_page_unlock_queues(); + vm_page_unlock(fs.m); fs.m = NULL; /* * XXX - we cannot just fall out at this @@ -746,18 +768,24 @@ vnode_locked: * We don't chase down the shadow chain */ fs.object == fs.first_object->backing_object) { + vm_page_lock(fs.first_m); vm_page_lock_queues(); /* * get rid of the unnecessary page */ vm_page_free(fs.first_m); + vm_page_unlock_queues(); + vm_page_unlock(fs.first_m); /* * grab the page and put it into the * process'es object. The page is * automatically made dirty. */ + vm_page_lock(fs.m); + vm_page_lock_queues(); vm_page_rename(fs.m, fs.first_object, fs.first_pindex); vm_page_unlock_queues(); + vm_page_unlock(fs.m); vm_page_busy(fs.m); fs.first_m = fs.m; fs.m = NULL; @@ -770,10 +798,17 @@ vnode_locked: fs.first_m->valid = VM_PAGE_BITS_ALL; if (wired && (fault_flags & VM_FAULT_CHANGE_WIRING) == 0) { + vm_page_lock(fs.first_m); vm_page_lock_queues(); vm_page_wire(fs.first_m); + vm_page_unlock_queues(); + vm_page_unlock(fs.first_m); + + vm_page_lock(fs.m); + vm_page_lock_queues(); vm_page_unwire(fs.m, FALSE); vm_page_unlock_queues(); + vm_page_unlock(fs.m); } /* * We no longer need the old page or object. @@ -923,6 +958,7 @@ vnode_locked: if ((fault_flags & VM_FAULT_CHANGE_WIRING) == 0 && wired == 0) vm_fault_prefault(fs.map->pmap, vaddr, fs.entry); VM_OBJECT_LOCK(fs.object); + vm_page_lock(fs.m); vm_page_lock_queues(); /* @@ -938,6 +974,7 @@ vnode_locked: vm_page_activate(fs.m); } vm_page_unlock_queues(); + vm_page_unlock(fs.m); vm_page_wakeup(fs.m); /* @@ -1014,9 +1051,11 @@ vm_fault_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry) } if (m->valid == VM_PAGE_BITS_ALL && (m->flags & PG_FICTITIOUS) == 0) { + vm_page_lock(m); vm_page_lock_queues(); pmap_enter_quick(pmap, addr, m, entry->protection); vm_page_unlock_queues(); + vm_page_unlock(m); } VM_OBJECT_UNLOCK(lobject); } @@ -1092,9 +1131,11 @@ vm_fault_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, if (pa != 0) { pmap_change_wiring(pmap, va, FALSE); if (!fictitious) { + vm_page_lock(PHYS_TO_VM_PAGE(pa)); vm_page_lock_queues(); vm_page_unwire(PHYS_TO_VM_PAGE(pa), 1); vm_page_unlock_queues(); + vm_page_unlock(PHYS_TO_VM_PAGE(pa)); } } } @@ -1237,13 +1278,26 @@ vm_fault_copy_entry(vm_map_t dst_map, vm_map_t src_map, * Mark it no longer busy, and put it on the active list. */ VM_OBJECT_LOCK(dst_object); - vm_page_lock_queues(); + if (upgrade) { + vm_page_lock(src_m); + vm_page_lock_queues(); vm_page_unwire(src_m, 0); + vm_page_unlock_queues(); + vm_page_lock(src_m); + + vm_page_lock(dst_m); + vm_page_lock_queues(); vm_page_wire(dst_m); - } else + vm_page_unlock_queues(); + vm_page_lock(dst_m); + } else { + vm_page_lock(dst_m); + vm_page_lock_queues(); vm_page_activate(dst_m); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_lock(dst_m); + } vm_page_wakeup(dst_m); } VM_OBJECT_UNLOCK(dst_object); diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c index 4eeaa4d..288c5d7 100644 --- a/sys/vm/vm_glue.c +++ b/sys/vm/vm_glue.c @@ -257,16 +257,18 @@ vm_imgact_hold_page(vm_object_t object, vm_ooffset_t offset) if (m == NULL) goto out; if (rv != VM_PAGER_OK) { + vm_page_lock(m); vm_page_lock_queues(); vm_page_free(m); vm_page_unlock_queues(); + vm_page_unlock(m); m = NULL; goto out; } } - vm_page_lock_queues(); + vm_page_lock(m); vm_page_hold(m); - vm_page_unlock_queues(); + vm_page_unlock(m); vm_page_wakeup(m); out: VM_OBJECT_UNLOCK(object); @@ -300,9 +302,9 @@ vm_imgact_unmap_page(struct sf_buf *sf) m = sf_buf_page(sf); sf_buf_free(sf); sched_unpin(); - vm_page_lock_queues(); + vm_page_lock(m); vm_page_unhold(m); - vm_page_unlock_queues(); + vm_page_unlock(m); } void @@ -434,10 +436,12 @@ vm_thread_stack_dispose(vm_object_t ksobj, vm_offset_t ks, int pages) m = vm_page_lookup(ksobj, i); if (m == NULL) panic("vm_thread_dispose: kstack already missing?"); + vm_page_lock(m); vm_page_lock_queues(); vm_page_unwire(m, 0); vm_page_free(m); vm_page_unlock_queues(); + vm_page_unlock(m); } VM_OBJECT_UNLOCK(ksobj); vm_object_deallocate(ksobj); @@ -524,9 +528,11 @@ vm_thread_swapout(struct thread *td) if (m == NULL) panic("vm_thread_swapout: kstack already missing?"); vm_page_dirty(m); + vm_page_lock(m); vm_page_lock_queues(); vm_page_unwire(m, 0); vm_page_unlock_queues(); + vm_page_unlock(m); } VM_OBJECT_UNLOCK(ksobj); } diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c index 739d289..ead6d0d 100644 --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -380,10 +380,12 @@ retry: i -= PAGE_SIZE; m = vm_page_lookup(kmem_object, OFF_TO_IDX(offset + i)); + vm_page_lock(m); vm_page_lock_queues(); vm_page_unwire(m, 0); vm_page_free(m); vm_page_unlock_queues(); + vm_page_unlock(m); } VM_OBJECT_UNLOCK(kmem_object); vm_map_delete(map, addr, addr + size); diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c index 88ed3d5..f9b3db3 100644 --- a/sys/vm/vm_mmap.c +++ b/sys/vm/vm_mmap.c @@ -866,6 +866,7 @@ RestartScan: */ if (m != NULL && m->valid != 0) { mincoreinfo = MINCORE_INCORE; + vm_page_lock(m); vm_page_lock_queues(); if (m->dirty || pmap_is_modified(m)) @@ -874,6 +875,7 @@ RestartScan: pmap_is_referenced(m)) mincoreinfo |= MINCORE_REFERENCED_OTHER; vm_page_unlock_queues(); + vm_page_unlock(m); } VM_OBJECT_UNLOCK(current->object.vm_object); } diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index c25ab77..47ef973 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -717,19 +717,21 @@ vm_object_terminate(vm_object_t object) * removes them from paging queues. Don't free wired pages, just * remove them from the object. */ - vm_page_lock_queues(); while ((p = TAILQ_FIRST(&object->memq)) != NULL) { KASSERT(!p->busy && (p->oflags & VPO_BUSY) == 0, ("vm_object_terminate: freeing busy page %p " "p->busy = %d, p->oflags %x\n", p, p->busy, p->oflags)); + vm_page_lock(p); + vm_page_lock_queues(); if (p->wire_count == 0) { vm_page_free(p); cnt.v_pfree++; } else { vm_page_remove(p); } + vm_page_unlock_queues(); + vm_page_unlock(p); } - vm_page_unlock_queues(); #if VM_NRESERVLEVEL > 0 if (__predict_false(!LIST_EMPTY(&object->rvq))) @@ -789,7 +791,6 @@ vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int tend = end; } - vm_page_lock_queues(); /* * If the caller is smart and only msync()s a range he knows is * dirty, we may be able to avoid an object scan. This results in @@ -818,8 +819,12 @@ vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int ++tscan; continue; } + vm_page_lock(p); + vm_page_lock_queues(); vm_page_test_dirty(p); if (p->dirty == 0) { + vm_page_unlock_queues(); + vm_page_unlock(p); if (--scanlimit == 0) break; ++tscan; @@ -830,6 +835,8 @@ vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int * this is a nosync page, we can't continue. */ if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) { + vm_page_unlock_queues(); + vm_page_unlock(p); if (--scanlimit == 0) break; ++tscan; @@ -842,6 +849,8 @@ vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int * page (i.e. had to sleep). */ tscan += vm_object_page_collect_flush(object, p, curgeneration, pagerflags); + vm_page_unlock_queues(); + vm_page_unlock(p); } /* @@ -871,8 +880,13 @@ vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int p->oflags |= VPO_CLEANCHK; if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) clearobjflags = 0; - else + else { + vm_page_lock(p); + vm_page_lock_queues(); pmap_remove_write(p); + vm_page_unlock_queues(); + vm_page_unlock(p); + } } if (clearobjflags && (tstart == 0) && (tend == object->size)) @@ -895,8 +909,12 @@ again: continue; } + vm_page_lock(p); + vm_page_lock_queues(); vm_page_test_dirty(p); if (p->dirty == 0) { + vm_page_unlock_queues(); + vm_page_unlock(p); p->oflags &= ~VPO_CLEANCHK; continue; } @@ -907,28 +925,37 @@ again: * not cleared in this case so we do not have to set them. */ if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) { + vm_page_unlock_queues(); + vm_page_unlock(p); p->oflags &= ~VPO_CLEANCHK; continue; } n = vm_object_page_collect_flush(object, p, curgeneration, pagerflags); - if (n == 0) + if (n == 0) { + vm_page_unlock_queues(); + vm_page_unlock(p); goto rescan; + } - if (object->generation != curgeneration) + if (object->generation != curgeneration) { + vm_page_unlock_queues(); + vm_page_unlock(p); goto rescan; + } /* * Try to optimize the next page. If we can't we pick up * our (random) scan where we left off. */ if (msync_flush_flags & MSYNC_FLUSH_SOFTSEQ) { + vm_page_unlock_queues(); + vm_page_unlock(p); if ((p = vm_page_lookup(object, pi + n)) != NULL) goto again; } } - vm_page_unlock_queues(); #if 0 VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc); #endif @@ -951,8 +978,10 @@ vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, vm_page_t ma[vm_pageout_page_count]; mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_assert(p, MA_OWNED); pi = p->pindex; while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) { + vm_page_lock(p); vm_page_lock_queues(); if (object->generation != curgeneration) { return(0); @@ -968,11 +997,16 @@ vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, (tp->oflags & VPO_CLEANCHK) == 0) || (tp->busy != 0)) break; + vm_page_unlock_queues(); + vm_page_lock(tp); + vm_page_lock_queues(); vm_page_test_dirty(tp); if (tp->dirty == 0) { + vm_page_unlock(tp); tp->oflags &= ~VPO_CLEANCHK; break; } + vm_page_unlock(tp); maf[ i - 1 ] = tp; maxf++; continue; @@ -992,11 +1026,16 @@ vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, (tp->oflags & VPO_CLEANCHK) == 0) || (tp->busy != 0)) break; + vm_page_unlock_queues(); + vm_page_lock(tp); + vm_page_lock_queues(); vm_page_test_dirty(tp); if (tp->dirty == 0) { + vm_page_unlock(tp); tp->oflags &= ~VPO_CLEANCHK; break; } + vm_page_unlock(tp); mab[ i - 1 ] = tp; maxb++; continue; @@ -1022,7 +1061,11 @@ vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, vm_pageout_flush(ma, runlen, pagerflags); for (i = 0; i < runlen; i++) { if (ma[i]->dirty) { + vm_page_unlock_queues(); + vm_page_lock(ma[i]); + vm_page_lock_queues(); pmap_remove_write(ma[i]); + vm_page_unlock(ma[i]); ma[i]->oflags |= VPO_CLEANCHK; /* @@ -1195,9 +1238,11 @@ shadowlookup: /* * If the page is not in a normal state, skip it. */ + vm_page_lock(m); vm_page_lock_queues(); if (m->hold_count != 0 || m->wire_count != 0) { vm_page_unlock_queues(); + vm_page_unlock(m); goto unlock_tobject; } if ((m->oflags & VPO_BUSY) || m->busy) { @@ -1209,6 +1254,7 @@ shadowlookup: */ vm_page_flag_set(m, PG_REFERENCED); vm_page_unlock_queues(); + vm_page_unlock(m); if (object != tobject) VM_OBJECT_UNLOCK(object); m->oflags |= VPO_WANTED; @@ -1243,6 +1289,7 @@ shadowlookup: vm_page_dontneed(m); } vm_page_unlock_queues(); + vm_page_unlock(m); if (advise == MADV_FREE && tobject->type == OBJT_SWAP) swap_pager_freespace(tobject, tpindex, 1); unlock_tobject: @@ -1405,7 +1452,6 @@ retry: m = TAILQ_NEXT(m, listq); } } - vm_page_lock_queues(); for (; m != NULL && (idx = m->pindex - offidxstart) < size; m = m_next) { m_next = TAILQ_NEXT(m, listq); @@ -1425,11 +1471,14 @@ retry: VM_OBJECT_LOCK(new_object); goto retry; } + vm_page_lock(m); + vm_page_lock_queues(); vm_page_rename(m, new_object, idx); + vm_page_unlock_queues(); + vm_page_unlock(m); /* page automatically made dirty by rename and cache handled */ vm_page_busy(m); } - vm_page_unlock_queues(); if (orig_object->type == OBJT_SWAP) { /* * swap_pager_copy() can sleep, in which case the orig_object's @@ -1597,6 +1646,7 @@ vm_object_backing_scan(vm_object_t object, int op) * Page is out of the parent object's range, we * can simply destroy it. */ + vm_page_lock(p); vm_page_lock_queues(); KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); @@ -1605,6 +1655,7 @@ vm_object_backing_scan(vm_object_t object, int op) else vm_page_remove(p); vm_page_unlock_queues(); + vm_page_unlock(p); p = next; continue; } @@ -1621,6 +1672,7 @@ vm_object_backing_scan(vm_object_t object, int op) * * Leave the parent's page alone */ + vm_page_lock(p); vm_page_lock_queues(); KASSERT(!pmap_page_is_mapped(p), ("freeing mapped page %p", p)); @@ -1629,6 +1681,7 @@ vm_object_backing_scan(vm_object_t object, int op) else vm_page_remove(p); vm_page_unlock_queues(); + vm_page_unlock(p); p = next; continue; } @@ -1648,9 +1701,11 @@ vm_object_backing_scan(vm_object_t object, int op) * If the page was mapped to a process, it can remain * mapped through the rename. */ + vm_page_lock(p); vm_page_lock_queues(); vm_page_rename(p, object, new_pindex); vm_page_unlock_queues(); + vm_page_unlock(p); /* page automatically made dirty by rename */ } p = next; @@ -1915,7 +1970,7 @@ again: p = TAILQ_NEXT(p, listq); } } - vm_page_lock_queues(); + /* * Assert: the variable p is either (1) the page with the * least pindex greater than or equal to the parameter pindex @@ -1934,6 +1989,8 @@ again: * cannot be freed. They can, however, be invalidated * if "clean_only" is FALSE. */ + vm_page_lock(p); + vm_page_lock_queues(); if ((wirings = p->wire_count) != 0 && (wirings = pmap_page_wired_mappings(p)) != p->wire_count) { /* Fictitious pages do not have managed mappings. */ @@ -1945,6 +2002,8 @@ again: p->valid = 0; vm_page_undirty(p); } + vm_page_unlock_queues(); + vm_page_unlock(p); continue; } if (vm_page_sleep_if_busy(p, TRUE, "vmopar")) @@ -1953,16 +2012,20 @@ again: ("vm_object_page_remove: page %p is fictitious", p)); if (clean_only && p->valid) { pmap_remove_write(p); - if (p->dirty) + if (p->dirty) { + vm_page_unlock_queues(); + vm_page_unlock(p); continue; + } } pmap_remove_all(p); /* Account for removal of managed, wired mappings. */ if (wirings != 0) p->wire_count -= wirings; vm_page_free(p); + vm_page_unlock_queues(); + vm_page_unlock(p); } - vm_page_unlock_queues(); vm_object_pip_wakeup(object); skipmemq: if (__predict_false(object->cache != NULL)) @@ -1997,9 +2060,11 @@ vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end) if (m == NULL) break; if (rv != VM_PAGER_OK) { + vm_page_lock(m); vm_page_lock_queues(); vm_page_free(m); vm_page_unlock_queues(); + vm_page_unlock(m); break; } } diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 7c149c9..4c0d385 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -115,6 +115,7 @@ __FBSDID("$FreeBSD$"); #include <sys/vnode.h> #include <vm/vm.h> +#include <vm/pmap.h> #include <vm/vm_param.h> #include <vm/vm_kern.h> #include <vm/vm_object.h> @@ -129,6 +130,24 @@ __FBSDID("$FreeBSD$"); #include <machine/md_var.h> +#if defined(__amd64__) || defined (__i386__) +extern struct sysctl_oid_list sysctl__vm_pmap_children; +#else +SYSCTL_NODE(_vm, OID_AUTO, pmap, CTLFLAG_RD, 0, "VM/pmap parameters"); +#endif + +static uint64_t pmap_tryrelock_calls; +SYSCTL_QUAD(_vm_pmap, OID_AUTO, tryrelock_calls, CTLFLAG_RD, + &pmap_tryrelock_calls, 0, "Number of tryrelock calls"); + +static int pmap_tryrelock_restart; +SYSCTL_INT(_vm_pmap, OID_AUTO, tryrelock_restart, CTLFLAG_RD, + &pmap_tryrelock_restart, 0, "Number of tryrelock restarts"); + +static int pmap_tryrelock_race; +SYSCTL_INT(_vm_pmap, OID_AUTO, tryrelock_race, CTLFLAG_RD, + &pmap_tryrelock_race, 0, "Number of tryrelock pmap race cases"); + /* * Associated with page of user-allocatable memory is a * page structure. @@ -138,6 +157,8 @@ struct vpgqueues vm_page_queues[PQ_COUNT]; struct vpglocks vm_page_queue_lock; struct vpglocks vm_page_queue_free_lock; +struct vpglocks pa_lock[PA_LOCK_COUNT] __aligned(CACHE_LINE_SIZE); + vm_page_t vm_page_array = 0; int vm_page_array_size = 0; long first_page = 0; @@ -158,6 +179,43 @@ CTASSERT(sizeof(u_long) >= 8); #endif /* + * Try to acquire a physical address lock while a pmap is locked. If we + * fail to trylock we unlock and lock the pmap directly and cache the + * locked pa in *locked. The caller should then restart their loop in case + * the virtual to physical mapping has changed. + */ +int +vm_page_pa_tryrelock(pmap_t pmap, vm_paddr_t pa, vm_paddr_t *locked) +{ + vm_paddr_t lockpa; + uint32_t gen_count; + + gen_count = pmap->pm_gen_count; + atomic_add_long((volatile long *)&pmap_tryrelock_calls, 1); + lockpa = *locked; + *locked = pa; + if (lockpa) { + PA_LOCK_ASSERT(lockpa, MA_OWNED); + if (PA_LOCKPTR(pa) == PA_LOCKPTR(lockpa)) + return (0); + PA_UNLOCK(lockpa); + } + if (PA_TRYLOCK(pa)) + return (0); + PMAP_UNLOCK(pmap); + atomic_add_int((volatile int *)&pmap_tryrelock_restart, 1); + PA_LOCK(pa); + PMAP_LOCK(pmap); + + if (pmap->pm_gen_count != gen_count + 1) { + pmap->pm_retries++; + atomic_add_int((volatile int *)&pmap_tryrelock_race, 1); + return (EAGAIN); + } + return (0); +} + +/* * vm_set_page_size: * * Sets the page size, perhaps based upon the memory @@ -271,6 +329,11 @@ vm_page_startup(vm_offset_t vaddr) mtx_init(&vm_page_queue_free_mtx, "vm page queue free mutex", NULL, MTX_DEF); + /* Setup page locks. */ + for (i = 0; i < PA_LOCK_COUNT; i++) + mtx_init(&pa_lock[i].data, "page lock", NULL, + MTX_DEF | MTX_RECURSE | MTX_DUPOK); + /* * Initialize the queue headers for the hold queue, the active queue, * and the inactive queue. @@ -489,7 +552,7 @@ void vm_page_hold(vm_page_t mem) { - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_assert(mem, MA_OWNED); mem->hold_count++; } @@ -497,7 +560,7 @@ void vm_page_unhold(vm_page_t mem) { - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_assert(mem, MA_OWNED); --mem->hold_count; KASSERT(mem->hold_count >= 0, ("vm_page_unhold: hold count < 0!!!")); if (mem->hold_count == 0 && VM_PAGE_INQUEUE2(mem, PQ_HOLD)) @@ -542,10 +605,13 @@ vm_page_sleep(vm_page_t m, const char *msg) { VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + if (!mtx_owned(vm_page_lockptr(m))) + vm_page_lock(m); if (!mtx_owned(&vm_page_queue_mtx)) vm_page_lock_queues(); vm_page_flag_set(m, PG_REFERENCED); vm_page_unlock_queues(); + vm_page_unlock(m); /* * It's possible that while we sleep, the page will get @@ -1425,6 +1491,7 @@ vm_page_free_toq(vm_page_t m) panic("vm_page_free: freeing wired page"); } if (m->hold_count != 0) { + vm_page_lock_assert(m, MA_OWNED); m->flags &= ~PG_ZERO; vm_page_enqueue(PQ_HOLD, m); } else { diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 662af98..35a81f8 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -177,9 +177,35 @@ struct vpglocks { } __aligned(CACHE_LINE_SIZE); extern struct vpglocks vm_page_queue_free_lock; +extern struct vpglocks pa_lock[]; -#define vm_page_queue_free_mtx vm_page_queue_free_lock.data +#if defined(__arm__) +#define PDRSHIFT PDR_SHIFT +#elif !defined(PDRSHIFT) +#define PDRSHIFT 21 +#endif +#define pa_index(pa) ((pa) >> PDRSHIFT) +#define PA_LOCKPTR(pa) &pa_lock[pa_index((pa)) % PA_LOCK_COUNT].data +#define PA_LOCKOBJPTR(pa) ((struct lock_object *)PA_LOCKPTR((pa))) +#define PA_LOCK(pa) mtx_lock(PA_LOCKPTR(pa)) +#define PA_TRYLOCK(pa) mtx_trylock(PA_LOCKPTR(pa)) +#define PA_UNLOCK(pa) mtx_unlock(PA_LOCKPTR(pa)) +#define PA_UNLOCK_COND(pa) \ + do { \ + if (pa) \ + PA_UNLOCK(pa); \ + } while (0) + +#define PA_LOCK_ASSERT(pa, a) mtx_assert(PA_LOCKPTR(pa), (a)) + +#define vm_page_lockptr(m) (PA_LOCKPTR(VM_PAGE_TO_PHYS((m)))) +#define vm_page_lock(m) mtx_lock(vm_page_lockptr((m))) +#define vm_page_unlock(m) mtx_unlock(vm_page_lockptr((m))) +#define vm_page_trylock(m) mtx_trylock(vm_page_lockptr((m))) +#define vm_page_lock_assert(m, a) mtx_assert(vm_page_lockptr((m)), (a)) + +#define vm_page_queue_free_mtx vm_page_queue_free_lock.data /* * These are the flags defined for vm_page. * @@ -324,6 +350,7 @@ void vm_page_dontneed(vm_page_t); void vm_page_deactivate (vm_page_t); void vm_page_insert (vm_page_t, vm_object_t, vm_pindex_t); vm_page_t vm_page_lookup (vm_object_t, vm_pindex_t); +int vm_page_pa_tryrelock(pmap_t, vm_paddr_t, vm_paddr_t *); void vm_page_remove (vm_page_t); void vm_page_rename (vm_page_t, vm_object_t, vm_pindex_t); void vm_page_requeue(vm_page_t m); diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 258da8b..95517de 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -252,7 +252,9 @@ vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next) TAILQ_INSERT_AFTER(&vm_page_queues[queue].pl, m, &marker, pageq); vm_page_unlock_queues(); + vm_page_unlock(m); VM_OBJECT_LOCK(object); + vm_page_lock(m); vm_page_lock_queues(); /* Page queue might have changed. */ @@ -275,8 +277,7 @@ vm_pageout_fallback_object_lock(vm_page_t m, vm_page_t *next) * late and we cannot do anything that will mess with the page. */ static int -vm_pageout_clean(m) - vm_page_t m; +vm_pageout_clean(vm_page_t m) { vm_object_t object; vm_page_t mc[2*vm_pageout_page_count]; @@ -284,7 +285,8 @@ vm_pageout_clean(m) int ib, is, page_base; vm_pindex_t pindex = m->pindex; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); + vm_page_lock_assert(m, MA_NOTOWNED); + vm_page_lock(m); VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); /* @@ -301,6 +303,7 @@ vm_pageout_clean(m) */ if ((m->hold_count != 0) || ((m->busy != 0) || (m->oflags & VPO_BUSY))) { + vm_page_unlock(m); return 0; } @@ -347,13 +350,19 @@ more: ib = 0; break; } + vm_page_lock(p); + vm_page_lock_queues(); vm_page_test_dirty(p); if (p->dirty == 0 || p->queue != PQ_INACTIVE || p->hold_count != 0) { /* may be undergoing I/O */ + vm_page_unlock(p); + vm_page_unlock_queues(); ib = 0; break; } + vm_page_unlock_queues(); + vm_page_unlock(p); mc[--page_base] = p; ++pageout_count; ++ib; @@ -374,12 +383,18 @@ more: if ((p->oflags & VPO_BUSY) || p->busy) { break; } + vm_page_lock(p); + vm_page_lock_queues(); vm_page_test_dirty(p); if (p->dirty == 0 || p->queue != PQ_INACTIVE || p->hold_count != 0) { /* may be undergoing I/O */ + vm_page_unlock_queues(); + vm_page_unlock(p); break; } + vm_page_unlock_queues(); + vm_page_unlock(p); mc[page_base + pageout_count] = p; ++pageout_count; ++is; @@ -393,6 +408,7 @@ more: if (ib && pageout_count < vm_pageout_page_count) goto more; + vm_page_unlock(m); /* * we allow reads during pageouts... */ @@ -416,7 +432,6 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags) int numpagedout = 0; int i; - mtx_assert(&vm_page_queue_mtx, MA_OWNED); VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); /* * Initiate I/O. Bump the vm_page_t->busy counter and @@ -433,17 +448,21 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags) ("vm_pageout_flush: partially invalid page %p index %d/%d", mc[i], i, count)); vm_page_io_start(mc[i]); + vm_page_lock(mc[i]); + vm_page_lock_queues(); pmap_remove_write(mc[i]); + vm_page_unlock(mc[i]); + vm_page_unlock_queues(); } - vm_page_unlock_queues(); vm_object_pip_add(object, count); vm_pager_put_pages(object, mc, count, flags, pageout_status); - vm_page_lock_queues(); for (i = 0; i < count; i++) { vm_page_t mt = mc[i]; + vm_page_lock(mt); + vm_page_lock_queues(); KASSERT(pageout_status[i] == VM_PAGER_PEND || (mt->flags & PG_WRITEABLE) == 0, ("vm_pageout_flush: page %p is not write protected", mt)); @@ -485,6 +504,8 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags) if (vm_page_count_severe()) vm_page_try_to_cache(mt); } + vm_page_unlock_queues(); + vm_page_unlock(mt); } return numpagedout; } @@ -527,10 +548,12 @@ vm_pageout_object_deactivate_pages(pmap, first_object, desired) * scan the objects entire memory queue */ p = TAILQ_FIRST(&object->memq); - vm_page_lock_queues(); while (p != NULL) { + vm_page_lock(p); + vm_page_lock_queues(); if (pmap_resident_count(pmap) <= desired) { vm_page_unlock_queues(); + vm_page_unlock(p); goto unlock_return; } next = TAILQ_NEXT(p, listq); @@ -540,6 +563,8 @@ vm_pageout_object_deactivate_pages(pmap, first_object, desired) p->busy != 0 || (p->oflags & VPO_BUSY) || !pmap_page_exists_quick(pmap, p)) { + vm_page_unlock_queues(); + vm_page_unlock(p); p = next; continue; } @@ -573,9 +598,10 @@ vm_pageout_object_deactivate_pages(pmap, first_object, desired) } else if (p->queue == PQ_INACTIVE) { pmap_remove_all(p); } + vm_page_unlock_queues(); + vm_page_unlock(p); p = next; } - vm_page_unlock_queues(); if ((backing_object = object->backing_object) == NULL) goto unlock_return; VM_OBJECT_LOCK(backing_object); @@ -742,7 +768,6 @@ rescan0: } next = TAILQ_NEXT(m, pageq); - object = m->object; /* * skip marker pages @@ -758,18 +783,26 @@ rescan0: addl_page_shortage++; continue; } + + if (!vm_page_trylock(m) || (object = m->object) == NULL) { + addl_page_shortage++; + continue; + } + /* * Don't mess with busy pages, keep in the front of the * queue, most likely are being paged out. */ if (!VM_OBJECT_TRYLOCK(object) && (!vm_pageout_fallback_object_lock(m, &next) || - m->hold_count != 0)) { + m->hold_count != 0)) { VM_OBJECT_UNLOCK(object); + vm_page_unlock(m); addl_page_shortage++; continue; } if (m->busy || (m->oflags & VPO_BUSY)) { + vm_page_unlock(m); VM_OBJECT_UNLOCK(object); addl_page_shortage++; continue; @@ -798,6 +831,7 @@ rescan0: vm_page_activate(m); VM_OBJECT_UNLOCK(object); m->act_count += (actcount + ACT_ADVANCE); + vm_page_unlock(m); continue; } @@ -813,6 +847,7 @@ rescan0: vm_page_activate(m); VM_OBJECT_UNLOCK(object); m->act_count += (actcount + ACT_ADVANCE + 1); + vm_page_unlock(m); continue; } @@ -898,6 +933,7 @@ rescan0: * Those objects are in a "rundown" state. */ if (!swap_pageouts_ok || (object->flags & OBJ_DEAD)) { + vm_page_unlock(m); VM_OBJECT_UNLOCK(object); vm_page_requeue(m); continue; @@ -937,6 +973,8 @@ rescan0: * of time. */ if (object->type == OBJT_VNODE) { + vm_page_unlock_queues(); + vm_page_unlock(m); vp = object->handle; if (vp->v_type == VREG && vn_start_write(vp, &mp, V_NOWAIT) != 0) { @@ -944,11 +982,11 @@ rescan0: ++pageout_lock_miss; if (object->flags & OBJ_MIGHTBEDIRTY) vnodes_skipped++; + vm_page_lock_queues(); goto unlock_and_continue; } KASSERT(mp != NULL, ("vp %p with NULL v_mount", vp)); - vm_page_unlock_queues(); vm_object_reference_locked(object); VM_OBJECT_UNLOCK(object); vfslocked = VFS_LOCK_GIANT(vp->v_mount); @@ -963,6 +1001,7 @@ rescan0: goto unlock_and_continue; } VM_OBJECT_LOCK(object); + vm_page_lock(m); vm_page_lock_queues(); /* * The page might have been moved to another @@ -973,6 +1012,7 @@ rescan0: if (VM_PAGE_GETQUEUE(m) != PQ_INACTIVE || m->object != object || TAILQ_NEXT(m, pageq) != &marker) { + vm_page_unlock(m); if (object->flags & OBJ_MIGHTBEDIRTY) vnodes_skipped++; goto unlock_and_continue; @@ -985,6 +1025,7 @@ rescan0: * statistics are more correct if we don't. */ if (m->busy || (m->oflags & VPO_BUSY)) { + vm_page_unlock(m); goto unlock_and_continue; } @@ -993,12 +1034,14 @@ rescan0: * be undergoing I/O, so skip it */ if (m->hold_count) { + vm_page_unlock(m); vm_page_requeue(m); if (object->flags & OBJ_MIGHTBEDIRTY) vnodes_skipped++; goto unlock_and_continue; } } + vm_page_unlock(m); /* * If a page is dirty, then it is either being washed @@ -1010,11 +1053,14 @@ rescan0: * the (future) cleaned page. Otherwise we could wind * up laundering or cleaning too many pages. */ + vm_page_unlock_queues(); if (vm_pageout_clean(m) != 0) { --page_shortage; --maxlaunder; } + vm_page_lock_queues(); unlock_and_continue: + vm_page_lock_assert(m, MA_NOTOWNED); VM_OBJECT_UNLOCK(object); if (mp != NULL) { vm_page_unlock_queues(); @@ -1028,8 +1074,10 @@ unlock_and_continue: next = TAILQ_NEXT(&marker, pageq); TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, &marker, pageq); + vm_page_lock_assert(m, MA_NOTOWNED); continue; } + vm_page_unlock(m); VM_OBJECT_UNLOCK(object); } @@ -1048,6 +1096,7 @@ unlock_and_continue: */ pcount = cnt.v_active_count; m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl); + mtx_assert(&vm_page_queue_mtx, MA_OWNED); while ((m != NULL) && (pcount-- > 0) && (page_shortage > 0)) { @@ -1060,9 +1109,14 @@ unlock_and_continue: m = next; continue; } + if (!vm_page_trylock(m) || (object = m->object) == NULL) { + m = next; + continue; + } if (!VM_OBJECT_TRYLOCK(object) && !vm_pageout_fallback_object_lock(m, &next)) { VM_OBJECT_UNLOCK(object); + vm_page_unlock(m); m = next; continue; } @@ -1073,6 +1127,7 @@ unlock_and_continue: if ((m->busy != 0) || (m->oflags & VPO_BUSY) || (m->hold_count != 0)) { + vm_page_unlock(m); VM_OBJECT_UNLOCK(object); vm_page_requeue(m); m = next; @@ -1132,6 +1187,7 @@ unlock_and_continue: vm_page_requeue(m); } } + vm_page_unlock(m); VM_OBJECT_UNLOCK(object); m = next; } @@ -1317,9 +1373,15 @@ vm_pageout_page_stats() m = next; continue; } + vm_page_lock_assert(m, MA_NOTOWNED); + if (vm_page_trylock(m) == 0 || (object = m->object) == NULL) { + m = next; + continue; + } if (!VM_OBJECT_TRYLOCK(object) && !vm_pageout_fallback_object_lock(m, &next)) { VM_OBJECT_UNLOCK(object); + vm_page_unlock(m); m = next; continue; } @@ -1330,6 +1392,7 @@ vm_pageout_page_stats() if ((m->busy != 0) || (m->oflags & VPO_BUSY) || (m->hold_count != 0)) { + vm_page_unlock(m); VM_OBJECT_UNLOCK(object); vm_page_requeue(m); m = next; @@ -1366,6 +1429,7 @@ vm_pageout_page_stats() vm_page_requeue(m); } } + vm_page_unlock(m); VM_OBJECT_UNLOCK(object); m = next; } diff --git a/sys/vm/vm_param.h b/sys/vm/vm_param.h index 2ff2603..c404989 100644 --- a/sys/vm/vm_param.h +++ b/sys/vm/vm_param.h @@ -126,6 +126,14 @@ struct xswdev { #define KERN_NOT_RECEIVER 7 #define KERN_NO_ACCESS 8 +#ifndef PA_LOCK_COUNT +#ifdef SMP +#define PA_LOCK_COUNT 32 +#else +#define PA_LOCK_COUNT 1 +#endif /* !SMP */ +#endif /* !PA_LOCK_COUNT */ + #ifndef ASSEMBLER #ifdef _KERNEL #define num_pages(x) \ diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index aedc794..eb21c60 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -429,9 +429,11 @@ vnode_pager_setsize(vp, nsize) * bits. This would prevent bogus_page * replacement from working properly. */ + vm_page_lock(m); vm_page_lock_queues(); vm_page_clear_dirty(m, base, PAGE_SIZE - base); vm_page_unlock_queues(); + vm_page_unlock(m); } else if ((nsize & PAGE_MASK) && __predict_false(object->cache != NULL)) { vm_page_cache_free(object, OFF_TO_IDX(nsize), @@ -719,11 +721,15 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) error = VOP_BMAP(vp, foff / bsize, &bo, &reqblock, NULL, NULL); if (error == EOPNOTSUPP) { VM_OBJECT_LOCK(object); - vm_page_lock_queues(); + for (i = 0; i < count; i++) - if (i != reqpage) + if (i != reqpage) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } PCPU_INC(cnt.v_vnodein); PCPU_INC(cnt.v_vnodepgsin); error = vnode_pager_input_old(object, m[reqpage]); @@ -731,11 +737,14 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) return (error); } else if (error != 0) { VM_OBJECT_LOCK(object); - vm_page_lock_queues(); for (i = 0; i < count; i++) - if (i != reqpage) + if (i != reqpage) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } VM_OBJECT_UNLOCK(object); return (VM_PAGER_ERROR); @@ -747,11 +756,14 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) } else if ((PAGE_SIZE / bsize) > 1 && (vp->v_mount->mnt_stat.f_type != nfs_mount_type)) { VM_OBJECT_LOCK(object); - vm_page_lock_queues(); for (i = 0; i < count; i++) - if (i != reqpage) + if (i != reqpage) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } VM_OBJECT_UNLOCK(object); PCPU_INC(cnt.v_vnodein); PCPU_INC(cnt.v_vnodepgsin); @@ -765,11 +777,14 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) */ VM_OBJECT_LOCK(object); if (m[reqpage]->valid == VM_PAGE_BITS_ALL) { - vm_page_lock_queues(); for (i = 0; i < count; i++) - if (i != reqpage) + if (i != reqpage) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } VM_OBJECT_UNLOCK(object); return VM_PAGER_OK; } else if (reqblock == -1) { @@ -777,11 +792,14 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) KASSERT(m[reqpage]->dirty == 0, ("vnode_pager_generic_getpages: page %p is dirty", m)); m[reqpage]->valid = VM_PAGE_BITS_ALL; - vm_page_lock_queues(); for (i = 0; i < count; i++) - if (i != reqpage) + if (i != reqpage) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } VM_OBJECT_UNLOCK(object); return (VM_PAGER_OK); } @@ -800,11 +818,14 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) if (vnode_pager_addr(vp, IDX_TO_OFF(m[i]->pindex), &firstaddr, &runpg) != 0) { VM_OBJECT_LOCK(object); - vm_page_lock_queues(); for (; i < count; i++) - if (i != reqpage) + if (i != reqpage) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } VM_OBJECT_UNLOCK(object); return (VM_PAGER_ERROR); } @@ -818,9 +839,11 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) (object->un_pager.vnp.vnp_size >> 32), (uintmax_t)object->un_pager.vnp.vnp_size); } + vm_page_lock(m[i]); vm_page_lock_queues(); vm_page_free(m[i]); vm_page_unlock_queues(); + vm_page_unlock(m[i]); VM_OBJECT_UNLOCK(object); runend = i + 1; first = runend; @@ -829,18 +852,24 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) runend = i + runpg; if (runend <= reqpage) { VM_OBJECT_LOCK(object); - vm_page_lock_queues(); - for (j = i; j < runend; j++) + for (j = i; j < runend; j++) { + vm_page_lock(m[j]); + vm_page_lock_queues(); vm_page_free(m[j]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[j]); + } VM_OBJECT_UNLOCK(object); } else { if (runpg < (count - first)) { VM_OBJECT_LOCK(object); - vm_page_lock_queues(); - for (i = first + runpg; i < count; i++) + for (i = first + runpg; i < count; i++) { + vm_page_lock(m[i]); + vm_page_lock_queues(); vm_page_free(m[i]); - vm_page_unlock_queues(); + vm_page_unlock_queues(); + vm_page_unlock(m[i]); + } VM_OBJECT_UNLOCK(object); count = first + runpg; } @@ -931,13 +960,14 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) relpbuf(bp, &vnode_pbuf_freecnt); VM_OBJECT_LOCK(object); - vm_page_lock_queues(); for (i = 0, tfoff = foff; i < count; i++, tfoff = nextoff) { vm_page_t mt; nextoff = tfoff + PAGE_SIZE; mt = m[i]; + vm_page_lock(mt); + vm_page_lock_queues(); if (nextoff <= object->un_pager.vnp.vnp_size) { /* * Read filled up entire page. @@ -989,8 +1019,9 @@ vnode_pager_generic_getpages(vp, m, bytecount, reqpage) vm_page_free(mt); } } + vm_page_unlock_queues(); + vm_page_unlock(mt); } - vm_page_unlock_queues(); VM_OBJECT_UNLOCK(object); if (error) { printf("vnode_pager_getpages: I/O read error\n"); @@ -1113,10 +1144,12 @@ vnode_pager_generic_putpages(vp, m, bytecount, flags, rtvals) maxsize = object->un_pager.vnp.vnp_size - poffset; ncount = btoc(maxsize); if ((pgoff = (int)maxsize & PAGE_MASK) != 0) { + vm_page_lock(m[ncount - 1]); vm_page_lock_queues(); vm_page_clear_dirty(m[ncount - 1], pgoff, PAGE_SIZE - pgoff); vm_page_unlock_queues(); + vm_page_unlock(m[ncount - 1]); } } else { maxsize = 0; |