diff options
| author | attilio <attilio@FreeBSD.org> | 2013-02-20 10:38:34 +0000 |
|---|---|---|
| committer | attilio <attilio@FreeBSD.org> | 2013-02-20 10:38:34 +0000 |
| commit | 658534ed5a02db4fef5b0630008502474d6c26d6 (patch) | |
| tree | 567755b13eb4e24198cc8098e62ff4a975f8486c | |
| parent | 5dce0c1384f698b3a27a82e72e3cbcf49b325404 (diff) | |
| download | FreeBSD-src-658534ed5a02db4fef5b0630008502474d6c26d6.zip FreeBSD-src-658534ed5a02db4fef5b0630008502474d6c26d6.tar.gz | |
Switch vm_object lock to be a rwlock.
* VM_OBJECT_LOCK and VM_OBJECT_UNLOCK are mapped to write operations
* VM_OBJECT_SLEEP() is introduced as a general purpose primitve to
get a sleep operation using a VM_OBJECT_LOCK() as protection
* The approach must bear with vm_pager.h namespace pollution so many
files require including directly rwlock.h
72 files changed, 237 insertions, 185 deletions
diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index abce826..046c7a1 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -80,6 +80,7 @@ __FBSDID("$FreeBSD$"); #include <sys/pcpu.h> #include <sys/ptrace.h> #include <sys/reboot.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/signalvar.h> #ifdef SMP diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index c6c62ae..662f926 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -3761,7 +3761,7 @@ pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m, mpte; vm_pindex_t diff, psize; - VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m_start->object, RA_WLOCKED); psize = atop(end - start); mpte = NULL; m = m_start; @@ -3943,7 +3943,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_page_t p, pdpg; int pat_mode; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); if ((addr & (NBPDR - 1)) == 0 && (size & (NBPDR - 1)) == 0) { @@ -4557,7 +4557,7 @@ pmap_is_modified(vm_page_t m) * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can have PG_M set. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); @@ -4688,7 +4688,7 @@ pmap_remove_write(vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; @@ -4832,7 +4832,7 @@ pmap_clear_modify(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("pmap_clear_modify: page %p is busy", m)); diff --git a/sys/arm/arm/pmap-v6.c b/sys/arm/arm/pmap-v6.c index f05f120..9d4bc61 100644 --- a/sys/arm/arm/pmap-v6.c +++ b/sys/arm/arm/pmap-v6.c @@ -2212,7 +2212,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } @@ -3428,7 +3428,7 @@ pmap_clear_modify(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("pmap_clear_modify: page %p is busy", m)); @@ -3475,7 +3475,7 @@ pmap_remove_write(vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) != 0 || (m->aflags & PGA_WRITEABLE) != 0) pmap_clearbit(m, PVF_WRITE); diff --git a/sys/arm/arm/pmap.c b/sys/arm/arm/pmap.c index c9cb42c..abc78c0 100644 --- a/sys/arm/arm/pmap.c +++ b/sys/arm/arm/pmap.c @@ -3006,7 +3006,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } @@ -4461,7 +4461,7 @@ pmap_clear_modify(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("pmap_clear_modify: page %p is busy", m)); @@ -4523,7 +4523,7 @@ pmap_remove_write(vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) != 0 || (m->aflags & PGA_WRITEABLE) != 0) pmap_clearbit(m, PVF_WRITE); diff --git a/sys/dev/agp/agp.c b/sys/dev/agp/agp.c index 79b2d54..5284248 100644 --- a/sys/dev/agp/agp.c +++ b/sys/dev/agp/agp.c @@ -41,6 +41,7 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/mutex.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <dev/agp/agppriv.h> #include <dev/agp/agpvar.h> @@ -622,7 +623,7 @@ agp_generic_bind_memory(device_t dev, struct agp_memory *mem, return 0; bad: mtx_unlock(&sc->as_lock); - VM_OBJECT_LOCK_ASSERT(mem->am_obj, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(mem->am_obj, RA_WLOCKED); for (k = 0; k < mem->am_size; k += PAGE_SIZE) { m = vm_page_lookup(mem->am_obj, OFF_TO_IDX(k)); if (k >= i) diff --git a/sys/dev/agp/agp_i810.c b/sys/dev/agp/agp_i810.c index 9ec3992..30c0dca 100644 --- a/sys/dev/agp/agp_i810.c +++ b/sys/dev/agp/agp_i810.c @@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/mutex.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <dev/agp/agppriv.h> #include <dev/agp/agpreg.h> diff --git a/sys/dev/drm/drmP.h b/sys/dev/drm/drmP.h index ffb69b4..bf5feed 100644 --- a/sys/dev/drm/drmP.h +++ b/sys/dev/drm/drmP.h @@ -59,6 +59,7 @@ struct drm_file; #include <sys/fcntl.h> #include <sys/uio.h> #include <sys/filio.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <sys/bus.h> #include <sys/queue.h> diff --git a/sys/dev/drm2/drmP.h b/sys/dev/drm2/drmP.h index 9b721a8..542844e 100644 --- a/sys/dev/drm2/drmP.h +++ b/sys/dev/drm2/drmP.h @@ -58,6 +58,7 @@ struct drm_file; #include <sys/fcntl.h> #include <sys/uio.h> #include <sys/filio.h> +#include <sys/rwlock.h> #include <sys/selinfo.h> #include <sys/sysctl.h> #include <sys/bus.h> diff --git a/sys/dev/drm2/i915/i915_gem.c b/sys/dev/drm2/i915/i915_gem.c index 73c0b53..5ab6f63 100644 --- a/sys/dev/drm2/i915/i915_gem.c +++ b/sys/dev/drm2/i915/i915_gem.c @@ -2488,7 +2488,7 @@ i915_gem_wire_page(vm_object_t object, vm_pindex_t pindex) vm_page_t m; int rv; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); if (m->valid != VM_PAGE_BITS_ALL) { if (vm_pager_has_page(object, pindex, NULL, NULL)) { diff --git a/sys/dev/hwpmc/hwpmc_mod.c b/sys/dev/hwpmc/hwpmc_mod.c index 2f2f05a..c2429a9 100644 --- a/sys/dev/hwpmc/hwpmc_mod.c +++ b/sys/dev/hwpmc/hwpmc_mod.c @@ -50,6 +50,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/queue.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/signalvar.h> #include <sys/smp.h> diff --git a/sys/dev/md/md.c b/sys/dev/md/md.c index b72f294..294fdf6 100644 --- a/sys/dev/md/md.c +++ b/sys/dev/md/md.c @@ -75,6 +75,7 @@ #include <sys/namei.h> #include <sys/proc.h> #include <sys/queue.h> +#include <sys/rwlock.h> #include <sys/sbuf.h> #include <sys/sched.h> #include <sys/sf_buf.h> diff --git a/sys/dev/netmap/netmap.c b/sys/dev/netmap/netmap.c index 35d5303..4fec312 100644 --- a/sys/dev/netmap/netmap.c +++ b/sys/dev/netmap/netmap.c @@ -81,6 +81,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mman.h> /* PROT_EXEC */ #include <sys/poll.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <vm/vm.h> /* vtophys */ #include <vm/pmap.h> /* vtophys */ #include <sys/socket.h> /* sockaddrs */ diff --git a/sys/dev/sound/pcm/dsp.c b/sys/dev/sound/pcm/dsp.c index 2cfc170..e52e46e 100644 --- a/sys/dev/sound/pcm/dsp.c +++ b/sys/dev/sound/pcm/dsp.c @@ -32,6 +32,8 @@ #include <dev/sound/pcm/sound.h> #include <sys/ctype.h> +#include <sys/lock.h> +#include <sys/rwlock.h> #include <sys/sysent.h> #include <vm/vm.h> diff --git a/sys/fs/fuse/fuse_io.c b/sys/fs/fuse/fuse_io.c index e576568..343077e 100644 --- a/sys/fs/fuse/fuse_io.c +++ b/sys/fs/fuse/fuse_io.c @@ -69,6 +69,7 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/sx.h> #include <sys/mutex.h> +#include <sys/rwlock.h> #include <sys/proc.h> #include <sys/mount.h> #include <sys/vnode.h> @@ -87,8 +88,6 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_map.h> #include <vm/vm_page.h> #include <vm/vm_object.h> -#include <vm/vm_pager.h> -#include <vm/vnode_pager.h> #include <vm/vm_object.h> #include "fuse.h" diff --git a/sys/fs/fuse/fuse_vnops.c b/sys/fs/fuse/fuse_vnops.c index cc9733d..b31e3d9 100644 --- a/sys/fs/fuse/fuse_vnops.c +++ b/sys/fs/fuse/fuse_vnops.c @@ -67,7 +67,7 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/queue.h> #include <sys/lock.h> -#include <sys/mutex.h> +#include <sys/rwlock.h> #include <sys/sx.h> #include <sys/proc.h> #include <sys/mount.h> diff --git a/sys/fs/nfsclient/nfs_clbio.c b/sys/fs/nfsclient/nfs_clbio.c index dba3bc9..64627f1 100644 --- a/sys/fs/nfsclient/nfs_clbio.c +++ b/sys/fs/nfsclient/nfs_clbio.c @@ -43,6 +43,7 @@ __FBSDID("$FreeBSD$"); #include <sys/buf.h> #include <sys/kernel.h> #include <sys/mount.h> +#include <sys/rwlock.h> #include <sys/vmmeter.h> #include <sys/vnode.h> diff --git a/sys/fs/procfs/procfs_map.c b/sys/fs/procfs/procfs_map.c index 3f57add..8244b90 100644 --- a/sys/fs/procfs/procfs_map.c +++ b/sys/fs/procfs/procfs_map.c @@ -43,9 +43,9 @@ #include <sys/filedesc.h> #include <sys/malloc.h> #include <sys/mount.h> -#include <sys/mutex.h> #include <sys/proc.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sbuf.h> #ifdef COMPAT_FREEBSD32 #include <sys/sysent.h> diff --git a/sys/fs/tmpfs/tmpfs_subr.c b/sys/fs/tmpfs/tmpfs_subr.c index 47ac2e6..581534a 100644 --- a/sys/fs/tmpfs/tmpfs_subr.c +++ b/sys/fs/tmpfs/tmpfs_subr.c @@ -38,9 +38,11 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/fnv_hash.h> +#include <sys/lock.h> #include <sys/namei.h> #include <sys/priv.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <sys/stat.h> #include <sys/systm.h> #include <sys/sysctl.h> diff --git a/sys/fs/tmpfs/tmpfs_vnops.c b/sys/fs/tmpfs/tmpfs_vnops.c index bddcf24..d62f357 100644 --- a/sys/fs/tmpfs/tmpfs_vnops.c +++ b/sys/fs/tmpfs/tmpfs_vnops.c @@ -39,9 +39,11 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/fcntl.h> #include <sys/lockf.h> +#include <sys/lock.h> #include <sys/namei.h> #include <sys/priv.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/sf_buf.h> #include <sys/stat.h> diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index 744c3cb..7a2e150 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -81,6 +81,7 @@ __FBSDID("$FreeBSD$"); #include <sys/pcpu.h> #include <sys/ptrace.h> #include <sys/reboot.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/signalvar.h> #ifdef SMP diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index 5fee565..ec1db58 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -3712,7 +3712,7 @@ pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m, mpte; vm_pindex_t diff, psize; - VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m_start->object, RA_WLOCKED); psize = atop(end - start); mpte = NULL; m = m_start; @@ -3890,7 +3890,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_page_t p; int pat_mode; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); if (pseflag && @@ -4509,7 +4509,7 @@ pmap_is_modified(vm_page_t m) * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can have PG_M set. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); @@ -4644,7 +4644,7 @@ pmap_remove_write(vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; @@ -4796,7 +4796,7 @@ pmap_clear_modify(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("pmap_clear_modify: page %p is busy", m)); diff --git a/sys/i386/xen/pmap.c b/sys/i386/xen/pmap.c index a8f11a4..a8281be 100644 --- a/sys/i386/xen/pmap.c +++ b/sys/i386/xen/pmap.c @@ -2871,7 +2871,7 @@ pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, multicall_entry_t *mclp = mcl; int error, count = 0; - VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m_start->object, RA_WLOCKED); psize = atop(end - start); mpte = NULL; m = m_start; @@ -3111,7 +3111,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_page_t p; int pat_mode; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); if (pseflag && @@ -3657,7 +3657,7 @@ pmap_is_modified(vm_page_t m) * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can have PG_M set. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return (rv); @@ -3788,7 +3788,7 @@ pmap_remove_write(vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; @@ -3889,7 +3889,7 @@ pmap_clear_modify(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("pmap_clear_modify: page %p is busy", m)); diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 594f8c6..f4bdb3f 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -1802,7 +1802,7 @@ pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m; vm_pindex_t diff, psize; - VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m_start->object, RA_WLOCKED); psize = atop(end - start); m = m_start; rw_wlock(&pvh_global_lock); @@ -1893,7 +1893,7 @@ pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_size_t size) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } @@ -2211,7 +2211,7 @@ pmap_is_modified(vm_page_t m) * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can be dirty. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return (rv); @@ -2295,7 +2295,7 @@ pmap_clear_modify(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("pmap_clear_modify: page %p is busy", m)); @@ -2373,7 +2373,7 @@ pmap_remove_write(vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c index e6f385a..2658411 100644 --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -45,7 +45,6 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/malloc.h> #include <sys/mount.h> -#include <sys/mutex.h> #include <sys/mman.h> #include <sys/namei.h> #include <sys/pioctl.h> @@ -53,6 +52,7 @@ __FBSDID("$FreeBSD$"); #include <sys/procfs.h> #include <sys/racct.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sf_buf.h> #include <sys/smp.h> #include <sys/systm.h> diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c index 965ce31..5f0160f 100644 --- a/sys/kern/kern_exec.c +++ b/sys/kern/kern_exec.c @@ -57,6 +57,7 @@ __FBSDID("$FreeBSD$"); #include <sys/pioctl.h> #include <sys/namei.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/sdt.h> #include <sys/sf_buf.h> diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c index 2a74a59..974eeb7 100644 --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$"); #include <sys/ptrace.h> #include <sys/refcount.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sbuf.h> #include <sys/sysent.h> #include <sys/sched.h> diff --git a/sys/kern/kern_sharedpage.c b/sys/kern/kern_sharedpage.c index 619304a..35bb36a 100644 --- a/sys/kern/kern_sharedpage.c +++ b/sys/kern/kern_sharedpage.c @@ -34,7 +34,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/kernel.h> #include <sys/lock.h> -#include <sys/mutex.h> +#include <sys/rwlock.h> #include <sys/sysent.h> #include <sys/sysctl.h> #include <sys/vdso.h> diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c index fcbae28..b120263 100644 --- a/sys/kern/kern_shutdown.c +++ b/sys/kern/kern_shutdown.c @@ -62,6 +62,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/reboot.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/smp.h> #include <sys/sysctl.h> diff --git a/sys/kern/subr_uio.c b/sys/kern/subr_uio.c index 2b47395..26ea52d 100644 --- a/sys/kern/subr_uio.c +++ b/sys/kern/subr_uio.c @@ -45,9 +45,9 @@ __FBSDID("$FreeBSD$"); #include <sys/limits.h> #include <sys/lock.h> #include <sys/mman.h> -#include <sys/mutex.h> #include <sys/proc.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/sysctl.h> #include <sys/vnode.h> diff --git a/sys/kern/sys_process.c b/sys/kern/sys_process.c index 63de37a..bad48ba 100644 --- a/sys/kern/sys_process.c +++ b/sys/kern/sys_process.c @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/vnode.h> #include <sys/ptrace.h> +#include <sys/rwlock.h> #include <sys/sx.h> #include <sys/malloc.h> #include <sys/signalvar.h> @@ -59,7 +60,6 @@ __FBSDID("$FreeBSD$"); #include <vm/vm_kern.h> #include <vm/vm_object.h> #include <vm/vm_page.h> -#include <vm/vm_pager.h> #include <vm/vm_param.h> #ifdef COMPAT_FREEBSD32 diff --git a/sys/kern/sysv_shm.c b/sys/kern/sysv_shm.c index 5c35514..ba419f3 100644 --- a/sys/kern/sysv_shm.c +++ b/sys/kern/sysv_shm.c @@ -79,6 +79,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mutex.h> #include <sys/racct.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/stat.h> #include <sys/syscall.h> #include <sys/syscallsubr.h> diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c index 7f75bdc..a7dec24 100644 --- a/sys/kern/uipc_shm.c +++ b/sys/kern/uipc_shm.c @@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/refcount.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/stat.h> #include <sys/sysctl.h> #include <sys/sysproto.h> diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c index 665eb6d..0ec26e9 100644 --- a/sys/kern/uipc_syscalls.c +++ b/sys/kern/uipc_syscalls.c @@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mount.h> #include <sys/mbuf.h> #include <sys/protosw.h> +#include <sys/rwlock.h> #include <sys/sf_buf.h> #include <sys/sysent.h> #include <sys/socket.h> diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c index 99b0197..bae419d 100644 --- a/sys/kern/vfs_aio.c +++ b/sys/kern/vfs_aio.c @@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$"); #include <sys/resourcevar.h> #include <sys/signalvar.h> #include <sys/protosw.h> +#include <sys/rwlock.h> #include <sys/sema.h> #include <sys/socket.h> #include <sys/socketvar.h> diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index 6393399..d8ed908 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kthread.h> #include <sys/proc.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <sys/vmmeter.h> #include <sys/vnode.h> @@ -458,7 +459,7 @@ vfs_buf_test_cache(struct buf *bp, vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if (bp->b_flags & B_CACHE) { int base = (foff + off) & PAGE_MASK; if (vm_page_is_valid(m, base, size) == 0) @@ -2533,7 +2534,7 @@ vfs_setdirty_locked_object(struct buf *bp) int i; object = bp->b_bufobj->bo_object; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); /* * We qualify the scan for modified pages on whether the @@ -3566,7 +3567,7 @@ vfs_drain_busy_pages(struct buf *bp) vm_page_t m; int i, last_busied; - VM_OBJECT_LOCK_ASSERT(bp->b_bufobj->bo_object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(bp->b_bufobj->bo_object, RA_WLOCKED); last_busied = 0; for (i = 0; i < bp->b_npages; i++) { m = bp->b_pages[i]; @@ -3719,7 +3720,7 @@ vfs_bio_clrbuf(struct buf *bp) if (bp->b_pages[0] == bogus_page) goto unlock; mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1; - VM_OBJECT_LOCK_ASSERT(bp->b_pages[0]->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(bp->b_pages[0]->object, RA_WLOCKED); if ((bp->b_pages[0]->valid & mask) == mask) goto unlock; if ((bp->b_pages[0]->valid & mask) == 0) { @@ -3738,7 +3739,7 @@ vfs_bio_clrbuf(struct buf *bp) continue; j = ((vm_offset_t)sa & PAGE_MASK) / DEV_BSIZE; mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j; - VM_OBJECT_LOCK_ASSERT(bp->b_pages[i]->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(bp->b_pages[i]->object, RA_WLOCKED); if ((bp->b_pages[i]->valid & mask) == mask) continue; if ((bp->b_pages[i]->valid & mask) == 0) diff --git a/sys/kern/vfs_cluster.c b/sys/kern/vfs_cluster.c index 663b66f..0f22be9 100644 --- a/sys/kern/vfs_cluster.c +++ b/sys/kern/vfs_cluster.c @@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/mount.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/vmmeter.h> #include <vm/vm.h> #include <vm/vm_object.h> @@ -413,7 +414,7 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp) if (toff + tinc > PAGE_SIZE) tinc = PAGE_SIZE - toff; VM_OBJECT_LOCK_ASSERT(tbp->b_pages[j]->object, - MA_OWNED); + RA_WLOCKED); if ((tbp->b_pages[j]->valid & vm_page_bits(toff, tinc)) != 0) break; @@ -489,7 +490,7 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp) */ VM_OBJECT_LOCK(bp->b_bufobj->bo_object); for (j = 0; j < bp->b_npages; j++) { - VM_OBJECT_LOCK_ASSERT(bp->b_pages[j]->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(bp->b_pages[j]->object, RA_WLOCKED); if (bp->b_pages[j]->valid == VM_PAGE_BITS_ALL) bp->b_pages[j] = bogus_page; } diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c index 00d064e..abfc2c8 100644 --- a/sys/kern/vfs_default.c +++ b/sys/kern/vfs_default.c @@ -47,8 +47,8 @@ __FBSDID("$FreeBSD$"); #include <sys/lockf.h> #include <sys/malloc.h> #include <sys/mount.h> -#include <sys/mutex.h> #include <sys/namei.h> +#include <sys/rwlock.h> #include <sys/fcntl.h> #include <sys/unistd.h> #include <sys/vnode.h> diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c index 1c26368..0fac234 100644 --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -67,6 +67,7 @@ __FBSDID("$FreeBSD$"); #include <sys/namei.h> #include <sys/priv.h> #include <sys/reboot.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/sleepqueue.h> #include <sys/smp.h> diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c index bbda70d..be1a70f 100644 --- a/sys/kern/vfs_syscalls.c +++ b/sys/kern/vfs_syscalls.c @@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$"); #include <sys/filio.h> #include <sys/limits.h> #include <sys/linker.h> +#include <sys/rwlock.h> #include <sys/sdt.h> #include <sys/stat.h> #include <sys/sx.h> diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c index 32c0978..5105a48 100644 --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$"); #include <sys/buf.h> #include <sys/filio.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sx.h> #include <sys/sysctl.h> #include <sys/ttycom.h> diff --git a/sys/mips/mips/pmap.c b/sys/mips/mips/pmap.c index 7925b8c..d1eaea7 100644 --- a/sys/mips/mips/pmap.c +++ b/sys/mips/mips/pmap.c @@ -2399,7 +2399,7 @@ pmap_enter_object(pmap_t pmap, vm_offset_t start, vm_offset_t end, vm_page_t m, mpte; vm_pindex_t diff, psize; - VM_OBJECT_LOCK_ASSERT(m_start->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m_start->object, RA_WLOCKED); psize = atop(end - start); mpte = NULL; m = m_start; @@ -2423,7 +2423,7 @@ void pmap_object_init_pt(pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } @@ -2768,7 +2768,7 @@ pmap_remove_write(vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; @@ -2834,7 +2834,7 @@ pmap_is_modified(vm_page_t m) * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can have PTE_D set. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); @@ -2882,7 +2882,7 @@ pmap_clear_modify(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("pmap_clear_modify: page %p is busy", m)); diff --git a/sys/nfsclient/nfs_bio.c b/sys/nfsclient/nfs_bio.c index 0003974..796037a 100644 --- a/sys/nfsclient/nfs_bio.c +++ b/sys/nfsclient/nfs_bio.c @@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mbuf.h> #include <sys/mount.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <sys/vmmeter.h> #include <sys/vnode.h> diff --git a/sys/nfsserver/nfs_serv.c b/sys/nfsserver/nfs_serv.c index bef72af..54e198e 100644 --- a/sys/nfsserver/nfs_serv.c +++ b/sys/nfsserver/nfs_serv.c @@ -87,6 +87,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sysctl.h> #include <sys/bio.h> #include <sys/buf.h> +#include <sys/rwlock.h> #include <vm/vm.h> #include <vm/vm_extern.h> diff --git a/sys/ofed/include/linux/linux_compat.c b/sys/ofed/include/linux/linux_compat.c index fb3783a..329376d 100644 --- a/sys/ofed/include/linux/linux_compat.c +++ b/sys/ofed/include/linux/linux_compat.c @@ -37,6 +37,7 @@ #include <sys/fcntl.h> #include <sys/file.h> #include <sys/filio.h> +#include <sys/rwlock.h> #include <vm/vm.h> #include <vm/pmap.h> diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index b173760..cd0f97c 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -1293,7 +1293,7 @@ moea_is_modified(mmu_t mmu, vm_page_t m) * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can have PTE_CHG set. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); @@ -1333,7 +1333,7 @@ moea_clear_modify(mmu_t mmu, vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("moea_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("moea_clear_modify: page %p is busy", m)); @@ -1368,7 +1368,7 @@ moea_remove_write(mmu_t mmu, vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c index 00dab9b..8660e88 100644 --- a/sys/powerpc/aim/mmu_oea64.c +++ b/sys/powerpc/aim/mmu_oea64.c @@ -1449,7 +1449,7 @@ moea64_is_modified(mmu_t mmu, vm_page_t m) * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can have LPTE_CHG set. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return (FALSE); @@ -1484,7 +1484,7 @@ moea64_clear_modify(mmu_t mmu, vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("moea64_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("moea64_clear_modify: page %p is busy", m)); @@ -1517,7 +1517,7 @@ moea64_remove_write(mmu_t mmu, vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; diff --git a/sys/powerpc/booke/pmap.c b/sys/powerpc/booke/pmap.c index f6e5f9c..435960a 100644 --- a/sys/powerpc/booke/pmap.c +++ b/sys/powerpc/booke/pmap.c @@ -1960,7 +1960,7 @@ mmu_booke_remove_write(mmu_t mmu, vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; @@ -2175,7 +2175,7 @@ mmu_booke_is_modified(mmu_t mmu, vm_page_t m) * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no PTEs can be modified. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return (rv); @@ -2247,7 +2247,7 @@ mmu_booke_clear_modify(mmu_t mmu, vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("mmu_booke_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("mmu_booke_clear_modify: page %p is busy", m)); @@ -2662,7 +2662,7 @@ mmu_booke_object_init_pt(mmu_t mmu, pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("mmu_booke_object_init_pt: non-device object")); } diff --git a/sys/security/mac/mac_process.c b/sys/security/mac/mac_process.c index 8e5e5bc..cb50f29 100644 --- a/sys/security/mac/mac_process.c +++ b/sys/security/mac/mac_process.c @@ -54,9 +54,9 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> #include <sys/lock.h> #include <sys/malloc.h> -#include <sys/mutex.h> #include <sys/mac.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <sys/sbuf.h> #include <sys/sdt.h> #include <sys/systm.h> diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index 08f008c..9c6d6aa 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -1663,7 +1663,7 @@ pmap_object_init_pt(pmap_t pm, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_size_t size) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_DEVICE || object->type == OBJT_SG, ("pmap_object_init_pt: non-device object")); } @@ -2061,7 +2061,7 @@ pmap_is_modified(vm_page_t m) * concurrently set while the object is locked. Thus, if PGA_WRITEABLE * is clear, no TTEs can have TD_W set. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return (rv); @@ -2129,7 +2129,7 @@ pmap_clear_modify(vm_page_t m) KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("pmap_clear_modify: page %p is not managed", m)); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("pmap_clear_modify: page %p is busy", m)); @@ -2184,7 +2184,7 @@ pmap_remove_write(vm_page_t m) * another thread while the object is locked. Thus, if PGA_WRITEABLE * is clear, no page table entries need updating. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && (m->aflags & PGA_WRITEABLE) == 0) return; diff --git a/sys/ufs/ffs/ffs_rawread.c b/sys/ufs/ffs/ffs_rawread.c index f8e3e00..31a01cb 100644 --- a/sys/ufs/ffs/ffs_rawread.c +++ b/sys/ufs/ffs/ffs_rawread.c @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include <sys/ttycom.h> #include <sys/bio.h> #include <sys/buf.h> +#include <sys/rwlock.h> #include <ufs/ufs/extattr.h> #include <ufs/ufs/quota.h> #include <ufs/ufs/inode.h> diff --git a/sys/ufs/ffs/ffs_vnops.c b/sys/ufs/ffs/ffs_vnops.c index 5c99d5b..84bcb1b 100644 --- a/sys/ufs/ffs/ffs_vnops.c +++ b/sys/ufs/ffs/ffs_vnops.c @@ -75,6 +75,7 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/mount.h> #include <sys/priv.h> +#include <sys/rwlock.h> #include <sys/stat.h> #include <sys/vmmeter.h> #include <sys/vnode.h> diff --git a/sys/vm/default_pager.c b/sys/vm/default_pager.c index 12dc823..16ee405 100644 --- a/sys/vm/default_pager.c +++ b/sys/vm/default_pager.c @@ -45,7 +45,7 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/proc.h> #include <sys/resourcevar.h> -#include <sys/mutex.h> +#include <sys/rwlock.h> #include <vm/vm.h> #include <vm/vm_object.h> diff --git a/sys/vm/device_pager.c b/sys/vm/device_pager.c index 809c32c..ae68d15 100644 --- a/sys/vm/device_pager.c +++ b/sys/vm/device_pager.c @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/mutex.h> #include <sys/mman.h> +#include <sys/rwlock.h> #include <sys/sx.h> #include <vm/vm.h> @@ -206,7 +207,7 @@ void cdev_pager_free_page(vm_object_t object, vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if (object->type == OBJT_MGTDEVICE) { KASSERT((m->oflags & VPO_UNMANAGED) == 0, ("unmanaged %p", m)); pmap_remove_all(m); @@ -221,7 +222,7 @@ static void dev_pager_free_page(vm_object_t object, vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT((object->type == OBJT_DEVICE && (m->oflags & VPO_UNMANAGED) != 0), ("Managed device or page obj %p m %p", object, m)); @@ -258,11 +259,11 @@ dev_pager_getpages(vm_object_t object, vm_page_t *ma, int count, int reqpage) { int error, i; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); error = object->un_pager.devp.ops->cdev_pg_fault(object, IDX_TO_OFF(ma[reqpage]->pindex), PROT_READ, &ma[reqpage]); - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); for (i = 0; i < count; i++) { if (i != reqpage) { diff --git a/sys/vm/phys_pager.c b/sys/vm/phys_pager.c index 0ffafea..f81a7d1 100644 --- a/sys/vm/phys_pager.c +++ b/sys/vm/phys_pager.c @@ -34,6 +34,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/mutex.h> #include <sys/mman.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <vm/vm.h> @@ -139,7 +140,7 @@ phys_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) { int i; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); for (i = 0; i < count; i++) { if (m[i]->valid == 0) { if ((m[i]->flags & PG_ZERO) == 0) diff --git a/sys/vm/sg_pager.c b/sys/vm/sg_pager.c index 097039e..b78b032 100644 --- a/sys/vm/sg_pager.c +++ b/sys/vm/sg_pager.c @@ -36,6 +36,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/lock.h> #include <sys/mutex.h> +#include <sys/rwlock.h> #include <sys/sglist.h> #include <vm/vm.h> #include <vm/vm_param.h> @@ -142,7 +143,7 @@ sg_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) size_t space; int i; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); sg = object->handle; memattr = object->memattr; VM_OBJECT_UNLOCK(object); diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index 44bff25..fcfefb5 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -89,6 +89,7 @@ __FBSDID("$FreeBSD$"); #include <sys/racct.h> #include <sys/resource.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <sys/sysproto.h> #include <sys/blist.h> @@ -675,7 +676,7 @@ swap_pager_dealloc(vm_object_t object) mtx_unlock(&sw_alloc_mtx); } - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); vm_object_pip_wait(object, "swpdea"); /* @@ -816,7 +817,7 @@ void swap_pager_freespace(vm_object_t object, vm_pindex_t start, vm_size_t size) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); swp_pager_meta_free(object, start, size); } @@ -884,8 +885,8 @@ swap_pager_copy(vm_object_t srcobject, vm_object_t dstobject, { vm_pindex_t i; - VM_OBJECT_LOCK_ASSERT(srcobject, MA_OWNED); - VM_OBJECT_LOCK_ASSERT(dstobject, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(srcobject, RA_WLOCKED); + VM_OBJECT_LOCK_ASSERT(dstobject, RA_WLOCKED); /* * If destroysource is set, we remove the source object from the @@ -988,7 +989,7 @@ swap_pager_haspage(vm_object_t object, vm_pindex_t pindex, int *before, int *aft { daddr_t blk0; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); /* * do we have good backing store at the requested index ? */ @@ -1059,7 +1060,7 @@ static void swap_pager_unswapped(vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); swp_pager_meta_ctl(m->object, m->pindex, SWM_FREE); } @@ -1213,7 +1214,7 @@ swap_pager_getpages(vm_object_t object, vm_page_t *m, int count, int reqpage) while ((mreq->oflags & VPO_SWAPINPROG) != 0) { mreq->oflags |= VPO_WANTED; PCPU_INC(cnt.v_intrans); - if (msleep(mreq, VM_OBJECT_MTX(object), PSWP, "swread", hz*20)) { + if (VM_OBJECT_SLEEP(mreq, object, PSWP, "swread", hz * 20)) { printf( "swap_pager: indefinite wait buffer: bufobj: %p, blkno: %jd, size: %ld\n", bp->b_bufobj, (intmax_t)bp->b_blkno, bp->b_bcount); @@ -1653,7 +1654,7 @@ swap_pager_isswapped(vm_object_t object, struct swdevt *sp) int bcount; int i; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if (object->type != OBJT_SWAP) return (0); @@ -1809,7 +1810,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk) struct swblock **pswap; int idx; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); /* * Convert default object to swap object if necessary */ @@ -1907,7 +1908,7 @@ static void swp_pager_meta_free(vm_object_t object, vm_pindex_t index, daddr_t count) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if (object->type != OBJT_SWAP) return; @@ -1953,7 +1954,7 @@ swp_pager_meta_free_all(vm_object_t object) { daddr_t index = 0; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if (object->type != OBJT_SWAP) return; @@ -2012,7 +2013,7 @@ swp_pager_meta_ctl(vm_object_t object, vm_pindex_t pindex, int flags) daddr_t r1; int idx; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); /* * The meta data only exists of the object is OBJT_SWAP * and even then might not be allocated yet. diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c index 2d5b555..aeeec39 100644 --- a/sys/vm/uma_core.c +++ b/sys/vm/uma_core.c @@ -72,6 +72,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sysctl.h> #include <sys/mutex.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <sys/sbuf.h> #include <sys/smp.h> #include <sys/vmmeter.h> @@ -3031,7 +3032,7 @@ uma_zone_set_obj(uma_zone_t zone, struct vm_object *obj, int count) if (obj == NULL) obj = vm_object_allocate(OBJT_PHYS, pages); else { - VM_OBJECT_LOCK_INIT(obj, "uma object"); + VM_OBJECT_LOCK_INIT(obj, "uma vm object"); _vm_object_allocate(OBJT_PHYS, pages, obj); } ZONE_LOCK(zone); diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index b79b3f5..9517cd3 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -81,9 +81,9 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/kernel.h> #include <sys/lock.h> -#include <sys/mutex.h> #include <sys/proc.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <sys/vmmeter.h> #include <sys/vnode.h> @@ -960,7 +960,7 @@ vm_fault_cache_behind(const struct faultstate *fs, int distance) vm_pindex_t pindex; object = fs->object; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); first_object = fs->first_object; if (first_object != object) { if (!VM_OBJECT_TRYLOCK(first_object)) { @@ -1403,7 +1403,7 @@ vm_fault_additional_pages(m, rbehind, rahead, marray, reqpage) vm_page_t rtm; int cbehind, cahead; - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); object = m->object; pindex = m->pindex; diff --git a/sys/vm/vm_glue.c b/sys/vm/vm_glue.c index f44f04c..a1f5db5 100644 --- a/sys/vm/vm_glue.c +++ b/sys/vm/vm_glue.c @@ -71,6 +71,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/racct.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/sf_buf.h> #include <sys/shm.h> diff --git a/sys/vm/vm_init.c b/sys/vm/vm_init.c index c507691..c265e30 100644 --- a/sys/vm/vm_init.c +++ b/sys/vm/vm_init.c @@ -68,8 +68,8 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/kernel.h> #include <sys/lock.h> -#include <sys/mutex.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <sys/systm.h> #include <sys/selinfo.h> diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c index ad9aa0d..593a37d 100644 --- a/sys/vm/vm_kern.c +++ b/sys/vm/vm_kern.c @@ -70,9 +70,9 @@ __FBSDID("$FreeBSD$"); #include <sys/kernel.h> /* for ticks and hz */ #include <sys/eventhandler.h> #include <sys/lock.h> -#include <sys/mutex.h> #include <sys/proc.h> #include <sys/malloc.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <vm/vm.h> diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index 26de826..e915079 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -77,6 +77,7 @@ __FBSDID("$FreeBSD$"); #include <sys/vnode.h> #include <sys/racct.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/file.h> #include <sys/sysctl.h> #include <sys/sysent.h> diff --git a/sys/vm/vm_meter.c b/sys/vm/vm_meter.c index 05174e9..2e1d11c 100644 --- a/sys/vm/vm_meter.c +++ b/sys/vm/vm_meter.c @@ -39,6 +39,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mutex.h> #include <sys/proc.h> #include <sys/resource.h> +#include <sys/rwlock.h> #include <sys/sx.h> #include <sys/vmmeter.h> #include <sys/smp.h> diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c index 05bb8ae..af271f1 100644 --- a/sys/vm/vm_mmap.c +++ b/sys/vm/vm_mmap.c @@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$"); #include <sys/racct.h> #include <sys/resource.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <sys/vnode.h> #include <sys/fcntl.h> diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 32b0779..5b9415b 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -78,6 +78,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> /* for curproc, pageproc */ #include <sys/socket.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/vnode.h> #include <sys/vmmeter.h> #include <sys/sx.h> @@ -193,8 +194,8 @@ vm_object_zinit(void *mem, int size, int flags) vm_object_t object; object = (vm_object_t)mem; - bzero(&object->mtx, sizeof(object->mtx)); - VM_OBJECT_LOCK_INIT(object, "standard object"); + bzero(&object->lock, sizeof(object->lock)); + VM_OBJECT_LOCK_INIT(object, "standard vm object"); /* These are true for any object that has been freed */ object->paging_in_progress = 0; @@ -266,7 +267,7 @@ vm_object_init(void) TAILQ_INIT(&vm_object_list); mtx_init(&vm_object_list_mtx, "vm object_list", NULL, MTX_DEF); - VM_OBJECT_LOCK_INIT(kernel_object, "kernel object"); + VM_OBJECT_LOCK_INIT(kernel_object, "kernel vm object"); _vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS), kernel_object); #if VM_NRESERVLEVEL > 0 @@ -274,7 +275,7 @@ vm_object_init(void) kernel_object->pg_color = (u_short)atop(VM_MIN_KERNEL_ADDRESS); #endif - VM_OBJECT_LOCK_INIT(kmem_object, "kmem object"); + VM_OBJECT_LOCK_INIT(kmem_object, "kmem vm object"); _vm_object_allocate(OBJT_PHYS, OFF_TO_IDX(VM_MAX_KERNEL_ADDRESS - VM_MIN_KERNEL_ADDRESS), kmem_object); #if VM_NRESERVLEVEL > 0 @@ -300,7 +301,7 @@ void vm_object_clear_flag(vm_object_t object, u_short bits) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); object->flags &= ~bits; } @@ -317,7 +318,7 @@ int vm_object_set_memattr(vm_object_t object, vm_memattr_t memattr) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); switch (object->type) { case OBJT_DEFAULT: case OBJT_DEVICE: @@ -343,7 +344,7 @@ void vm_object_pip_add(vm_object_t object, short i) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); object->paging_in_progress += i; } @@ -351,7 +352,7 @@ void vm_object_pip_subtract(vm_object_t object, short i) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); object->paging_in_progress -= i; } @@ -359,7 +360,7 @@ void vm_object_pip_wakeup(vm_object_t object) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); object->paging_in_progress--; if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) { vm_object_clear_flag(object, OBJ_PIPWNT); @@ -371,7 +372,7 @@ void vm_object_pip_wakeupn(vm_object_t object, short i) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if (i) object->paging_in_progress -= i; if ((object->flags & OBJ_PIPWNT) && object->paging_in_progress == 0) { @@ -384,10 +385,10 @@ void vm_object_pip_wait(vm_object_t object, char *waitid) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); while (object->paging_in_progress) { object->flags |= OBJ_PIPWNT; - msleep(object, VM_OBJECT_MTX(object), PVM, waitid, 0); + VM_OBJECT_SLEEP(object, object, PVM, waitid, 0); } } @@ -435,7 +436,7 @@ vm_object_reference_locked(vm_object_t object) { struct vnode *vp; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); object->ref_count++; if (object->type == OBJT_VNODE) { vp = object->handle; @@ -451,7 +452,7 @@ vm_object_vndeallocate(vm_object_t object) { struct vnode *vp = (struct vnode *) object->handle; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_VNODE, ("vm_object_vndeallocate: not a vnode object")); KASSERT(vp != NULL, ("vm_object_vndeallocate: missing vp")); @@ -579,9 +580,8 @@ retry: } else if (object->paging_in_progress) { VM_OBJECT_UNLOCK(robject); object->flags |= OBJ_PIPWNT; - msleep(object, - VM_OBJECT_MTX(object), - PDROP | PVM, "objde2", 0); + VM_OBJECT_SLEEP(object, object, + PDROP | PVM, "objde2" , 0); VM_OBJECT_LOCK(robject); temp = robject->backing_object; if (object == temp) { @@ -675,7 +675,7 @@ vm_object_terminate(vm_object_t object) { vm_page_t p, p_next; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); /* * Make sure no one uses us. @@ -816,7 +816,7 @@ vm_object_page_clean(vm_object_t object, vm_ooffset_t start, vm_ooffset_t end, int curgeneration, n, pagerflags; boolean_t clearobjflags, eio, res; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_VNODE, ("Not a vnode object")); if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 || object->resident_page_count == 0) @@ -902,7 +902,7 @@ vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int pagerflags, int count, i, mreq, runlen; vm_page_lock_assert(p, MA_NOTOWNED); - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); count = 1; mreq = 0; @@ -1139,8 +1139,7 @@ shadowlookup: if (object != tobject) VM_OBJECT_UNLOCK(object); m->oflags |= VPO_WANTED; - msleep(m, VM_OBJECT_MTX(tobject), PDROP | PVM, "madvpo", - 0); + VM_OBJECT_SLEEP(m, tobject, PDROP | PVM, "madvpo" , 0); VM_OBJECT_LOCK(object); goto relookup; } @@ -1338,7 +1337,7 @@ retry: if ((m->oflags & VPO_BUSY) || m->busy) { VM_OBJECT_UNLOCK(new_object); m->oflags |= VPO_WANTED; - msleep(m, VM_OBJECT_MTX(orig_object), PVM, "spltwt", 0); + VM_OBJECT_SLEEP(m, orig_object, PVM, "spltwt" , 0); VM_OBJECT_LOCK(new_object); goto retry; } @@ -1405,8 +1404,8 @@ vm_object_backing_scan(vm_object_t object, int op) vm_object_t backing_object; vm_pindex_t backing_offset_index; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); - VM_OBJECT_LOCK_ASSERT(object->backing_object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); + VM_OBJECT_LOCK_ASSERT(object->backing_object, RA_WLOCKED); backing_object = object->backing_object; backing_offset_index = OFF_TO_IDX(object->backing_object_offset); @@ -1496,7 +1495,7 @@ vm_object_backing_scan(vm_object_t object, int op) if ((p->oflags & VPO_BUSY) || p->busy) { VM_OBJECT_UNLOCK(object); p->oflags |= VPO_WANTED; - msleep(p, VM_OBJECT_MTX(backing_object), + VM_OBJECT_SLEEP(p, backing_object, PDROP | PVM, "vmocol", 0); VM_OBJECT_LOCK(object); VM_OBJECT_LOCK(backing_object); @@ -1626,8 +1625,8 @@ vm_object_qcollapse(vm_object_t object) { vm_object_t backing_object = object->backing_object; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); - VM_OBJECT_LOCK_ASSERT(backing_object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); + VM_OBJECT_LOCK_ASSERT(backing_object, RA_WLOCKED); if (backing_object->ref_count != 1) return; @@ -1645,7 +1644,7 @@ vm_object_qcollapse(vm_object_t object) void vm_object_collapse(vm_object_t object) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); while (TRUE) { vm_object_t backing_object; @@ -1853,7 +1852,7 @@ vm_object_page_remove(vm_object_t object, vm_pindex_t start, vm_pindex_t end, vm_page_t p, next; int wirings; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT((object->flags & OBJ_UNMANAGED) == 0 || (options & (OBJPR_CLEANONLY | OBJPR_NOTMAPPED)) == OBJPR_NOTMAPPED, ("vm_object_page_remove: illegal options for object %p", object)); @@ -1948,7 +1947,7 @@ vm_object_page_cache(vm_object_t object, vm_pindex_t start, vm_pindex_t end) struct mtx *mtx, *new_mtx; vm_page_t p, next; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT((object->flags & (OBJ_FICTITIOUS | OBJ_UNMANAGED)) == 0, ("vm_object_page_cache: illegal object %p", object)); if (object->resident_page_count == 0) @@ -1996,7 +1995,7 @@ vm_object_populate(vm_object_t object, vm_pindex_t start, vm_pindex_t end) vm_pindex_t pindex; int rv; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); for (pindex = start; pindex < end; pindex++) { m = vm_page_grab(object, pindex, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); @@ -2147,7 +2146,7 @@ void vm_object_set_writeable_dirty(vm_object_t object) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if (object->type != OBJT_VNODE) return; object->generation++; diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h index 8134752..c01c638 100644 --- a/sys/vm/vm_object.h +++ b/sys/vm/vm_object.h @@ -70,15 +70,16 @@ #include <sys/queue.h> #include <sys/_lock.h> #include <sys/_mutex.h> +#include <sys/_rwlock.h> /* * Types defined: * * vm_object_t Virtual memory object. * - * The root of cached pages pool is protected by both the per-object mutex + * The root of cached pages pool is protected by both the per-object lock * and the free pages queue mutex. - * On insert in the cache splay tree, the per-object mutex is expected + * On insert in the cache splay tree, the per-object lock is expected * to be already held and the free pages queue mutex will be * acquired during the operation too. * On remove and lookup from the cache splay tree, only the free @@ -89,13 +90,13 @@ * * List of locks * (c) const until freed - * (o) per-object mutex + * (o) per-object lock * (f) free pages queue mutex * */ struct vm_object { - struct mtx mtx; + struct rwlock lock; TAILQ_ENTRY(vm_object) object_list; /* list of all objects */ LIST_HEAD(, vm_object) shadow_head; /* objects that this is a shadow for */ LIST_ENTRY(vm_object) shadow_list; /* chain of shadow objects */ @@ -203,16 +204,22 @@ extern struct vm_object kmem_object_store; #define kernel_object (&kernel_object_store) #define kmem_object (&kmem_object_store) -#define VM_OBJECT_LOCK(object) mtx_lock(&(object)->mtx) -#define VM_OBJECT_LOCK_ASSERT(object, type) \ - mtx_assert(&(object)->mtx, (type)) -#define VM_OBJECT_LOCK_INIT(object, type) \ - mtx_init(&(object)->mtx, "vm object", \ - (type), MTX_DEF | MTX_DUPOK) -#define VM_OBJECT_LOCKED(object) mtx_owned(&(object)->mtx) -#define VM_OBJECT_MTX(object) (&(object)->mtx) -#define VM_OBJECT_TRYLOCK(object) mtx_trylock(&(object)->mtx) -#define VM_OBJECT_UNLOCK(object) mtx_unlock(&(object)->mtx) +#define VM_OBJECT_LOCK(object) \ + rw_wlock(&(object)->lock) +#define VM_OBJECT_LOCK_ASSERT(object, type) \ + rw_assert(&(object)->lock, (type)) +#define VM_OBJECT_LOCK_INIT(object, name) \ + rw_init_flags(&(object)->lock, (name), RW_DUPOK) +#define VM_OBJECT_LOCKED(object) \ + rw_wowned(&(object)->lock) +#define VM_OBJECT_LOCKPTR(object) \ + (&(object)->lock) +#define VM_OBJECT_SLEEP(wchan, object, pri, wmesg, timo) \ + rw_sleep((wchan), &(object)->lock, (pri), (wmesg), (timo)) +#define VM_OBJECT_TRYLOCK(object) \ + rw_try_wlock(&(object)->lock) +#define VM_OBJECT_UNLOCK(object) \ + rw_wunlock(&(object)->lock) /* * The object must be locked or thread private. diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index c952c05..2908d11 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -96,6 +96,7 @@ __FBSDID("$FreeBSD$"); #include <sys/msgbuf.h> #include <sys/mutex.h> #include <sys/proc.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <sys/vmmeter.h> #include <sys/vnode.h> @@ -468,7 +469,7 @@ void vm_page_busy(vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("vm_page_busy: page already busy!!!")); m->oflags |= VPO_BUSY; @@ -483,7 +484,7 @@ void vm_page_flash(vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if (m->oflags & VPO_WANTED) { m->oflags &= ~VPO_WANTED; wakeup(m); @@ -501,7 +502,7 @@ void vm_page_wakeup(vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT(m->oflags & VPO_BUSY, ("vm_page_wakeup: page not busy!!!")); m->oflags &= ~VPO_BUSY; vm_page_flash(m); @@ -511,7 +512,7 @@ void vm_page_io_start(vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); m->busy++; } @@ -519,7 +520,7 @@ void vm_page_io_finish(vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT(m->busy > 0, ("vm_page_io_finish: page %p is not busy", m)); m->busy--; if (m->busy == 0) @@ -751,7 +752,7 @@ void vm_page_sleep(vm_page_t m, const char *msg) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if (mtx_owned(vm_page_lockptr(m))) vm_page_unlock(m); @@ -763,7 +764,7 @@ vm_page_sleep(vm_page_t m, const char *msg) * it. */ m->oflags |= VPO_WANTED; - msleep(m, VM_OBJECT_MTX(m->object), PVM, msg, 0); + VM_OBJECT_SLEEP(m, m->object, PVM, msg, 0); } /* @@ -866,7 +867,7 @@ vm_page_insert(vm_page_t m, vm_object_t object, vm_pindex_t pindex) { vm_page_t root; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if (m->object != NULL) panic("vm_page_insert: page already inserted"); @@ -942,7 +943,7 @@ vm_page_remove(vm_page_t m) vm_page_lock_assert(m, MA_OWNED); if ((object = m->object) == NULL) return; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if (m->oflags & VPO_BUSY) { m->oflags &= ~VPO_BUSY; vm_page_flash(m); @@ -1016,7 +1017,7 @@ vm_page_lookup(vm_object_t object, vm_pindex_t pindex) { vm_page_t m; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if ((m = object->root) != NULL && m->pindex != pindex) { m = vm_page_splay(pindex, m); if ((object->root = m)->pindex != pindex) @@ -1038,7 +1039,7 @@ vm_page_find_least(vm_object_t object, vm_pindex_t pindex) { vm_page_t m; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if ((m = TAILQ_FIRST(&object->memq)) != NULL) { if (m->pindex < pindex) { m = vm_page_splay(pindex, object->root); @@ -1060,7 +1061,7 @@ vm_page_next(vm_page_t m) { vm_page_t next; - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((next = TAILQ_NEXT(m, listq)) != NULL && next->pindex != m->pindex + 1) next = NULL; @@ -1078,7 +1079,7 @@ vm_page_prev(vm_page_t m) { vm_page_t prev; - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((prev = TAILQ_PREV(m, pglist, listq)) != NULL && prev->pindex != m->pindex - 1) prev = NULL; @@ -1256,7 +1257,7 @@ vm_page_cache_transfer(vm_object_t orig_object, vm_pindex_t offidxstart, * requires the object to be locked. In contrast, removal does * not. */ - VM_OBJECT_LOCK_ASSERT(new_object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(new_object, RA_WLOCKED); KASSERT(new_object->cache == NULL, ("vm_page_cache_transfer: object %p has cached pages", new_object)); @@ -1326,7 +1327,7 @@ vm_page_is_cached(vm_object_t object, vm_pindex_t pindex) * page queues lock in order to prove that the specified page doesn't * exist. */ - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if (__predict_true(object->cache == NULL)) return (FALSE); mtx_lock(&vm_page_queue_free_mtx); @@ -1375,7 +1376,7 @@ vm_page_alloc(vm_object_t object, vm_pindex_t pindex, int req) KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0), ("vm_page_alloc: inconsistent object/req")); if (object != NULL) - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); req_class = req & VM_ALLOC_CLASS_MASK; @@ -1582,7 +1583,7 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req, KASSERT((object != NULL) == ((req & VM_ALLOC_NOOBJ) == 0), ("vm_page_alloc_contig: inconsistent object/req")); if (object != NULL) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->type == OBJT_PHYS, ("vm_page_alloc_contig: object %p isn't OBJT_PHYS", object)); @@ -1992,7 +1993,7 @@ vm_page_activate(vm_page_t m) int queue; vm_page_lock_assert(m, MA_OWNED); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((queue = m->queue) != PQ_ACTIVE) { if (m->wire_count == 0 && (m->oflags & VPO_UNMANAGED) == 0) { if (m->act_count < ACT_INIT) @@ -2276,7 +2277,7 @@ vm_page_try_to_cache(vm_page_t m) { vm_page_lock_assert(m, MA_OWNED); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if (m->dirty || m->hold_count || m->busy || m->wire_count || (m->oflags & (VPO_BUSY | VPO_UNMANAGED)) != 0) return (0); @@ -2299,7 +2300,7 @@ vm_page_try_to_free(vm_page_t m) vm_page_lock_assert(m, MA_OWNED); if (m->object != NULL) - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if (m->dirty || m->hold_count || m->busy || m->wire_count || (m->oflags & (VPO_BUSY | VPO_UNMANAGED)) != 0) return (0); @@ -2325,7 +2326,7 @@ vm_page_cache(vm_page_t m) vm_page_lock_assert(m, MA_OWNED); object = m->object; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if ((m->oflags & (VPO_UNMANAGED | VPO_BUSY)) || m->busy || m->hold_count || m->wire_count) panic("vm_page_cache: attempting to cache busy page"); @@ -2482,7 +2483,7 @@ vm_page_dontneed(vm_page_t m) int head; vm_page_lock_assert(m, MA_OWNED); - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); dnw = PCPU_GET(dnweight); PCPU_INC(dnweight); @@ -2547,7 +2548,7 @@ vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags) { vm_page_t m; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT((allocflags & VM_ALLOC_RETRY) != 0, ("vm_page_grab: VM_ALLOC_RETRY is required")); retrylookup: @@ -2628,7 +2629,7 @@ vm_page_set_valid_range(vm_page_t m, int base, int size) { int endoff, frag; - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if (size == 0) /* handle degenerate case */ return; @@ -2681,7 +2682,7 @@ vm_page_clear_dirty_mask(vm_page_t m, vm_page_bits_t pagebits) * write mapped, then the page's dirty field cannot possibly be * set by a concurrent pmap operation. */ - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if ((m->oflags & VPO_BUSY) == 0 && !pmap_page_is_write_mapped(m)) m->dirty &= ~pagebits; else { @@ -2735,7 +2736,7 @@ vm_page_set_validclean(vm_page_t m, int base, int size) vm_page_bits_t oldvalid, pagebits; int endoff, frag; - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if (size == 0) /* handle degenerate case */ return; @@ -2825,7 +2826,7 @@ vm_page_set_invalid(vm_page_t m, int base, int size) { vm_page_bits_t bits; - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); KASSERT((m->oflags & VPO_BUSY) == 0, ("vm_page_set_invalid: page %p is busy", m)); bits = vm_page_bits(base, size); @@ -2854,7 +2855,7 @@ vm_page_zero_invalid(vm_page_t m, boolean_t setvalid) int b; int i; - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); /* * Scan the valid bits looking for invalid sections that * must be zerod. Invalid sub-DEV_BSIZE'd areas ( where the @@ -2893,7 +2894,7 @@ vm_page_is_valid(vm_page_t m, int base, int size) { vm_page_bits_t bits; - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); bits = vm_page_bits(base, size); if (m->valid && ((m->valid & bits) == bits)) return 1; @@ -2908,7 +2909,7 @@ void vm_page_test_dirty(vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if (m->dirty != VM_PAGE_BITS_ALL && pmap_is_modified(m)) vm_page_dirty(m); } @@ -2962,7 +2963,7 @@ vm_page_cowfault(vm_page_t m) vm_page_lock_assert(m, MA_OWNED); object = m->object; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(object->paging_in_progress != 0, ("vm_page_cowfault: object %p's paging-in-progress count is zero.", object)); @@ -3055,7 +3056,7 @@ vm_page_object_lock_assert(vm_page_t m) * here. */ if (m->object != NULL && (m->oflags & VPO_BUSY) == 0) - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); } #endif diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index ac593a4..f6fe303 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -92,6 +92,7 @@ __FBSDID("$FreeBSD$"); #include <sys/signalvar.h> #include <sys/vnode.h> #include <sys/vmmeter.h> +#include <sys/rwlock.h> #include <sys/sx.h> #include <sys/sysctl.h> @@ -346,7 +347,7 @@ vm_pageout_clean(vm_page_t m) vm_page_lock_assert(m, MA_OWNED); object = m->object; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); /* * It doesn't cost us anything to pageout OBJT_DEFAULT or OBJT_SWAP @@ -484,7 +485,7 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen, int numpagedout = 0; int i, runlen; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); /* * Initiate I/O. Bump the vm_page_t->busy counter and @@ -713,13 +714,13 @@ vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object, vm_page_t p; int actcount, remove_mode; - VM_OBJECT_LOCK_ASSERT(first_object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(first_object, RA_WLOCKED); if ((first_object->flags & OBJ_FICTITIOUS) != 0) return; for (object = first_object;; object = backing_object) { if (pmap_resident_count(pmap) <= desired) goto unlock_return; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); if ((object->flags & OBJ_UNMANAGED) != 0 || object->paging_in_progress != 0) goto unlock_return; diff --git a/sys/vm/vm_pager.c b/sys/vm/vm_pager.c index 6ed64ea..d2c114f 100644 --- a/sys/vm/vm_pager.c +++ b/sys/vm/vm_pager.c @@ -74,6 +74,7 @@ __FBSDID("$FreeBSD$"); #include <sys/buf.h> #include <sys/ucred.h> #include <sys/malloc.h> +#include <sys/rwlock.h> #include <vm/vm.h> #include <vm/vm_param.h> @@ -248,7 +249,7 @@ vm_pager_deallocate(object) vm_object_t object; { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); (*pagertab[object->type]->pgo_dealloc) (object); } diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h index bb7a5ec..eaf45a07 100644 --- a/sys/vm/vm_pager.h +++ b/sys/vm/vm_pager.h @@ -124,7 +124,7 @@ vm_pager_get_pages( ) { int r; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); r = (*pagertab[object->type]->pgo_getpages)(object, m, count, reqpage); if (r == VM_PAGER_OK && m[reqpage]->valid != VM_PAGE_BITS_ALL) { vm_page_zero_invalid(m[reqpage], TRUE); @@ -141,7 +141,7 @@ vm_pager_put_pages( int *rtvals ) { - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); (*pagertab[object->type]->pgo_putpages) (object, m, count, flags, rtvals); } @@ -165,7 +165,7 @@ vm_pager_has_page( ) { boolean_t ret; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); ret = (*pagertab[object->type]->pgo_haspage) (object, offset, before, after); return (ret); @@ -188,7 +188,7 @@ static __inline void vm_pager_page_unswapped(vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(m->object, RA_WLOCKED); if (pagertab[m->object->type]->pgo_pageunswapped) (*pagertab[m->object->type]->pgo_pageunswapped)(m); } diff --git a/sys/vm/vm_reserv.c b/sys/vm/vm_reserv.c index e5ac1a5..869742e 100644 --- a/sys/vm/vm_reserv.c +++ b/sys/vm/vm_reserv.c @@ -47,6 +47,7 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/mutex.h> #include <sys/queue.h> +#include <sys/rwlock.h> #include <sys/sbuf.h> #include <sys/sysctl.h> #include <sys/systm.h> @@ -311,7 +312,7 @@ vm_reserv_alloc_contig(vm_object_t object, vm_pindex_t pindex, u_long npages, int i, index, n; mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); KASSERT(npages != 0, ("vm_reserv_alloc_contig: npages is 0")); /* @@ -495,7 +496,7 @@ vm_reserv_alloc_page(vm_object_t object, vm_pindex_t pindex) vm_reserv_t rv; mtx_assert(&vm_page_queue_free_mtx, MA_OWNED); - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); /* * Is a reservation fundamentally impossible? @@ -870,7 +871,7 @@ vm_reserv_rename(vm_page_t m, vm_object_t new_object, vm_object_t old_object, { vm_reserv_t rv; - VM_OBJECT_LOCK_ASSERT(new_object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(new_object, RA_WLOCKED); rv = vm_reserv_from_page(m); if (rv->object == old_object) { mtx_lock(&vm_page_queue_free_mtx); diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index a6d78f4..597c6a7 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -63,6 +63,7 @@ __FBSDID("$FreeBSD$"); #include <sys/vmmeter.h> #include <sys/limits.h> #include <sys/conf.h> +#include <sys/rwlock.h> #include <sys/sf_buf.h> #include <machine/atomic.h> @@ -116,7 +117,7 @@ vnode_create_vobject(struct vnode *vp, off_t isize, struct thread *td) } VOP_UNLOCK(vp, 0); vm_object_set_flag(object, OBJ_DISCONNECTWNT); - msleep(object, VM_OBJECT_MTX(object), PDROP | PVM, "vodead", 0); + VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vodead" , 0); vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); } @@ -210,7 +211,7 @@ retry: if ((object->flags & OBJ_DEAD) == 0) break; vm_object_set_flag(object, OBJ_DISCONNECTWNT); - msleep(object, VM_OBJECT_MTX(object), PDROP | PVM, "vadead", 0); + VM_OBJECT_SLEEP(object, object, PDROP | PVM, "vadead" , 0); } if (vp->v_usecount == 0) @@ -259,7 +260,7 @@ vnode_pager_dealloc(object) if (vp == NULL) panic("vnode_pager_dealloc: pager already dealloced"); - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); vm_object_pip_wait(object, "vnpdea"); refs = object->ref_count; @@ -299,7 +300,7 @@ vnode_pager_haspage(object, pindex, before, after) int bsize; int pagesperblock, blocksperpage; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); /* * If no vp or vp is doomed or marked transparent to VM, we do not * have the page. @@ -594,7 +595,7 @@ vnode_pager_input_old(object, m) struct sf_buf *sf; struct vnode *vp; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_LOCK_ASSERT(object, RA_WLOCKED); error = 0; /* |
