diff options
Diffstat (limited to 'sys/kern')
-rw-r--r-- | sys/kern/imgact_elf.c | 10 | ||||
-rw-r--r-- | sys/kern/kern_exec.c | 7 | ||||
-rw-r--r-- | sys/kern/kern_proc.c | 21 | ||||
-rw-r--r-- | sys/kern/kern_sharedpage.c | 6 | ||||
-rw-r--r-- | sys/kern/kern_shutdown.c | 1 | ||||
-rw-r--r-- | sys/kern/subr_uio.c | 6 | ||||
-rw-r--r-- | sys/kern/sys_process.c | 11 | ||||
-rw-r--r-- | sys/kern/sysv_shm.c | 5 | ||||
-rw-r--r-- | sys/kern/uipc_shm.c | 35 | ||||
-rw-r--r-- | sys/kern/uipc_syscalls.c | 21 | ||||
-rw-r--r-- | sys/kern/vfs_aio.c | 5 | ||||
-rw-r--r-- | sys/kern/vfs_bio.c | 57 | ||||
-rw-r--r-- | sys/kern/vfs_cluster.c | 24 | ||||
-rw-r--r-- | sys/kern/vfs_default.c | 6 | ||||
-rw-r--r-- | sys/kern/vfs_subr.c | 21 | ||||
-rw-r--r-- | sys/kern/vfs_syscalls.c | 5 | ||||
-rw-r--r-- | sys/kern/vfs_vnops.c | 5 |
17 files changed, 129 insertions, 117 deletions
diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c index e6f385a..8e16ca0 100644 --- a/sys/kern/imgact_elf.c +++ b/sys/kern/imgact_elf.c @@ -45,7 +45,6 @@ __FBSDID("$FreeBSD$"); #include <sys/lock.h> #include <sys/malloc.h> #include <sys/mount.h> -#include <sys/mutex.h> #include <sys/mman.h> #include <sys/namei.h> #include <sys/pioctl.h> @@ -53,6 +52,7 @@ __FBSDID("$FreeBSD$"); #include <sys/procfs.h> #include <sys/racct.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sf_buf.h> #include <sys/smp.h> #include <sys/systm.h> @@ -1278,15 +1278,15 @@ each_writable_segment(td, func, closure) continue; /* Ignore memory-mapped devices and such things. */ - VM_OBJECT_LOCK(object); + VM_OBJECT_WLOCK(object); while ((backing_object = object->backing_object) != NULL) { - VM_OBJECT_LOCK(backing_object); - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WLOCK(backing_object); + VM_OBJECT_WUNLOCK(object); object = backing_object; } ignore_entry = object->type != OBJT_DEFAULT && object->type != OBJT_SWAP && object->type != OBJT_VNODE; - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); if (ignore_entry) continue; diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c index 7c0d2d6..3890157 100644 --- a/sys/kern/kern_exec.c +++ b/sys/kern/kern_exec.c @@ -57,6 +57,7 @@ __FBSDID("$FreeBSD$"); #include <sys/pioctl.h> #include <sys/namei.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/sdt.h> #include <sys/sf_buf.h> @@ -929,7 +930,7 @@ exec_map_first_page(imgp) object = imgp->vp->v_object; if (object == NULL) return (EACCES); - VM_OBJECT_LOCK(object); + VM_OBJECT_WLOCK(object); #if VM_NRESERVLEVEL > 0 if ((object->flags & OBJ_COLORED) == 0) { object->flags |= OBJ_COLORED; @@ -964,7 +965,7 @@ exec_map_first_page(imgp) vm_page_free(ma[0]); vm_page_unlock(ma[0]); } - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); return (EIO); } } @@ -972,7 +973,7 @@ exec_map_first_page(imgp) vm_page_hold(ma[0]); vm_page_unlock(ma[0]); vm_page_wakeup(ma[0]); - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); imgp->firstpage = sf_buf_alloc(ma[0], 0); imgp->image_header = (char *)sf_buf_kva(imgp->firstpage); diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c index 2a74a59..6c46801 100644 --- a/sys/kern/kern_proc.c +++ b/sys/kern/kern_proc.c @@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$"); #include <sys/ptrace.h> #include <sys/refcount.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sbuf.h> #include <sys/sysent.h> #include <sys/sched.h> @@ -1994,7 +1995,7 @@ sysctl_kern_proc_ovmmap(SYSCTL_HANDLER_ARGS) kve->kve_private_resident = 0; obj = entry->object.vm_object; if (obj != NULL) { - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); if (obj->shadow_count == 1) kve->kve_private_resident = obj->resident_page_count; @@ -2009,9 +2010,9 @@ sysctl_kern_proc_ovmmap(SYSCTL_HANDLER_ARGS) for (lobj = tobj = obj; tobj; tobj = tobj->backing_object) { if (tobj != obj) - VM_OBJECT_LOCK(tobj); + VM_OBJECT_WLOCK(tobj); if (lobj != obj) - VM_OBJECT_UNLOCK(lobj); + VM_OBJECT_WUNLOCK(lobj); lobj = tobj; } @@ -2071,11 +2072,11 @@ sysctl_kern_proc_ovmmap(SYSCTL_HANDLER_ARGS) break; } if (lobj != obj) - VM_OBJECT_UNLOCK(lobj); + VM_OBJECT_WUNLOCK(lobj); kve->kve_ref_count = obj->ref_count; kve->kve_shadow_count = obj->shadow_count; - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); if (vp != NULL) { vn_fullpath(curthread, vp, &fullpath, &freepath); @@ -2161,7 +2162,7 @@ sysctl_kern_proc_vmmap(SYSCTL_HANDLER_ARGS) kve->kve_private_resident = 0; obj = entry->object.vm_object; if (obj != NULL) { - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); if (obj->shadow_count == 1) kve->kve_private_resident = obj->resident_page_count; @@ -2182,9 +2183,9 @@ sysctl_kern_proc_vmmap(SYSCTL_HANDLER_ARGS) for (lobj = tobj = obj; tobj; tobj = tobj->backing_object) { if (tobj != obj) - VM_OBJECT_LOCK(tobj); + VM_OBJECT_WLOCK(tobj); if (lobj != obj) - VM_OBJECT_UNLOCK(lobj); + VM_OBJECT_WUNLOCK(lobj); lobj = tobj; } @@ -2246,11 +2247,11 @@ sysctl_kern_proc_vmmap(SYSCTL_HANDLER_ARGS) break; } if (lobj != obj) - VM_OBJECT_UNLOCK(lobj); + VM_OBJECT_WUNLOCK(lobj); kve->kve_ref_count = obj->ref_count; kve->kve_shadow_count = obj->shadow_count; - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); if (vp != NULL) { vn_fullpath(curthread, vp, &fullpath, &freepath); diff --git a/sys/kern/kern_sharedpage.c b/sys/kern/kern_sharedpage.c index 619304a..20b9038 100644 --- a/sys/kern/kern_sharedpage.c +++ b/sys/kern/kern_sharedpage.c @@ -34,7 +34,7 @@ __FBSDID("$FreeBSD$"); #include <sys/systm.h> #include <sys/kernel.h> #include <sys/lock.h> -#include <sys/mutex.h> +#include <sys/rwlock.h> #include <sys/sysent.h> #include <sys/sysctl.h> #include <sys/vdso.h> @@ -107,11 +107,11 @@ shared_page_init(void *dummy __unused) sx_init(&shared_page_alloc_sx, "shpsx"); shared_page_obj = vm_pager_allocate(OBJT_PHYS, 0, PAGE_SIZE, VM_PROT_DEFAULT, 0, NULL); - VM_OBJECT_LOCK(shared_page_obj); + VM_OBJECT_WLOCK(shared_page_obj); m = vm_page_grab(shared_page_obj, 0, VM_ALLOC_RETRY | VM_ALLOC_NOBUSY | VM_ALLOC_ZERO); m->valid = VM_PAGE_BITS_ALL; - VM_OBJECT_UNLOCK(shared_page_obj); + VM_OBJECT_WUNLOCK(shared_page_obj); addr = kmem_alloc_nofault(kernel_map, PAGE_SIZE); pmap_qenter(addr, &m, 1); shared_page_mapping = (char *)addr; diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c index fcbae28..b120263 100644 --- a/sys/kern/kern_shutdown.c +++ b/sys/kern/kern_shutdown.c @@ -62,6 +62,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/reboot.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/smp.h> #include <sys/sysctl.h> diff --git a/sys/kern/subr_uio.c b/sys/kern/subr_uio.c index 2b47395..1ee265c 100644 --- a/sys/kern/subr_uio.c +++ b/sys/kern/subr_uio.c @@ -45,9 +45,9 @@ __FBSDID("$FreeBSD$"); #include <sys/limits.h> #include <sys/lock.h> #include <sys/mman.h> -#include <sys/mutex.h> #include <sys/proc.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/sysctl.h> #include <sys/vnode.h> @@ -104,7 +104,7 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr) &upindex, &prot, &wired)) != KERN_SUCCESS) { return(EFAULT); } - VM_OBJECT_LOCK(uobject); + VM_OBJECT_WLOCK(uobject); retry: if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) { if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco")) @@ -124,7 +124,7 @@ retry: } vm_page_insert(kern_pg, uobject, upindex); vm_page_dirty(kern_pg); - VM_OBJECT_UNLOCK(uobject); + VM_OBJECT_WUNLOCK(uobject); vm_map_lookup_done(map, entry); return(KERN_SUCCESS); } diff --git a/sys/kern/sys_process.c b/sys/kern/sys_process.c index e70e60e..7dc43c4 100644 --- a/sys/kern/sys_process.c +++ b/sys/kern/sys_process.c @@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/vnode.h> #include <sys/ptrace.h> +#include <sys/rwlock.h> #include <sys/sx.h> #include <sys/malloc.h> #include <sys/signalvar.h> @@ -381,7 +382,7 @@ ptrace_vm_entry(struct thread *td, struct proc *p, struct ptrace_vm_entry *pve) obj = entry->object.vm_object; if (obj != NULL) - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); } while (0); vm_map_unlock_read(map); @@ -394,9 +395,9 @@ ptrace_vm_entry(struct thread *td, struct proc *p, struct ptrace_vm_entry *pve) lobj = obj; for (tobj = obj; tobj != NULL; tobj = tobj->backing_object) { if (tobj != obj) - VM_OBJECT_LOCK(tobj); + VM_OBJECT_WLOCK(tobj); if (lobj != obj) - VM_OBJECT_UNLOCK(lobj); + VM_OBJECT_WUNLOCK(lobj); lobj = tobj; pve->pve_offset += tobj->backing_object_offset; } @@ -404,8 +405,8 @@ ptrace_vm_entry(struct thread *td, struct proc *p, struct ptrace_vm_entry *pve) if (vp != NULL) vref(vp); if (lobj != obj) - VM_OBJECT_UNLOCK(lobj); - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(lobj); + VM_OBJECT_WUNLOCK(obj); if (vp != NULL) { freepath = NULL; diff --git a/sys/kern/sysv_shm.c b/sys/kern/sysv_shm.c index 5c35514..a1c6b34 100644 --- a/sys/kern/sysv_shm.c +++ b/sys/kern/sysv_shm.c @@ -79,6 +79,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mutex.h> #include <sys/racct.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/stat.h> #include <sys/syscall.h> #include <sys/syscallsubr.h> @@ -707,10 +708,10 @@ shmget_allocate_segment(td, uap, mode) #endif return (ENOMEM); } - VM_OBJECT_LOCK(shm_object); + VM_OBJECT_WLOCK(shm_object); vm_object_clear_flag(shm_object, OBJ_ONEMAPPING); vm_object_set_flag(shm_object, OBJ_NOSPLIT); - VM_OBJECT_UNLOCK(shm_object); + VM_OBJECT_WUNLOCK(shm_object); shmseg->object = shm_object; shmseg->u.shm_perm.cuid = shmseg->u.shm_perm.uid = cred->cr_uid; diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c index 0cbb8b3..fc33de5 100644 --- a/sys/kern/uipc_shm.c +++ b/sys/kern/uipc_shm.c @@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/refcount.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/stat.h> #include <sys/sysctl.h> #include <sys/sysproto.h> @@ -253,9 +254,9 @@ shm_dotruncate(struct shmfd *shmfd, off_t length) int base, rv; object = shmfd->shm_object; - VM_OBJECT_LOCK(object); + VM_OBJECT_WLOCK(object); if (length == shmfd->shm_size) { - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); return (0); } nobjsize = OFF_TO_IDX(length + PAGE_MASK); @@ -267,7 +268,7 @@ shm_dotruncate(struct shmfd *shmfd, off_t length) * object is mapped into the kernel. */ if (shmfd->shm_kmappings > 0) { - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); return (EBUSY); } @@ -288,9 +289,9 @@ retry: } else if (vm_pager_has_page(object, idx, NULL, NULL)) { m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL); if (m == NULL) { - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); VM_WAIT; - VM_OBJECT_LOCK(object); + VM_OBJECT_WLOCK(object); goto retry; } else if (m->valid != VM_PAGE_BITS_ALL) { ma[0] = m; @@ -308,7 +309,7 @@ retry: } else { vm_page_free(m); vm_page_unlock(m); - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); return (EIO); } } @@ -338,7 +339,7 @@ retry: /* Attempt to reserve the swap */ delta = ptoa(nobjsize - object->size); if (!swap_reserve_by_cred(delta, object->cred)) { - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); return (ENOMEM); } object->charge += delta; @@ -349,7 +350,7 @@ retry: shmfd->shm_mtime = shmfd->shm_ctime; mtx_unlock(&shm_timestamp_lock); object->size = nobjsize; - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); return (0); } @@ -370,10 +371,10 @@ shm_alloc(struct ucred *ucred, mode_t mode) shmfd->shm_object = vm_pager_allocate(OBJT_DEFAULT, NULL, shmfd->shm_size, VM_PROT_DEFAULT, 0, ucred); KASSERT(shmfd->shm_object != NULL, ("shm_create: vm_pager_allocate")); - VM_OBJECT_LOCK(shmfd->shm_object); + VM_OBJECT_WLOCK(shmfd->shm_object); vm_object_clear_flag(shmfd->shm_object, OBJ_ONEMAPPING); vm_object_set_flag(shmfd->shm_object, OBJ_NOSPLIT); - VM_OBJECT_UNLOCK(shmfd->shm_object); + VM_OBJECT_WUNLOCK(shmfd->shm_object); vfs_timestamp(&shmfd->shm_birthtime); shmfd->shm_atime = shmfd->shm_mtime = shmfd->shm_ctime = shmfd->shm_birthtime; @@ -761,20 +762,20 @@ shm_map(struct file *fp, size_t size, off_t offset, void **memp) return (EINVAL); shmfd = fp->f_data; obj = shmfd->shm_object; - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); /* * XXXRW: This validation is probably insufficient, and subject to * sign errors. It should be fixed. */ if (offset >= shmfd->shm_size || offset + size > round_page(shmfd->shm_size)) { - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); return (EINVAL); } shmfd->shm_kmappings++; vm_object_reference_locked(obj); - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); /* Map the object into the kernel_map and wire it. */ kva = vm_map_min(kernel_map); @@ -796,9 +797,9 @@ shm_map(struct file *fp, size_t size, off_t offset, void **memp) vm_object_deallocate(obj); /* On failure, drop our mapping reference. */ - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); shmfd->shm_kmappings--; - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); return (vm_mmap_to_errno(rv)); } @@ -840,10 +841,10 @@ shm_unmap(struct file *fp, void *mem, size_t size) if (obj != shmfd->shm_object) return (EINVAL); vm_map_remove(map, kva, kva + size); - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); KASSERT(shmfd->shm_kmappings > 0, ("shm_unmap: object not mapped")); shmfd->shm_kmappings--; - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); return (0); } diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c index 847db35..eca171c 100644 --- a/sys/kern/uipc_syscalls.c +++ b/sys/kern/uipc_syscalls.c @@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$"); #include <sys/mount.h> #include <sys/mbuf.h> #include <sys/protosw.h> +#include <sys/rwlock.h> #include <sys/sf_buf.h> #include <sys/sysent.h> #include <sys/socket.h> @@ -1848,12 +1849,12 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap, * reclamation of its vnode does not * immediately destroy it. */ - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); if ((obj->flags & OBJ_DEAD) == 0) { vm_object_reference_locked(obj); - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); } else { - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); obj = NULL; } } @@ -2030,7 +2031,7 @@ retry_space: vm_offset_t pgoff; struct mbuf *m0; - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); /* * Calculate the amount to transfer. * Not to exceed a page, the EOF, @@ -2048,7 +2049,7 @@ retry_space: xfsize = omin(rem, xfsize); xfsize = omin(space - loopbytes, xfsize); if (xfsize <= 0) { - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); done = 1; /* all data sent */ break; } @@ -2069,7 +2070,7 @@ retry_space: * block. */ if (pg->valid && vm_page_is_valid(pg, pgoff, xfsize)) - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); else if (m != NULL) error = EAGAIN; /* send what we already got */ else if (uap->flags & SF_NODISKIO) @@ -2083,7 +2084,7 @@ retry_space: * when the I/O completes. */ vm_page_io_start(pg); - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); /* * Get the page from backing store. @@ -2105,10 +2106,10 @@ retry_space: td->td_ucred, NOCRED, &resid, td); VOP_UNLOCK(vp, 0); after_read: - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); vm_page_io_finish(pg); if (!error) - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); mbstat.sf_iocnt++; } if (error) { @@ -2123,7 +2124,7 @@ retry_space: pg->busy == 0 && !(pg->oflags & VPO_BUSY)) vm_page_free(pg); vm_page_unlock(pg); - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); if (error == EAGAIN) error = 0; /* not a real error */ break; diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c index cba1638..cafe440 100644 --- a/sys/kern/vfs_aio.c +++ b/sys/kern/vfs_aio.c @@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$"); #include <sys/resourcevar.h> #include <sys/signalvar.h> #include <sys/protosw.h> +#include <sys/rwlock.h> #include <sys/sema.h> #include <sys/socket.h> #include <sys/socketvar.h> @@ -841,9 +842,9 @@ aio_fsync_vnode(struct thread *td, struct vnode *vp) goto drop; vn_lock(vp, LK_EXCLUSIVE | LK_RETRY); if (vp->v_object != NULL) { - VM_OBJECT_LOCK(vp->v_object); + VM_OBJECT_WLOCK(vp->v_object); vm_object_page_clean(vp->v_object, 0, 0, 0); - VM_OBJECT_UNLOCK(vp->v_object); + VM_OBJECT_WUNLOCK(vp->v_object); } error = VOP_FSYNC(vp, MNT_WAIT, td); diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c index f9b4dbb..6d110ab 100644 --- a/sys/kern/vfs_bio.c +++ b/sys/kern/vfs_bio.c @@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$"); #include <sys/kthread.h> #include <sys/proc.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sysctl.h> #include <sys/vmmeter.h> #include <sys/vnode.h> @@ -458,7 +459,7 @@ vfs_buf_test_cache(struct buf *bp, vm_page_t m) { - VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED); + VM_OBJECT_ASSERT_WLOCKED(m->object); if (bp->b_flags & B_CACHE) { int base = (foff + off) & PAGE_MASK; if (vm_page_is_valid(m, base, size) == 0) @@ -1388,7 +1389,7 @@ brelse(struct buf *bp) */ resid = bp->b_bufsize; foff = bp->b_offset; - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); for (i = 0; i < bp->b_npages; i++) { int had_bogus = 0; @@ -1436,7 +1437,7 @@ brelse(struct buf *bp) resid -= PAGE_SIZE - (foff & PAGE_MASK); foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; } - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); if (bp->b_flags & (B_INVAL | B_RELBUF)) vfs_vmio_release(bp); @@ -1658,7 +1659,7 @@ vfs_vmio_release(struct buf *bp) vm_page_t m; pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages); - VM_OBJECT_LOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); for (i = 0; i < bp->b_npages; i++) { m = bp->b_pages[i]; bp->b_pages[i] = NULL; @@ -1690,7 +1691,7 @@ vfs_vmio_release(struct buf *bp) } vm_page_unlock(m); } - VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); if (bp->b_bufsize) { bufspacewakeup(); @@ -2467,7 +2468,7 @@ inmem(struct vnode * vp, daddr_t blkno) size = vp->v_mount->mnt_stat.f_iosize; off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize; - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) { m = vm_page_lookup(obj, OFF_TO_IDX(off + toff)); if (!m) @@ -2479,11 +2480,11 @@ inmem(struct vnode * vp, daddr_t blkno) (vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0) goto notinmem; } - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); return 1; notinmem: - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); return (0); } @@ -2513,7 +2514,7 @@ vfs_clean_pages_dirty_buf(struct buf *bp) KASSERT(bp->b_offset != NOOFFSET, ("vfs_clean_pages_dirty_buf: no buffer offset")); - VM_OBJECT_LOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); vfs_drain_busy_pages(bp); vfs_setdirty_locked_object(bp); for (i = 0; i < bp->b_npages; i++) { @@ -2526,7 +2527,7 @@ vfs_clean_pages_dirty_buf(struct buf *bp) /* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */ foff = noff; } - VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); } static void @@ -2536,7 +2537,7 @@ vfs_setdirty_locked_object(struct buf *bp) int i; object = bp->b_bufobj->bo_object; - VM_OBJECT_LOCK_ASSERT(object, MA_OWNED); + VM_OBJECT_ASSERT_WLOCKED(object); /* * We qualify the scan for modified pages on whether the @@ -3042,7 +3043,7 @@ allocbuf(struct buf *bp, int size) (vm_offset_t)bp->b_data) + (desiredpages << PAGE_SHIFT), (bp->b_npages - desiredpages)); - VM_OBJECT_LOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); for (i = desiredpages; i < bp->b_npages; i++) { /* * the page is not freed here -- it @@ -3061,7 +3062,7 @@ allocbuf(struct buf *bp, int size) vm_page_unwire(m, 0); vm_page_unlock(m); } - VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); bp->b_npages = desiredpages; } } else if (size > bp->b_bcount) { @@ -3082,7 +3083,7 @@ allocbuf(struct buf *bp, int size) obj = bp->b_bufobj->bo_object; - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); while (bp->b_npages < desiredpages) { vm_page_t m; @@ -3144,7 +3145,7 @@ allocbuf(struct buf *bp, int size) toff += tinc; tinc = PAGE_SIZE; } - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); /* * Step 3, fixup the KVM pmap. Remember that @@ -3399,7 +3400,7 @@ bufdone_finish(struct buf *bp) bp->b_flags |= B_CACHE; } bogus = 0; - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); for (i = 0; i < bp->b_npages; i++) { int bogusflag = 0; int resid; @@ -3441,7 +3442,7 @@ bufdone_finish(struct buf *bp) iosize -= resid; } vm_object_pip_wakeupn(obj, 0); - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); if (bogus) pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages); @@ -3479,7 +3480,7 @@ vfs_unbusy_pages(struct buf *bp) return; obj = bp->b_bufobj->bo_object; - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); for (i = 0; i < bp->b_npages; i++) { m = bp->b_pages[i]; if (m == bogus_page) { @@ -3494,7 +3495,7 @@ vfs_unbusy_pages(struct buf *bp) vm_page_io_finish(m); } vm_object_pip_wakeupn(obj, 0); - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); } /* @@ -3573,7 +3574,7 @@ vfs_drain_busy_pages(struct buf *bp) vm_page_t m; int i, last_busied; - VM_OBJECT_LOCK_ASSERT(bp->b_bufobj->bo_object, MA_OWNED); + VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object); last_busied = 0; for (i = 0; i < bp->b_npages; i++) { m = bp->b_pages[i]; @@ -3615,7 +3616,7 @@ vfs_busy_pages(struct buf *bp, int clear_modify) foff = bp->b_offset; KASSERT(bp->b_offset != NOOFFSET, ("vfs_busy_pages: no buffer offset")); - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); vfs_drain_busy_pages(bp); if (bp->b_bufsize != 0) vfs_setdirty_locked_object(bp); @@ -3652,7 +3653,7 @@ vfs_busy_pages(struct buf *bp, int clear_modify) } foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK; } - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); if (bogus) pmap_qenter(trunc_page((vm_offset_t)bp->b_data), bp->b_pages, bp->b_npages); @@ -3683,7 +3684,7 @@ vfs_bio_set_valid(struct buf *bp, int base, int size) base += (bp->b_offset & PAGE_MASK); n = PAGE_SIZE - (base & PAGE_MASK); - VM_OBJECT_LOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) { m = bp->b_pages[i]; if (n > size) @@ -3693,7 +3694,7 @@ vfs_bio_set_valid(struct buf *bp, int base, int size) size -= n; n = PAGE_SIZE; } - VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); } /* @@ -3720,13 +3721,13 @@ vfs_bio_clrbuf(struct buf *bp) } bp->b_flags &= ~B_INVAL; bp->b_ioflags &= ~BIO_ERROR; - VM_OBJECT_LOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) && (bp->b_offset & PAGE_MASK) == 0) { if (bp->b_pages[0] == bogus_page) goto unlock; mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1; - VM_OBJECT_LOCK_ASSERT(bp->b_pages[0]->object, MA_OWNED); + VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[0]->object); if ((bp->b_pages[0]->valid & mask) == mask) goto unlock; if ((bp->b_pages[0]->valid & mask) == 0) { @@ -3745,7 +3746,7 @@ vfs_bio_clrbuf(struct buf *bp) continue; j = ((vm_offset_t)sa & PAGE_MASK) / DEV_BSIZE; mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j; - VM_OBJECT_LOCK_ASSERT(bp->b_pages[i]->object, MA_OWNED); + VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[i]->object); if ((bp->b_pages[i]->valid & mask) == mask) continue; if ((bp->b_pages[i]->valid & mask) == 0) @@ -3759,7 +3760,7 @@ vfs_bio_clrbuf(struct buf *bp) bp->b_pages[i]->valid |= mask; } unlock: - VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); bp->b_resid = 0; } diff --git a/sys/kern/vfs_cluster.c b/sys/kern/vfs_cluster.c index 663b66f..70937a2 100644 --- a/sys/kern/vfs_cluster.c +++ b/sys/kern/vfs_cluster.c @@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$"); #include <sys/malloc.h> #include <sys/mount.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/vmmeter.h> #include <vm/vm.h> #include <vm/vm_object.h> @@ -406,21 +407,20 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp) */ off = tbp->b_offset; tsize = size; - VM_OBJECT_LOCK(tbp->b_bufobj->bo_object); + VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); for (j = 0; tsize > 0; j++) { toff = off & PAGE_MASK; tinc = tsize; if (toff + tinc > PAGE_SIZE) tinc = PAGE_SIZE - toff; - VM_OBJECT_LOCK_ASSERT(tbp->b_pages[j]->object, - MA_OWNED); + VM_OBJECT_ASSERT_WLOCKED(tbp->b_pages[j]->object); if ((tbp->b_pages[j]->valid & vm_page_bits(toff, tinc)) != 0) break; off += tinc; tsize -= tinc; } - VM_OBJECT_UNLOCK(tbp->b_bufobj->bo_object); + VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); if (tsize > 0) { bqrelse(tbp); break; @@ -455,7 +455,7 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp) BUF_KERNPROC(tbp); TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head, tbp, b_cluster.cluster_entry); - VM_OBJECT_LOCK(tbp->b_bufobj->bo_object); + VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); for (j = 0; j < tbp->b_npages; j += 1) { vm_page_t m; m = tbp->b_pages[j]; @@ -469,7 +469,7 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp) if (m->valid == VM_PAGE_BITS_ALL) tbp->b_pages[j] = bogus_page; } - VM_OBJECT_UNLOCK(tbp->b_bufobj->bo_object); + VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); /* * Don't inherit tbp->b_bufsize as it may be larger due to * a non-page-aligned size. Instead just aggregate using @@ -487,13 +487,13 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp) * Fully valid pages in the cluster are already good and do not need * to be re-read from disk. Replace the page with bogus_page */ - VM_OBJECT_LOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WLOCK(bp->b_bufobj->bo_object); for (j = 0; j < bp->b_npages; j++) { - VM_OBJECT_LOCK_ASSERT(bp->b_pages[j]->object, MA_OWNED); + VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[j]->object); if (bp->b_pages[j]->valid == VM_PAGE_BITS_ALL) bp->b_pages[j] = bogus_page; } - VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object); + VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object); if (bp->b_bufsize > bp->b_kvasize) panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n", bp->b_bufsize, bp->b_kvasize); @@ -918,12 +918,12 @@ cluster_wbuild(vp, size, start_lbn, len) if (tbp->b_flags & B_VMIO) { vm_page_t m; - VM_OBJECT_LOCK(tbp->b_bufobj->bo_object); + VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object); if (i != 0) { /* if not first buffer */ for (j = 0; j < tbp->b_npages; j += 1) { m = tbp->b_pages[j]; if (m->oflags & VPO_BUSY) { - VM_OBJECT_UNLOCK( + VM_OBJECT_WUNLOCK( tbp->b_object); bqrelse(tbp); goto finishcluster; @@ -940,7 +940,7 @@ cluster_wbuild(vp, size, start_lbn, len) bp->b_npages++; } } - VM_OBJECT_UNLOCK(tbp->b_bufobj->bo_object); + VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object); } bp->b_bcount += size; bp->b_bufsize += size; diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c index 1dd0185..ed071b0 100644 --- a/sys/kern/vfs_default.c +++ b/sys/kern/vfs_default.c @@ -47,8 +47,8 @@ __FBSDID("$FreeBSD$"); #include <sys/lockf.h> #include <sys/malloc.h> #include <sys/mount.h> -#include <sys/mutex.h> #include <sys/namei.h> +#include <sys/rwlock.h> #include <sys/fcntl.h> #include <sys/unistd.h> #include <sys/vnode.h> @@ -1047,10 +1047,10 @@ vop_stdadvise(struct vop_advise_args *ap) if (vp->v_object != NULL) { start = trunc_page(ap->a_start); end = round_page(ap->a_end); - VM_OBJECT_LOCK(vp->v_object); + VM_OBJECT_WLOCK(vp->v_object); vm_object_page_cache(vp->v_object, OFF_TO_IDX(start), OFF_TO_IDX(end)); - VM_OBJECT_UNLOCK(vp->v_object); + VM_OBJECT_WUNLOCK(vp->v_object); } VOP_UNLOCK(vp, 0); break; diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c index 1c26368..d0c6bb8 100644 --- a/sys/kern/vfs_subr.c +++ b/sys/kern/vfs_subr.c @@ -67,6 +67,7 @@ __FBSDID("$FreeBSD$"); #include <sys/namei.h> #include <sys/priv.h> #include <sys/reboot.h> +#include <sys/rwlock.h> #include <sys/sched.h> #include <sys/sleepqueue.h> #include <sys/smp.h> @@ -1244,9 +1245,9 @@ bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo) bufobj_wwait(bo, 0, 0); BO_UNLOCK(bo); if (bo->bo_object != NULL) { - VM_OBJECT_LOCK(bo->bo_object); + VM_OBJECT_WLOCK(bo->bo_object); vm_object_pip_wait(bo->bo_object, "bovlbx"); - VM_OBJECT_UNLOCK(bo->bo_object); + VM_OBJECT_WUNLOCK(bo->bo_object); } BO_LOCK(bo); } while (bo->bo_numoutput > 0); @@ -1257,10 +1258,10 @@ bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo) */ if (bo->bo_object != NULL && (flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0) { - VM_OBJECT_LOCK(bo->bo_object); + VM_OBJECT_WLOCK(bo->bo_object); vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ? OBJPR_CLEANONLY : 0); - VM_OBJECT_UNLOCK(bo->bo_object); + VM_OBJECT_WUNLOCK(bo->bo_object); } #ifdef INVARIANTS @@ -2520,9 +2521,9 @@ vinactive(struct vnode *vp, struct thread *td) */ obj = vp->v_object; if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0) { - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); vm_object_page_clean(obj, 0, 0, OBJPC_NOSYNC); - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); } VOP_INACTIVE(vp, td); VI_LOCK(vp); @@ -2603,9 +2604,9 @@ loop: */ if (flags & WRITECLOSE) { if (vp->v_object != NULL) { - VM_OBJECT_LOCK(vp->v_object); + VM_OBJECT_WLOCK(vp->v_object); vm_object_page_clean(vp->v_object, 0, 0, 0); - VM_OBJECT_UNLOCK(vp->v_object); + VM_OBJECT_WUNLOCK(vp->v_object); } error = VOP_FSYNC(vp, MNT_WAIT, td); if (error != 0) { @@ -3503,11 +3504,11 @@ vfs_msync(struct mount *mp, int flags) obj = vp->v_object; if (obj != NULL) { - VM_OBJECT_LOCK(obj); + VM_OBJECT_WLOCK(obj); vm_object_page_clean(obj, 0, 0, flags == MNT_WAIT ? OBJPC_SYNC : OBJPC_NOSYNC); - VM_OBJECT_UNLOCK(obj); + VM_OBJECT_WUNLOCK(obj); } vput(vp); } diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c index 4c1d97c..fc78235 100644 --- a/sys/kern/vfs_syscalls.c +++ b/sys/kern/vfs_syscalls.c @@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$"); #include <sys/filio.h> #include <sys/limits.h> #include <sys/linker.h> +#include <sys/rwlock.h> #include <sys/sdt.h> #include <sys/stat.h> #include <sys/sx.h> @@ -3437,9 +3438,9 @@ sys_fsync(td, uap) vn_lock(vp, lock_flags | LK_RETRY); AUDIT_ARG_VNODE1(vp); if (vp->v_object != NULL) { - VM_OBJECT_LOCK(vp->v_object); + VM_OBJECT_WLOCK(vp->v_object); vm_object_page_clean(vp->v_object, 0, 0, 0); - VM_OBJECT_UNLOCK(vp->v_object); + VM_OBJECT_WUNLOCK(vp->v_object); } error = VOP_FSYNC(vp, MNT_WAIT, td); diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c index 96ce9e2..d367340 100644 --- a/sys/kern/vfs_vnops.c +++ b/sys/kern/vfs_vnops.c @@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$"); #include <sys/buf.h> #include <sys/filio.h> #include <sys/resourcevar.h> +#include <sys/rwlock.h> #include <sys/sx.h> #include <sys/sysctl.h> #include <sys/ttycom.h> @@ -1892,9 +1893,9 @@ vn_pages_remove(struct vnode *vp, vm_pindex_t start, vm_pindex_t end) if ((object = vp->v_object) == NULL) return; - VM_OBJECT_LOCK(object); + VM_OBJECT_WLOCK(object); vm_object_page_remove(object, start, end, 0); - VM_OBJECT_UNLOCK(object); + VM_OBJECT_WUNLOCK(object); } int |