summaryrefslogtreecommitdiffstats
path: root/sys/kern
diff options
context:
space:
mode:
Diffstat (limited to 'sys/kern')
-rw-r--r--sys/kern/imgact_elf.c10
-rw-r--r--sys/kern/kern_exec.c7
-rw-r--r--sys/kern/kern_proc.c21
-rw-r--r--sys/kern/kern_sharedpage.c6
-rw-r--r--sys/kern/kern_shutdown.c1
-rw-r--r--sys/kern/subr_uio.c6
-rw-r--r--sys/kern/sys_process.c11
-rw-r--r--sys/kern/sysv_shm.c5
-rw-r--r--sys/kern/uipc_shm.c35
-rw-r--r--sys/kern/uipc_syscalls.c21
-rw-r--r--sys/kern/vfs_aio.c5
-rw-r--r--sys/kern/vfs_bio.c57
-rw-r--r--sys/kern/vfs_cluster.c24
-rw-r--r--sys/kern/vfs_default.c6
-rw-r--r--sys/kern/vfs_subr.c21
-rw-r--r--sys/kern/vfs_syscalls.c5
-rw-r--r--sys/kern/vfs_vnops.c5
17 files changed, 129 insertions, 117 deletions
diff --git a/sys/kern/imgact_elf.c b/sys/kern/imgact_elf.c
index e6f385a..8e16ca0 100644
--- a/sys/kern/imgact_elf.c
+++ b/sys/kern/imgact_elf.c
@@ -45,7 +45,6 @@ __FBSDID("$FreeBSD$");
#include <sys/lock.h>
#include <sys/malloc.h>
#include <sys/mount.h>
-#include <sys/mutex.h>
#include <sys/mman.h>
#include <sys/namei.h>
#include <sys/pioctl.h>
@@ -53,6 +52,7 @@ __FBSDID("$FreeBSD$");
#include <sys/procfs.h>
#include <sys/racct.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/sf_buf.h>
#include <sys/smp.h>
#include <sys/systm.h>
@@ -1278,15 +1278,15 @@ each_writable_segment(td, func, closure)
continue;
/* Ignore memory-mapped devices and such things. */
- VM_OBJECT_LOCK(object);
+ VM_OBJECT_WLOCK(object);
while ((backing_object = object->backing_object) != NULL) {
- VM_OBJECT_LOCK(backing_object);
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WLOCK(backing_object);
+ VM_OBJECT_WUNLOCK(object);
object = backing_object;
}
ignore_entry = object->type != OBJT_DEFAULT &&
object->type != OBJT_SWAP && object->type != OBJT_VNODE;
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
if (ignore_entry)
continue;
diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c
index 7c0d2d6..3890157 100644
--- a/sys/kern/kern_exec.c
+++ b/sys/kern/kern_exec.c
@@ -57,6 +57,7 @@ __FBSDID("$FreeBSD$");
#include <sys/pioctl.h>
#include <sys/namei.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/sched.h>
#include <sys/sdt.h>
#include <sys/sf_buf.h>
@@ -929,7 +930,7 @@ exec_map_first_page(imgp)
object = imgp->vp->v_object;
if (object == NULL)
return (EACCES);
- VM_OBJECT_LOCK(object);
+ VM_OBJECT_WLOCK(object);
#if VM_NRESERVLEVEL > 0
if ((object->flags & OBJ_COLORED) == 0) {
object->flags |= OBJ_COLORED;
@@ -964,7 +965,7 @@ exec_map_first_page(imgp)
vm_page_free(ma[0]);
vm_page_unlock(ma[0]);
}
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
return (EIO);
}
}
@@ -972,7 +973,7 @@ exec_map_first_page(imgp)
vm_page_hold(ma[0]);
vm_page_unlock(ma[0]);
vm_page_wakeup(ma[0]);
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
imgp->firstpage = sf_buf_alloc(ma[0], 0);
imgp->image_header = (char *)sf_buf_kva(imgp->firstpage);
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index 2a74a59..6c46801 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
#include <sys/ptrace.h>
#include <sys/refcount.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/sbuf.h>
#include <sys/sysent.h>
#include <sys/sched.h>
@@ -1994,7 +1995,7 @@ sysctl_kern_proc_ovmmap(SYSCTL_HANDLER_ARGS)
kve->kve_private_resident = 0;
obj = entry->object.vm_object;
if (obj != NULL) {
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
if (obj->shadow_count == 1)
kve->kve_private_resident =
obj->resident_page_count;
@@ -2009,9 +2010,9 @@ sysctl_kern_proc_ovmmap(SYSCTL_HANDLER_ARGS)
for (lobj = tobj = obj; tobj; tobj = tobj->backing_object) {
if (tobj != obj)
- VM_OBJECT_LOCK(tobj);
+ VM_OBJECT_WLOCK(tobj);
if (lobj != obj)
- VM_OBJECT_UNLOCK(lobj);
+ VM_OBJECT_WUNLOCK(lobj);
lobj = tobj;
}
@@ -2071,11 +2072,11 @@ sysctl_kern_proc_ovmmap(SYSCTL_HANDLER_ARGS)
break;
}
if (lobj != obj)
- VM_OBJECT_UNLOCK(lobj);
+ VM_OBJECT_WUNLOCK(lobj);
kve->kve_ref_count = obj->ref_count;
kve->kve_shadow_count = obj->shadow_count;
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
if (vp != NULL) {
vn_fullpath(curthread, vp, &fullpath,
&freepath);
@@ -2161,7 +2162,7 @@ sysctl_kern_proc_vmmap(SYSCTL_HANDLER_ARGS)
kve->kve_private_resident = 0;
obj = entry->object.vm_object;
if (obj != NULL) {
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
if (obj->shadow_count == 1)
kve->kve_private_resident =
obj->resident_page_count;
@@ -2182,9 +2183,9 @@ sysctl_kern_proc_vmmap(SYSCTL_HANDLER_ARGS)
for (lobj = tobj = obj; tobj; tobj = tobj->backing_object) {
if (tobj != obj)
- VM_OBJECT_LOCK(tobj);
+ VM_OBJECT_WLOCK(tobj);
if (lobj != obj)
- VM_OBJECT_UNLOCK(lobj);
+ VM_OBJECT_WUNLOCK(lobj);
lobj = tobj;
}
@@ -2246,11 +2247,11 @@ sysctl_kern_proc_vmmap(SYSCTL_HANDLER_ARGS)
break;
}
if (lobj != obj)
- VM_OBJECT_UNLOCK(lobj);
+ VM_OBJECT_WUNLOCK(lobj);
kve->kve_ref_count = obj->ref_count;
kve->kve_shadow_count = obj->shadow_count;
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
if (vp != NULL) {
vn_fullpath(curthread, vp, &fullpath,
&freepath);
diff --git a/sys/kern/kern_sharedpage.c b/sys/kern/kern_sharedpage.c
index 619304a..20b9038 100644
--- a/sys/kern/kern_sharedpage.c
+++ b/sys/kern/kern_sharedpage.c
@@ -34,7 +34,7 @@ __FBSDID("$FreeBSD$");
#include <sys/systm.h>
#include <sys/kernel.h>
#include <sys/lock.h>
-#include <sys/mutex.h>
+#include <sys/rwlock.h>
#include <sys/sysent.h>
#include <sys/sysctl.h>
#include <sys/vdso.h>
@@ -107,11 +107,11 @@ shared_page_init(void *dummy __unused)
sx_init(&shared_page_alloc_sx, "shpsx");
shared_page_obj = vm_pager_allocate(OBJT_PHYS, 0, PAGE_SIZE,
VM_PROT_DEFAULT, 0, NULL);
- VM_OBJECT_LOCK(shared_page_obj);
+ VM_OBJECT_WLOCK(shared_page_obj);
m = vm_page_grab(shared_page_obj, 0, VM_ALLOC_RETRY | VM_ALLOC_NOBUSY |
VM_ALLOC_ZERO);
m->valid = VM_PAGE_BITS_ALL;
- VM_OBJECT_UNLOCK(shared_page_obj);
+ VM_OBJECT_WUNLOCK(shared_page_obj);
addr = kmem_alloc_nofault(kernel_map, PAGE_SIZE);
pmap_qenter(addr, &m, 1);
shared_page_mapping = (char *)addr;
diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c
index fcbae28..b120263 100644
--- a/sys/kern/kern_shutdown.c
+++ b/sys/kern/kern_shutdown.c
@@ -62,6 +62,7 @@ __FBSDID("$FreeBSD$");
#include <sys/proc.h>
#include <sys/reboot.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <sys/sysctl.h>
diff --git a/sys/kern/subr_uio.c b/sys/kern/subr_uio.c
index 2b47395..1ee265c 100644
--- a/sys/kern/subr_uio.c
+++ b/sys/kern/subr_uio.c
@@ -45,9 +45,9 @@ __FBSDID("$FreeBSD$");
#include <sys/limits.h>
#include <sys/lock.h>
#include <sys/mman.h>
-#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/sched.h>
#include <sys/sysctl.h>
#include <sys/vnode.h>
@@ -104,7 +104,7 @@ vm_pgmoveco(vm_map_t mapa, vm_offset_t kaddr, vm_offset_t uaddr)
&upindex, &prot, &wired)) != KERN_SUCCESS) {
return(EFAULT);
}
- VM_OBJECT_LOCK(uobject);
+ VM_OBJECT_WLOCK(uobject);
retry:
if ((user_pg = vm_page_lookup(uobject, upindex)) != NULL) {
if (vm_page_sleep_if_busy(user_pg, TRUE, "vm_pgmoveco"))
@@ -124,7 +124,7 @@ retry:
}
vm_page_insert(kern_pg, uobject, upindex);
vm_page_dirty(kern_pg);
- VM_OBJECT_UNLOCK(uobject);
+ VM_OBJECT_WUNLOCK(uobject);
vm_map_lookup_done(map, entry);
return(KERN_SUCCESS);
}
diff --git a/sys/kern/sys_process.c b/sys/kern/sys_process.c
index e70e60e..7dc43c4 100644
--- a/sys/kern/sys_process.c
+++ b/sys/kern/sys_process.c
@@ -44,6 +44,7 @@ __FBSDID("$FreeBSD$");
#include <sys/proc.h>
#include <sys/vnode.h>
#include <sys/ptrace.h>
+#include <sys/rwlock.h>
#include <sys/sx.h>
#include <sys/malloc.h>
#include <sys/signalvar.h>
@@ -381,7 +382,7 @@ ptrace_vm_entry(struct thread *td, struct proc *p, struct ptrace_vm_entry *pve)
obj = entry->object.vm_object;
if (obj != NULL)
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
} while (0);
vm_map_unlock_read(map);
@@ -394,9 +395,9 @@ ptrace_vm_entry(struct thread *td, struct proc *p, struct ptrace_vm_entry *pve)
lobj = obj;
for (tobj = obj; tobj != NULL; tobj = tobj->backing_object) {
if (tobj != obj)
- VM_OBJECT_LOCK(tobj);
+ VM_OBJECT_WLOCK(tobj);
if (lobj != obj)
- VM_OBJECT_UNLOCK(lobj);
+ VM_OBJECT_WUNLOCK(lobj);
lobj = tobj;
pve->pve_offset += tobj->backing_object_offset;
}
@@ -404,8 +405,8 @@ ptrace_vm_entry(struct thread *td, struct proc *p, struct ptrace_vm_entry *pve)
if (vp != NULL)
vref(vp);
if (lobj != obj)
- VM_OBJECT_UNLOCK(lobj);
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(lobj);
+ VM_OBJECT_WUNLOCK(obj);
if (vp != NULL) {
freepath = NULL;
diff --git a/sys/kern/sysv_shm.c b/sys/kern/sysv_shm.c
index 5c35514..a1c6b34 100644
--- a/sys/kern/sysv_shm.c
+++ b/sys/kern/sysv_shm.c
@@ -79,6 +79,7 @@ __FBSDID("$FreeBSD$");
#include <sys/mutex.h>
#include <sys/racct.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/stat.h>
#include <sys/syscall.h>
#include <sys/syscallsubr.h>
@@ -707,10 +708,10 @@ shmget_allocate_segment(td, uap, mode)
#endif
return (ENOMEM);
}
- VM_OBJECT_LOCK(shm_object);
+ VM_OBJECT_WLOCK(shm_object);
vm_object_clear_flag(shm_object, OBJ_ONEMAPPING);
vm_object_set_flag(shm_object, OBJ_NOSPLIT);
- VM_OBJECT_UNLOCK(shm_object);
+ VM_OBJECT_WUNLOCK(shm_object);
shmseg->object = shm_object;
shmseg->u.shm_perm.cuid = shmseg->u.shm_perm.uid = cred->cr_uid;
diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c
index 0cbb8b3..fc33de5 100644
--- a/sys/kern/uipc_shm.c
+++ b/sys/kern/uipc_shm.c
@@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$");
#include <sys/proc.h>
#include <sys/refcount.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/stat.h>
#include <sys/sysctl.h>
#include <sys/sysproto.h>
@@ -253,9 +254,9 @@ shm_dotruncate(struct shmfd *shmfd, off_t length)
int base, rv;
object = shmfd->shm_object;
- VM_OBJECT_LOCK(object);
+ VM_OBJECT_WLOCK(object);
if (length == shmfd->shm_size) {
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
return (0);
}
nobjsize = OFF_TO_IDX(length + PAGE_MASK);
@@ -267,7 +268,7 @@ shm_dotruncate(struct shmfd *shmfd, off_t length)
* object is mapped into the kernel.
*/
if (shmfd->shm_kmappings > 0) {
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
return (EBUSY);
}
@@ -288,9 +289,9 @@ retry:
} else if (vm_pager_has_page(object, idx, NULL, NULL)) {
m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL);
if (m == NULL) {
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
VM_WAIT;
- VM_OBJECT_LOCK(object);
+ VM_OBJECT_WLOCK(object);
goto retry;
} else if (m->valid != VM_PAGE_BITS_ALL) {
ma[0] = m;
@@ -308,7 +309,7 @@ retry:
} else {
vm_page_free(m);
vm_page_unlock(m);
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
return (EIO);
}
}
@@ -338,7 +339,7 @@ retry:
/* Attempt to reserve the swap */
delta = ptoa(nobjsize - object->size);
if (!swap_reserve_by_cred(delta, object->cred)) {
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
return (ENOMEM);
}
object->charge += delta;
@@ -349,7 +350,7 @@ retry:
shmfd->shm_mtime = shmfd->shm_ctime;
mtx_unlock(&shm_timestamp_lock);
object->size = nobjsize;
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
return (0);
}
@@ -370,10 +371,10 @@ shm_alloc(struct ucred *ucred, mode_t mode)
shmfd->shm_object = vm_pager_allocate(OBJT_DEFAULT, NULL,
shmfd->shm_size, VM_PROT_DEFAULT, 0, ucred);
KASSERT(shmfd->shm_object != NULL, ("shm_create: vm_pager_allocate"));
- VM_OBJECT_LOCK(shmfd->shm_object);
+ VM_OBJECT_WLOCK(shmfd->shm_object);
vm_object_clear_flag(shmfd->shm_object, OBJ_ONEMAPPING);
vm_object_set_flag(shmfd->shm_object, OBJ_NOSPLIT);
- VM_OBJECT_UNLOCK(shmfd->shm_object);
+ VM_OBJECT_WUNLOCK(shmfd->shm_object);
vfs_timestamp(&shmfd->shm_birthtime);
shmfd->shm_atime = shmfd->shm_mtime = shmfd->shm_ctime =
shmfd->shm_birthtime;
@@ -761,20 +762,20 @@ shm_map(struct file *fp, size_t size, off_t offset, void **memp)
return (EINVAL);
shmfd = fp->f_data;
obj = shmfd->shm_object;
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
/*
* XXXRW: This validation is probably insufficient, and subject to
* sign errors. It should be fixed.
*/
if (offset >= shmfd->shm_size ||
offset + size > round_page(shmfd->shm_size)) {
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
return (EINVAL);
}
shmfd->shm_kmappings++;
vm_object_reference_locked(obj);
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
/* Map the object into the kernel_map and wire it. */
kva = vm_map_min(kernel_map);
@@ -796,9 +797,9 @@ shm_map(struct file *fp, size_t size, off_t offset, void **memp)
vm_object_deallocate(obj);
/* On failure, drop our mapping reference. */
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
shmfd->shm_kmappings--;
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
return (vm_mmap_to_errno(rv));
}
@@ -840,10 +841,10 @@ shm_unmap(struct file *fp, void *mem, size_t size)
if (obj != shmfd->shm_object)
return (EINVAL);
vm_map_remove(map, kva, kva + size);
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
KASSERT(shmfd->shm_kmappings > 0, ("shm_unmap: object not mapped"));
shmfd->shm_kmappings--;
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
return (0);
}
diff --git a/sys/kern/uipc_syscalls.c b/sys/kern/uipc_syscalls.c
index 847db35..eca171c 100644
--- a/sys/kern/uipc_syscalls.c
+++ b/sys/kern/uipc_syscalls.c
@@ -60,6 +60,7 @@ __FBSDID("$FreeBSD$");
#include <sys/mount.h>
#include <sys/mbuf.h>
#include <sys/protosw.h>
+#include <sys/rwlock.h>
#include <sys/sf_buf.h>
#include <sys/sysent.h>
#include <sys/socket.h>
@@ -1848,12 +1849,12 @@ kern_sendfile(struct thread *td, struct sendfile_args *uap,
* reclamation of its vnode does not
* immediately destroy it.
*/
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
if ((obj->flags & OBJ_DEAD) == 0) {
vm_object_reference_locked(obj);
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
} else {
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
obj = NULL;
}
}
@@ -2030,7 +2031,7 @@ retry_space:
vm_offset_t pgoff;
struct mbuf *m0;
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
/*
* Calculate the amount to transfer.
* Not to exceed a page, the EOF,
@@ -2048,7 +2049,7 @@ retry_space:
xfsize = omin(rem, xfsize);
xfsize = omin(space - loopbytes, xfsize);
if (xfsize <= 0) {
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
done = 1; /* all data sent */
break;
}
@@ -2069,7 +2070,7 @@ retry_space:
* block.
*/
if (pg->valid && vm_page_is_valid(pg, pgoff, xfsize))
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
else if (m != NULL)
error = EAGAIN; /* send what we already got */
else if (uap->flags & SF_NODISKIO)
@@ -2083,7 +2084,7 @@ retry_space:
* when the I/O completes.
*/
vm_page_io_start(pg);
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
/*
* Get the page from backing store.
@@ -2105,10 +2106,10 @@ retry_space:
td->td_ucred, NOCRED, &resid, td);
VOP_UNLOCK(vp, 0);
after_read:
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
vm_page_io_finish(pg);
if (!error)
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
mbstat.sf_iocnt++;
}
if (error) {
@@ -2123,7 +2124,7 @@ retry_space:
pg->busy == 0 && !(pg->oflags & VPO_BUSY))
vm_page_free(pg);
vm_page_unlock(pg);
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
if (error == EAGAIN)
error = 0; /* not a real error */
break;
diff --git a/sys/kern/vfs_aio.c b/sys/kern/vfs_aio.c
index cba1638..cafe440 100644
--- a/sys/kern/vfs_aio.c
+++ b/sys/kern/vfs_aio.c
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
#include <sys/resourcevar.h>
#include <sys/signalvar.h>
#include <sys/protosw.h>
+#include <sys/rwlock.h>
#include <sys/sema.h>
#include <sys/socket.h>
#include <sys/socketvar.h>
@@ -841,9 +842,9 @@ aio_fsync_vnode(struct thread *td, struct vnode *vp)
goto drop;
vn_lock(vp, LK_EXCLUSIVE | LK_RETRY);
if (vp->v_object != NULL) {
- VM_OBJECT_LOCK(vp->v_object);
+ VM_OBJECT_WLOCK(vp->v_object);
vm_object_page_clean(vp->v_object, 0, 0, 0);
- VM_OBJECT_UNLOCK(vp->v_object);
+ VM_OBJECT_WUNLOCK(vp->v_object);
}
error = VOP_FSYNC(vp, MNT_WAIT, td);
diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index f9b4dbb..6d110ab 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -58,6 +58,7 @@ __FBSDID("$FreeBSD$");
#include <sys/kthread.h>
#include <sys/proc.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/sysctl.h>
#include <sys/vmmeter.h>
#include <sys/vnode.h>
@@ -458,7 +459,7 @@ vfs_buf_test_cache(struct buf *bp,
vm_page_t m)
{
- VM_OBJECT_LOCK_ASSERT(m->object, MA_OWNED);
+ VM_OBJECT_ASSERT_WLOCKED(m->object);
if (bp->b_flags & B_CACHE) {
int base = (foff + off) & PAGE_MASK;
if (vm_page_is_valid(m, base, size) == 0)
@@ -1388,7 +1389,7 @@ brelse(struct buf *bp)
*/
resid = bp->b_bufsize;
foff = bp->b_offset;
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
for (i = 0; i < bp->b_npages; i++) {
int had_bogus = 0;
@@ -1436,7 +1437,7 @@ brelse(struct buf *bp)
resid -= PAGE_SIZE - (foff & PAGE_MASK);
foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
}
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
if (bp->b_flags & (B_INVAL | B_RELBUF))
vfs_vmio_release(bp);
@@ -1658,7 +1659,7 @@ vfs_vmio_release(struct buf *bp)
vm_page_t m;
pmap_qremove(trunc_page((vm_offset_t)bp->b_data), bp->b_npages);
- VM_OBJECT_LOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
for (i = 0; i < bp->b_npages; i++) {
m = bp->b_pages[i];
bp->b_pages[i] = NULL;
@@ -1690,7 +1691,7 @@ vfs_vmio_release(struct buf *bp)
}
vm_page_unlock(m);
}
- VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
if (bp->b_bufsize) {
bufspacewakeup();
@@ -2467,7 +2468,7 @@ inmem(struct vnode * vp, daddr_t blkno)
size = vp->v_mount->mnt_stat.f_iosize;
off = (vm_ooffset_t)blkno * (vm_ooffset_t)vp->v_mount->mnt_stat.f_iosize;
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
for (toff = 0; toff < vp->v_mount->mnt_stat.f_iosize; toff += tinc) {
m = vm_page_lookup(obj, OFF_TO_IDX(off + toff));
if (!m)
@@ -2479,11 +2480,11 @@ inmem(struct vnode * vp, daddr_t blkno)
(vm_offset_t) ((toff + off) & PAGE_MASK), tinc) == 0)
goto notinmem;
}
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
return 1;
notinmem:
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
return (0);
}
@@ -2513,7 +2514,7 @@ vfs_clean_pages_dirty_buf(struct buf *bp)
KASSERT(bp->b_offset != NOOFFSET,
("vfs_clean_pages_dirty_buf: no buffer offset"));
- VM_OBJECT_LOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
vfs_drain_busy_pages(bp);
vfs_setdirty_locked_object(bp);
for (i = 0; i < bp->b_npages; i++) {
@@ -2526,7 +2527,7 @@ vfs_clean_pages_dirty_buf(struct buf *bp)
/* vm_page_clear_dirty(m, foff & PAGE_MASK, eoff - foff); */
foff = noff;
}
- VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
}
static void
@@ -2536,7 +2537,7 @@ vfs_setdirty_locked_object(struct buf *bp)
int i;
object = bp->b_bufobj->bo_object;
- VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
+ VM_OBJECT_ASSERT_WLOCKED(object);
/*
* We qualify the scan for modified pages on whether the
@@ -3042,7 +3043,7 @@ allocbuf(struct buf *bp, int size)
(vm_offset_t)bp->b_data) +
(desiredpages << PAGE_SHIFT),
(bp->b_npages - desiredpages));
- VM_OBJECT_LOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
for (i = desiredpages; i < bp->b_npages; i++) {
/*
* the page is not freed here -- it
@@ -3061,7 +3062,7 @@ allocbuf(struct buf *bp, int size)
vm_page_unwire(m, 0);
vm_page_unlock(m);
}
- VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
bp->b_npages = desiredpages;
}
} else if (size > bp->b_bcount) {
@@ -3082,7 +3083,7 @@ allocbuf(struct buf *bp, int size)
obj = bp->b_bufobj->bo_object;
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
while (bp->b_npages < desiredpages) {
vm_page_t m;
@@ -3144,7 +3145,7 @@ allocbuf(struct buf *bp, int size)
toff += tinc;
tinc = PAGE_SIZE;
}
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
/*
* Step 3, fixup the KVM pmap. Remember that
@@ -3399,7 +3400,7 @@ bufdone_finish(struct buf *bp)
bp->b_flags |= B_CACHE;
}
bogus = 0;
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
for (i = 0; i < bp->b_npages; i++) {
int bogusflag = 0;
int resid;
@@ -3441,7 +3442,7 @@ bufdone_finish(struct buf *bp)
iosize -= resid;
}
vm_object_pip_wakeupn(obj, 0);
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
if (bogus)
pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
bp->b_pages, bp->b_npages);
@@ -3479,7 +3480,7 @@ vfs_unbusy_pages(struct buf *bp)
return;
obj = bp->b_bufobj->bo_object;
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
for (i = 0; i < bp->b_npages; i++) {
m = bp->b_pages[i];
if (m == bogus_page) {
@@ -3494,7 +3495,7 @@ vfs_unbusy_pages(struct buf *bp)
vm_page_io_finish(m);
}
vm_object_pip_wakeupn(obj, 0);
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
}
/*
@@ -3573,7 +3574,7 @@ vfs_drain_busy_pages(struct buf *bp)
vm_page_t m;
int i, last_busied;
- VM_OBJECT_LOCK_ASSERT(bp->b_bufobj->bo_object, MA_OWNED);
+ VM_OBJECT_ASSERT_WLOCKED(bp->b_bufobj->bo_object);
last_busied = 0;
for (i = 0; i < bp->b_npages; i++) {
m = bp->b_pages[i];
@@ -3615,7 +3616,7 @@ vfs_busy_pages(struct buf *bp, int clear_modify)
foff = bp->b_offset;
KASSERT(bp->b_offset != NOOFFSET,
("vfs_busy_pages: no buffer offset"));
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
vfs_drain_busy_pages(bp);
if (bp->b_bufsize != 0)
vfs_setdirty_locked_object(bp);
@@ -3652,7 +3653,7 @@ vfs_busy_pages(struct buf *bp, int clear_modify)
}
foff = (foff + PAGE_SIZE) & ~(off_t)PAGE_MASK;
}
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
if (bogus)
pmap_qenter(trunc_page((vm_offset_t)bp->b_data),
bp->b_pages, bp->b_npages);
@@ -3683,7 +3684,7 @@ vfs_bio_set_valid(struct buf *bp, int base, int size)
base += (bp->b_offset & PAGE_MASK);
n = PAGE_SIZE - (base & PAGE_MASK);
- VM_OBJECT_LOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
for (i = base / PAGE_SIZE; size > 0 && i < bp->b_npages; ++i) {
m = bp->b_pages[i];
if (n > size)
@@ -3693,7 +3694,7 @@ vfs_bio_set_valid(struct buf *bp, int base, int size)
size -= n;
n = PAGE_SIZE;
}
- VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
}
/*
@@ -3720,13 +3721,13 @@ vfs_bio_clrbuf(struct buf *bp)
}
bp->b_flags &= ~B_INVAL;
bp->b_ioflags &= ~BIO_ERROR;
- VM_OBJECT_LOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
if ((bp->b_npages == 1) && (bp->b_bufsize < PAGE_SIZE) &&
(bp->b_offset & PAGE_MASK) == 0) {
if (bp->b_pages[0] == bogus_page)
goto unlock;
mask = (1 << (bp->b_bufsize / DEV_BSIZE)) - 1;
- VM_OBJECT_LOCK_ASSERT(bp->b_pages[0]->object, MA_OWNED);
+ VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[0]->object);
if ((bp->b_pages[0]->valid & mask) == mask)
goto unlock;
if ((bp->b_pages[0]->valid & mask) == 0) {
@@ -3745,7 +3746,7 @@ vfs_bio_clrbuf(struct buf *bp)
continue;
j = ((vm_offset_t)sa & PAGE_MASK) / DEV_BSIZE;
mask = ((1 << ((ea - sa) / DEV_BSIZE)) - 1) << j;
- VM_OBJECT_LOCK_ASSERT(bp->b_pages[i]->object, MA_OWNED);
+ VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[i]->object);
if ((bp->b_pages[i]->valid & mask) == mask)
continue;
if ((bp->b_pages[i]->valid & mask) == 0)
@@ -3759,7 +3760,7 @@ vfs_bio_clrbuf(struct buf *bp)
bp->b_pages[i]->valid |= mask;
}
unlock:
- VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
bp->b_resid = 0;
}
diff --git a/sys/kern/vfs_cluster.c b/sys/kern/vfs_cluster.c
index 663b66f..70937a2 100644
--- a/sys/kern/vfs_cluster.c
+++ b/sys/kern/vfs_cluster.c
@@ -46,6 +46,7 @@ __FBSDID("$FreeBSD$");
#include <sys/malloc.h>
#include <sys/mount.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/vmmeter.h>
#include <vm/vm.h>
#include <vm/vm_object.h>
@@ -406,21 +407,20 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp)
*/
off = tbp->b_offset;
tsize = size;
- VM_OBJECT_LOCK(tbp->b_bufobj->bo_object);
+ VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
for (j = 0; tsize > 0; j++) {
toff = off & PAGE_MASK;
tinc = tsize;
if (toff + tinc > PAGE_SIZE)
tinc = PAGE_SIZE - toff;
- VM_OBJECT_LOCK_ASSERT(tbp->b_pages[j]->object,
- MA_OWNED);
+ VM_OBJECT_ASSERT_WLOCKED(tbp->b_pages[j]->object);
if ((tbp->b_pages[j]->valid &
vm_page_bits(toff, tinc)) != 0)
break;
off += tinc;
tsize -= tinc;
}
- VM_OBJECT_UNLOCK(tbp->b_bufobj->bo_object);
+ VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
if (tsize > 0) {
bqrelse(tbp);
break;
@@ -455,7 +455,7 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp)
BUF_KERNPROC(tbp);
TAILQ_INSERT_TAIL(&bp->b_cluster.cluster_head,
tbp, b_cluster.cluster_entry);
- VM_OBJECT_LOCK(tbp->b_bufobj->bo_object);
+ VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
for (j = 0; j < tbp->b_npages; j += 1) {
vm_page_t m;
m = tbp->b_pages[j];
@@ -469,7 +469,7 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp)
if (m->valid == VM_PAGE_BITS_ALL)
tbp->b_pages[j] = bogus_page;
}
- VM_OBJECT_UNLOCK(tbp->b_bufobj->bo_object);
+ VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
/*
* Don't inherit tbp->b_bufsize as it may be larger due to
* a non-page-aligned size. Instead just aggregate using
@@ -487,13 +487,13 @@ cluster_rbuild(vp, filesize, lbn, blkno, size, run, fbp)
* Fully valid pages in the cluster are already good and do not need
* to be re-read from disk. Replace the page with bogus_page
*/
- VM_OBJECT_LOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
for (j = 0; j < bp->b_npages; j++) {
- VM_OBJECT_LOCK_ASSERT(bp->b_pages[j]->object, MA_OWNED);
+ VM_OBJECT_ASSERT_WLOCKED(bp->b_pages[j]->object);
if (bp->b_pages[j]->valid == VM_PAGE_BITS_ALL)
bp->b_pages[j] = bogus_page;
}
- VM_OBJECT_UNLOCK(bp->b_bufobj->bo_object);
+ VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
if (bp->b_bufsize > bp->b_kvasize)
panic("cluster_rbuild: b_bufsize(%ld) > b_kvasize(%d)\n",
bp->b_bufsize, bp->b_kvasize);
@@ -918,12 +918,12 @@ cluster_wbuild(vp, size, start_lbn, len)
if (tbp->b_flags & B_VMIO) {
vm_page_t m;
- VM_OBJECT_LOCK(tbp->b_bufobj->bo_object);
+ VM_OBJECT_WLOCK(tbp->b_bufobj->bo_object);
if (i != 0) { /* if not first buffer */
for (j = 0; j < tbp->b_npages; j += 1) {
m = tbp->b_pages[j];
if (m->oflags & VPO_BUSY) {
- VM_OBJECT_UNLOCK(
+ VM_OBJECT_WUNLOCK(
tbp->b_object);
bqrelse(tbp);
goto finishcluster;
@@ -940,7 +940,7 @@ cluster_wbuild(vp, size, start_lbn, len)
bp->b_npages++;
}
}
- VM_OBJECT_UNLOCK(tbp->b_bufobj->bo_object);
+ VM_OBJECT_WUNLOCK(tbp->b_bufobj->bo_object);
}
bp->b_bcount += size;
bp->b_bufsize += size;
diff --git a/sys/kern/vfs_default.c b/sys/kern/vfs_default.c
index 1dd0185..ed071b0 100644
--- a/sys/kern/vfs_default.c
+++ b/sys/kern/vfs_default.c
@@ -47,8 +47,8 @@ __FBSDID("$FreeBSD$");
#include <sys/lockf.h>
#include <sys/malloc.h>
#include <sys/mount.h>
-#include <sys/mutex.h>
#include <sys/namei.h>
+#include <sys/rwlock.h>
#include <sys/fcntl.h>
#include <sys/unistd.h>
#include <sys/vnode.h>
@@ -1047,10 +1047,10 @@ vop_stdadvise(struct vop_advise_args *ap)
if (vp->v_object != NULL) {
start = trunc_page(ap->a_start);
end = round_page(ap->a_end);
- VM_OBJECT_LOCK(vp->v_object);
+ VM_OBJECT_WLOCK(vp->v_object);
vm_object_page_cache(vp->v_object, OFF_TO_IDX(start),
OFF_TO_IDX(end));
- VM_OBJECT_UNLOCK(vp->v_object);
+ VM_OBJECT_WUNLOCK(vp->v_object);
}
VOP_UNLOCK(vp, 0);
break;
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index 1c26368..d0c6bb8 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -67,6 +67,7 @@ __FBSDID("$FreeBSD$");
#include <sys/namei.h>
#include <sys/priv.h>
#include <sys/reboot.h>
+#include <sys/rwlock.h>
#include <sys/sched.h>
#include <sys/sleepqueue.h>
#include <sys/smp.h>
@@ -1244,9 +1245,9 @@ bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
bufobj_wwait(bo, 0, 0);
BO_UNLOCK(bo);
if (bo->bo_object != NULL) {
- VM_OBJECT_LOCK(bo->bo_object);
+ VM_OBJECT_WLOCK(bo->bo_object);
vm_object_pip_wait(bo->bo_object, "bovlbx");
- VM_OBJECT_UNLOCK(bo->bo_object);
+ VM_OBJECT_WUNLOCK(bo->bo_object);
}
BO_LOCK(bo);
} while (bo->bo_numoutput > 0);
@@ -1257,10 +1258,10 @@ bufobj_invalbuf(struct bufobj *bo, int flags, int slpflag, int slptimeo)
*/
if (bo->bo_object != NULL &&
(flags & (V_ALT | V_NORMAL | V_CLEANONLY)) == 0) {
- VM_OBJECT_LOCK(bo->bo_object);
+ VM_OBJECT_WLOCK(bo->bo_object);
vm_object_page_remove(bo->bo_object, 0, 0, (flags & V_SAVE) ?
OBJPR_CLEANONLY : 0);
- VM_OBJECT_UNLOCK(bo->bo_object);
+ VM_OBJECT_WUNLOCK(bo->bo_object);
}
#ifdef INVARIANTS
@@ -2520,9 +2521,9 @@ vinactive(struct vnode *vp, struct thread *td)
*/
obj = vp->v_object;
if (obj != NULL && (obj->flags & OBJ_MIGHTBEDIRTY) != 0) {
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
vm_object_page_clean(obj, 0, 0, OBJPC_NOSYNC);
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
}
VOP_INACTIVE(vp, td);
VI_LOCK(vp);
@@ -2603,9 +2604,9 @@ loop:
*/
if (flags & WRITECLOSE) {
if (vp->v_object != NULL) {
- VM_OBJECT_LOCK(vp->v_object);
+ VM_OBJECT_WLOCK(vp->v_object);
vm_object_page_clean(vp->v_object, 0, 0, 0);
- VM_OBJECT_UNLOCK(vp->v_object);
+ VM_OBJECT_WUNLOCK(vp->v_object);
}
error = VOP_FSYNC(vp, MNT_WAIT, td);
if (error != 0) {
@@ -3503,11 +3504,11 @@ vfs_msync(struct mount *mp, int flags)
obj = vp->v_object;
if (obj != NULL) {
- VM_OBJECT_LOCK(obj);
+ VM_OBJECT_WLOCK(obj);
vm_object_page_clean(obj, 0, 0,
flags == MNT_WAIT ?
OBJPC_SYNC : OBJPC_NOSYNC);
- VM_OBJECT_UNLOCK(obj);
+ VM_OBJECT_WUNLOCK(obj);
}
vput(vp);
}
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index 4c1d97c..fc78235 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -61,6 +61,7 @@ __FBSDID("$FreeBSD$");
#include <sys/filio.h>
#include <sys/limits.h>
#include <sys/linker.h>
+#include <sys/rwlock.h>
#include <sys/sdt.h>
#include <sys/stat.h>
#include <sys/sx.h>
@@ -3437,9 +3438,9 @@ sys_fsync(td, uap)
vn_lock(vp, lock_flags | LK_RETRY);
AUDIT_ARG_VNODE1(vp);
if (vp->v_object != NULL) {
- VM_OBJECT_LOCK(vp->v_object);
+ VM_OBJECT_WLOCK(vp->v_object);
vm_object_page_clean(vp->v_object, 0, 0, 0);
- VM_OBJECT_UNLOCK(vp->v_object);
+ VM_OBJECT_WUNLOCK(vp->v_object);
}
error = VOP_FSYNC(vp, MNT_WAIT, td);
diff --git a/sys/kern/vfs_vnops.c b/sys/kern/vfs_vnops.c
index 96ce9e2..d367340 100644
--- a/sys/kern/vfs_vnops.c
+++ b/sys/kern/vfs_vnops.c
@@ -55,6 +55,7 @@ __FBSDID("$FreeBSD$");
#include <sys/buf.h>
#include <sys/filio.h>
#include <sys/resourcevar.h>
+#include <sys/rwlock.h>
#include <sys/sx.h>
#include <sys/sysctl.h>
#include <sys/ttycom.h>
@@ -1892,9 +1893,9 @@ vn_pages_remove(struct vnode *vp, vm_pindex_t start, vm_pindex_t end)
if ((object = vp->v_object) == NULL)
return;
- VM_OBJECT_LOCK(object);
+ VM_OBJECT_WLOCK(object);
vm_object_page_remove(object, start, end, 0);
- VM_OBJECT_UNLOCK(object);
+ VM_OBJECT_WUNLOCK(object);
}
int
OpenPOWER on IntegriCloud