- Acquire the vm_object's lock when performing vm_object_page_clean().

 - Add a parameter to vm_pageout_flush() that tells vm_pageout_flush()
   whether its caller has locked the vm_object.  (This is a temporary
   measure to bootstrap vm_object locking.)

18 files changed, 45 insertions, 14 deletions

diff --git a/sys/kern/kern_mac.c b/sys/kern/kern_mac.c
index 9289d7f..15c95a0 100644
--- a/sys/kern/kern_mac.c
+++ b/sys/kern/kern_mac.c
@@ -2093,11 +2093,13 @@ mac_cred_mmapped_drop_perms_recurse(struct thread *td, struct ucred *cred,
 				 */
 				vm_object_reference(object);
 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object,
 				    OFF_TO_IDX(offset),
 				    OFF_TO_IDX(offset + vme->end - vme->start +
 					PAGE_MASK),
 				    OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(vp, 0, td);
 				vm_object_deallocate(object);
 				/*
diff --git a/sys/kern/vfs_extattr.c b/sys/kern/vfs_extattr.c
index 45ba5a1..21b2046 100644
--- a/sys/kern/vfs_extattr.c
+++ b/sys/kern/vfs_extattr.c
@@ -2739,7 +2739,9 @@ fsync(td, uap)
 	}
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
 	if (VOP_GETVOBJECT(vp, &obj) == 0) {
+		VM_OBJECT_LOCK(obj);
 		vm_object_page_clean(obj, 0, 0, 0);
+		VM_OBJECT_UNLOCK(obj);
 	}
 	error = VOP_FSYNC(vp, fp->f_cred, MNT_WAIT, td);
 	if (error == 0 && vp->v_mount && (vp->v_mount->mnt_flag & MNT_SOFTDEP)
diff --git a/sys/kern/vfs_subr.c b/sys/kern/vfs_subr.c
index b2bee4e..f63e5ca 100644
--- a/sys/kern/vfs_subr.c
+++ b/sys/kern/vfs_subr.c
@@ -3206,9 +3206,11 @@ loop:
 				}
 
 				if (VOP_GETVOBJECT(vp, &obj) == 0) {
+					VM_OBJECT_LOCK(obj);
 					vm_object_page_clean(obj, 0, 0,
 					    flags == MNT_WAIT ?
 					    OBJPC_SYNC : OBJPC_NOSYNC);
+					VM_OBJECT_UNLOCK(obj);
 				}
 				vput(vp);
 			}
diff --git a/sys/kern/vfs_syscalls.c b/sys/kern/vfs_syscalls.c
index 45ba5a1..21b2046 100644
--- a/sys/kern/vfs_syscalls.c
+++ b/sys/kern/vfs_syscalls.c
@@ -2739,7 +2739,9 @@ fsync(td, uap)
 	}
 	vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
 	if (VOP_GETVOBJECT(vp, &obj) == 0) {
+		VM_OBJECT_LOCK(obj);
 		vm_object_page_clean(obj, 0, 0, 0);
+		VM_OBJECT_UNLOCK(obj);
 	}
 	error = VOP_FSYNC(vp, fp->f_cred, MNT_WAIT, td);
 	if (error == 0 && vp->v_mount && (vp->v_mount->mnt_flag & MNT_SOFTDEP)
diff --git a/sys/nfsserver/nfs_serv.c b/sys/nfsserver/nfs_serv.c
index 5c8554e..7573444 100644
--- a/sys/nfsserver/nfs_serv.c
+++ b/sys/nfsserver/nfs_serv.c
@@ -3654,7 +3654,9 @@ nfsrv_commit(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 		 */
 		if (vp->v_object &&
 		   (vp->v_object->flags & OBJ_MIGHTBEDIRTY)) {
+			VM_OBJECT_LOCK(vp->v_object);
 			vm_object_page_clean(vp->v_object, 0, 0, OBJPC_SYNC);
+			VM_OBJECT_UNLOCK(vp->v_object);
 		}
 		error = VOP_FSYNC(vp, cred, MNT_WAIT, td);
 	} else {
@@ -3683,7 +3685,9 @@ nfsrv_commit(struct nfsrv_descript *nfsd, struct nfssvc_sock *slp,
 
 		if (vp->v_object &&
 		   (vp->v_object->flags & OBJ_MIGHTBEDIRTY)) {
+			VM_OBJECT_LOCK(vp->v_object);
 			vm_object_page_clean(vp->v_object, off / PAGE_SIZE, (cnt + PAGE_MASK) / PAGE_SIZE, OBJPC_SYNC);
+			VM_OBJECT_UNLOCK(vp->v_object);
 		}
 
 		s = splbio();
diff --git a/sys/security/mac/mac_framework.c b/sys/security/mac/mac_framework.c
index 9289d7f..15c95a0 100644
--- a/sys/security/mac/mac_framework.c
+++ b/sys/security/mac/mac_framework.c
@@ -2093,11 +2093,13 @@ mac_cred_mmapped_drop_perms_recurse(struct thread *td, struct ucred *cred,
 				 */
 				vm_object_reference(object);
 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object,
 				    OFF_TO_IDX(offset),
 				    OFF_TO_IDX(offset + vme->end - vme->start +
 					PAGE_MASK),
 				    OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(vp, 0, td);
 				vm_object_deallocate(object);
 				/*
diff --git a/sys/security/mac/mac_internal.h b/sys/security/mac/mac_internal.h
index 9289d7f..15c95a0 100644
--- a/sys/security/mac/mac_internal.h
+++ b/sys/security/mac/mac_internal.h
@@ -2093,11 +2093,13 @@ mac_cred_mmapped_drop_perms_recurse(struct thread *td, struct ucred *cred,
 				 */
 				vm_object_reference(object);
 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object,
 				    OFF_TO_IDX(offset),
 				    OFF_TO_IDX(offset + vme->end - vme->start +
 					PAGE_MASK),
 				    OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(vp, 0, td);
 				vm_object_deallocate(object);
 				/*
diff --git a/sys/security/mac/mac_net.c b/sys/security/mac/mac_net.c
index 9289d7f..15c95a0 100644
--- a/sys/security/mac/mac_net.c
+++ b/sys/security/mac/mac_net.c
@@ -2093,11 +2093,13 @@ mac_cred_mmapped_drop_perms_recurse(struct thread *td, struct ucred *cred,
 				 */
 				vm_object_reference(object);
 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object,
 				    OFF_TO_IDX(offset),
 				    OFF_TO_IDX(offset + vme->end - vme->start +
 					PAGE_MASK),
 				    OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(vp, 0, td);
 				vm_object_deallocate(object);
 				/*
diff --git a/sys/security/mac/mac_pipe.c b/sys/security/mac/mac_pipe.c
index 9289d7f..15c95a0 100644
--- a/sys/security/mac/mac_pipe.c
+++ b/sys/security/mac/mac_pipe.c
@@ -2093,11 +2093,13 @@ mac_cred_mmapped_drop_perms_recurse(struct thread *td, struct ucred *cred,
 				 */
 				vm_object_reference(object);
 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object,
 				    OFF_TO_IDX(offset),
 				    OFF_TO_IDX(offset + vme->end - vme->start +
 					PAGE_MASK),
 				    OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(vp, 0, td);
 				vm_object_deallocate(object);
 				/*
diff --git a/sys/security/mac/mac_process.c b/sys/security/mac/mac_process.c
index 9289d7f..15c95a0 100644
--- a/sys/security/mac/mac_process.c
+++ b/sys/security/mac/mac_process.c
@@ -2093,11 +2093,13 @@ mac_cred_mmapped_drop_perms_recurse(struct thread *td, struct ucred *cred,
 				 */
 				vm_object_reference(object);
 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object,
 				    OFF_TO_IDX(offset),
 				    OFF_TO_IDX(offset + vme->end - vme->start +
 					PAGE_MASK),
 				    OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(vp, 0, td);
 				vm_object_deallocate(object);
 				/*
diff --git a/sys/security/mac/mac_syscalls.c b/sys/security/mac/mac_syscalls.c
index 9289d7f..15c95a0 100644
--- a/sys/security/mac/mac_syscalls.c
+++ b/sys/security/mac/mac_syscalls.c
@@ -2093,11 +2093,13 @@ mac_cred_mmapped_drop_perms_recurse(struct thread *td, struct ucred *cred,
 				 */
 				vm_object_reference(object);
 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object,
 				    OFF_TO_IDX(offset),
 				    OFF_TO_IDX(offset + vme->end - vme->start +
 					PAGE_MASK),
 				    OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(vp, 0, td);
 				vm_object_deallocate(object);
 				/*
diff --git a/sys/security/mac/mac_system.c b/sys/security/mac/mac_system.c
index 9289d7f..15c95a0 100644
--- a/sys/security/mac/mac_system.c
+++ b/sys/security/mac/mac_system.c
@@ -2093,11 +2093,13 @@ mac_cred_mmapped_drop_perms_recurse(struct thread *td, struct ucred *cred,
 				 */
 				vm_object_reference(object);
 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object,
 				    OFF_TO_IDX(offset),
 				    OFF_TO_IDX(offset + vme->end - vme->start +
 					PAGE_MASK),
 				    OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(vp, 0, td);
 				vm_object_deallocate(object);
 				/*
diff --git a/sys/security/mac/mac_vfs.c b/sys/security/mac/mac_vfs.c
index 9289d7f..15c95a0 100644
--- a/sys/security/mac/mac_vfs.c
+++ b/sys/security/mac/mac_vfs.c
@@ -2093,11 +2093,13 @@ mac_cred_mmapped_drop_perms_recurse(struct thread *td, struct ucred *cred,
 				 */
 				vm_object_reference(object);
 				vn_lock(vp, LK_EXCLUSIVE | LK_RETRY, td);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object,
 				    OFF_TO_IDX(offset),
 				    OFF_TO_IDX(offset + vme->end - vme->start +
 					PAGE_MASK),
 				    OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(vp, 0, td);
 				vm_object_deallocate(object);
 				/*
diff --git a/sys/vm/vm_contig.c b/sys/vm/vm_contig.c
index da30a00..cf5ee09 100644
--- a/sys/vm/vm_contig.c
+++ b/sys/vm/vm_contig.c
@@ -105,14 +105,16 @@ vm_contig_launder(int queue)
 				vm_page_unlock_queues();
 				vn_lock(object->handle,
 				    LK_EXCLUSIVE | LK_RETRY, curthread);
+				VM_OBJECT_LOCK(object);
 				vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
+				VM_OBJECT_UNLOCK(object);
 				VOP_UNLOCK(object->handle, 0, curthread);
 				vm_page_lock_queues();
 				return (TRUE);
 			} else if (object->type == OBJT_SWAP ||
 				   object->type == OBJT_DEFAULT) {
 				m_tmp = m;
-				vm_pageout_flush(&m_tmp, 1, 0);
+				vm_pageout_flush(&m_tmp, 1, 0, FALSE);
 				return (TRUE);
 			}
 		} else if (m->busy == 0 && m->hold_count == 0)
diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index fdce55c..f45c471 100644
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -1976,10 +1976,12 @@ vm_map_clean(
 			vn_lock(object->handle, LK_EXCLUSIVE | LK_RETRY, curthread);
 			flags = (syncio || invalidate) ? OBJPC_SYNC : 0;
 			flags |= invalidate ? OBJPC_INVAL : 0;
+			VM_OBJECT_LOCK(object);
 			vm_object_page_clean(object,
 			    OFF_TO_IDX(offset),
 			    OFF_TO_IDX(offset + size + PAGE_MASK),
 			    flags);
+			VM_OBJECT_UNLOCK(object);
 			VOP_UNLOCK(object->handle, 0, curthread);
 			vm_object_deallocate(object);
 		}
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 1baf834..486b7c6 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -557,7 +557,9 @@ vm_object_terminate(vm_object_t object)
 		/*
 		 * Clean pages and flush buffers.
 		 */
+		VM_OBJECT_LOCK(object);
 		vm_object_page_clean(object, 0, 0, OBJPC_SYNC);
+		VM_OBJECT_UNLOCK(object);
 
 		vp = (struct vnode *) object->handle;
 		vinvalbuf(vp, V_SAVE, NOCRED, NULL, 0, 0);
@@ -638,7 +640,7 @@ vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int
 	int curgeneration;
 
 	GIANT_REQUIRED;
-
+	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 	if (object->type != OBJT_VNODE ||
 		(object->flags & OBJ_MIGHTBEDIRTY) == 0)
 		return;
@@ -721,9 +723,7 @@ vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int
 		 */
 		if (tscan >= tend && (tstart || tend < object->size)) {
 			vm_page_unlock_queues();
-			VM_OBJECT_LOCK(object);
 			vm_object_clear_flag(object, OBJ_CLEANING);
-			VM_OBJECT_UNLOCK(object);
 			return;
 		}
 		pagerflags &= ~VM_PAGER_IGNORE_CLEANCHK;
@@ -749,9 +749,7 @@ vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int
 	if (clearobjflags && (tstart == 0) && (tend == object->size)) {
 		struct vnode *vp;
 
-		VM_OBJECT_LOCK(object);
 		vm_object_clear_flag(object, OBJ_WRITEABLE|OBJ_MIGHTBEDIRTY);
-		VM_OBJECT_UNLOCK(object);
 		if (object->type == OBJT_VNODE &&
 		    (vp = (struct vnode *)object->handle) != NULL) {
 			VI_LOCK(vp);
@@ -817,9 +815,7 @@ again:
 	VOP_FSYNC(vp, NULL, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc);
 #endif
 
-	VM_OBJECT_LOCK(object);
 	vm_object_clear_flag(object, OBJ_CLEANING);
-	VM_OBJECT_UNLOCK(object);
 	return;
 }
 
@@ -917,7 +913,7 @@ vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration,
 	runlen = maxb + maxf + 1;
 
 	splx(s);
-	vm_pageout_flush(ma, runlen, pagerflags);
+	vm_pageout_flush(ma, runlen, pagerflags, TRUE);
 	for (i = 0; i < runlen; i++) {
 		if (ma[i]->valid & ma[i]->dirty) {
 			pmap_page_protect(ma[i], VM_PROT_READ);
diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index 9c5afcf..c688d61 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -348,7 +348,7 @@ more:
 	/*
 	 * we allow reads during pageouts...
 	 */
-	return vm_pageout_flush(&mc[page_base], pageout_count, 0);
+	return vm_pageout_flush(&mc[page_base], pageout_count, 0, FALSE);
 }
 
 /*
@@ -361,10 +361,11 @@ more:
  *	the ordering.
  */
 int
-vm_pageout_flush(mc, count, flags)
+vm_pageout_flush(mc, count, flags, is_object_locked)
 	vm_page_t *mc;
 	int count;
 	int flags;
+	int is_object_locked;
 {
 	vm_object_t object;
 	int pageout_status[count];
@@ -389,7 +390,8 @@ vm_pageout_flush(mc, count, flags)
 	}
 	object = mc[0]->object;
 	vm_page_unlock_queues();
-	VM_OBJECT_LOCK(object);
+	if (!is_object_locked)
+		VM_OBJECT_LOCK(object);
 	vm_object_pip_add(object, count);
 	VM_OBJECT_UNLOCK(object);
 
@@ -442,7 +444,8 @@ vm_pageout_flush(mc, count, flags)
 				pmap_page_protect(mt, VM_PROT_READ);
 		}
 	}
-	VM_OBJECT_UNLOCK(object);
+	if (!is_object_locked)
+		VM_OBJECT_UNLOCK(object);
 	return numpagedout;
 }
 
diff --git a/sys/vm/vm_pageout.h b/sys/vm/vm_pageout.h
index 8ebebe2..04386df 100644
--- a/sys/vm/vm_pageout.h
+++ b/sys/vm/vm_pageout.h
@@ -113,6 +113,6 @@ int swap_pager_isswapped(vm_object_t, int);
 #ifdef _KERNEL
 void vm_pageout_page(vm_page_t, vm_object_t);
 void vm_pageout_cluster(vm_page_t, vm_object_t);
-int vm_pageout_flush(vm_page_t *, int, int);
+int vm_pageout_flush(vm_page_t *, int, int, int is_object_locked);
 #endif
 #endif	/* _VM_VM_PAGEOUT_H_ */