MFC r307218:

Fix a race in vm_page_busy_sleep(9).

7 files changed, 23 insertions, 23 deletions

diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
index 23778fb..3262029 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zfs_vnops.c
@@ -421,7 +421,7 @@ page_busy(vnode_t *vp, int64_t start, int64_t off, int64_t nbytes)
 				vm_page_reference(pp);
 				vm_page_lock(pp);
 				zfs_vmobject_wunlock(obj);
-				vm_page_busy_sleep(pp, "zfsmwb");
+				vm_page_busy_sleep(pp, "zfsmwb", true);
 				zfs_vmobject_wlock(obj);
 				continue;
 			}
@@ -476,7 +476,7 @@ page_hold(vnode_t *vp, int64_t start)
 				vm_page_reference(pp);
 				vm_page_lock(pp);
 				zfs_vmobject_wunlock(obj);
-				vm_page_busy_sleep(pp, "zfsmwb");
+				vm_page_busy_sleep(pp, "zfsmwb", true);
 				zfs_vmobject_wlock(obj);
 				continue;
 			}
diff --git a/sys/dev/drm2/i915/i915_gem.c b/sys/dev/drm2/i915/i915_gem.c
index 5114008..4624997 100644
--- a/sys/dev/drm2/i915/i915_gem.c
+++ b/sys/dev/drm2/i915/i915_gem.c
@@ -1976,7 +1976,7 @@ retry:
 			DRM_UNLOCK(dev);
 			vm_page_lock(m);
 			VM_OBJECT_WUNLOCK(vm_obj);
-			vm_page_busy_sleep(m, "915pee");
+			vm_page_busy_sleep(m, "915pee", false);
 			goto retry;
 		}
 		goto have_page;
@@ -2037,7 +2037,7 @@ retry:
 		DRM_UNLOCK(dev);
 		vm_page_lock(m);
 		VM_OBJECT_WUNLOCK(vm_obj);
-		vm_page_busy_sleep(m, "915pbs");
+		vm_page_busy_sleep(m, "915pbs", false);
 		goto retry;
 	}
 	if (vm_page_insert(m, vm_obj, OFF_TO_IDX(offset))) {
diff --git a/sys/dev/drm2/ttm/ttm_bo_vm.c b/sys/dev/drm2/ttm/ttm_bo_vm.c
index 9c98448..f172a03 100644
--- a/sys/dev/drm2/ttm/ttm_bo_vm.c
+++ b/sys/dev/drm2/ttm/ttm_bo_vm.c
@@ -239,7 +239,7 @@ reserve:
 	if (vm_page_busied(m)) {
 		vm_page_lock(m);
 		VM_OBJECT_WUNLOCK(vm_obj);
-		vm_page_busy_sleep(m, "ttmpbs");
+		vm_page_busy_sleep(m, "ttmpbs", false);
 		VM_OBJECT_WLOCK(vm_obj);
 		ttm_mem_io_unlock(man);
 		ttm_bo_unreserve(bo);
diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index f728859..f28db3e 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -1724,7 +1724,7 @@ brelse(struct buf *bp)
 				while (vm_page_xbusied(m)) {
 					vm_page_lock(m);
 					VM_OBJECT_WUNLOCK(obj);
-					vm_page_busy_sleep(m, "mbncsh");
+					vm_page_busy_sleep(m, "mbncsh", true);
 					VM_OBJECT_WLOCK(obj);
 				}
 				if (pmap_page_wired_mappings(m) == 0)
@@ -4069,7 +4069,7 @@ vfs_drain_busy_pages(struct buf *bp)
 			while (vm_page_xbusied(m)) {
 				vm_page_lock(m);
 				VM_OBJECT_WUNLOCK(bp->b_bufobj->bo_object);
-				vm_page_busy_sleep(m, "vbpage");
+				vm_page_busy_sleep(m, "vbpage", true);
 				VM_OBJECT_WLOCK(bp->b_bufobj->bo_object);
 			}
 		}
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index b2a7013..d4c7116 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -1182,7 +1182,7 @@ shadowlookup:
 			if (object != tobject)
 				VM_OBJECT_WUNLOCK(object);
 			VM_OBJECT_WUNLOCK(tobject);
-			vm_page_busy_sleep(m, "madvpo");
+			vm_page_busy_sleep(m, "madvpo", false);
 			VM_OBJECT_WLOCK(object);
   			goto relookup;
 		}
@@ -1361,7 +1361,7 @@ retry:
 			VM_OBJECT_WUNLOCK(new_object);
 			vm_page_lock(m);
 			VM_OBJECT_WUNLOCK(orig_object);
-			vm_page_busy_sleep(m, "spltwt");
+			vm_page_busy_sleep(m, "spltwt", false);
 			VM_OBJECT_WLOCK(orig_object);
 			VM_OBJECT_WLOCK(new_object);
 			goto retry;
@@ -1450,7 +1450,7 @@ vm_object_backing_scan_wait(vm_object_t object, vm_page_t p, vm_page_t next,
 	if (p == NULL)
 		VM_WAIT;
 	else
-		vm_page_busy_sleep(p, "vmocol");
+		vm_page_busy_sleep(p, "vmocol", false);
 	VM_OBJECT_WLOCK(object);
 	VM_OBJECT_WLOCK(backing_object);
 	return (TAILQ_FIRST(&backing_object->memq));
@@ -1917,7 +1917,7 @@ again:
 		vm_page_lock(p);
 		if (vm_page_xbusied(p)) {
 			VM_OBJECT_WUNLOCK(object);
-			vm_page_busy_sleep(p, "vmopax");
+			vm_page_busy_sleep(p, "vmopax", true);
 			VM_OBJECT_WLOCK(object);
 			goto again;
 		}
@@ -1932,7 +1932,7 @@ again:
 		}
 		if (vm_page_busied(p)) {
 			VM_OBJECT_WUNLOCK(object);
-			vm_page_busy_sleep(p, "vmopar");
+			vm_page_busy_sleep(p, "vmopar", false);
 			VM_OBJECT_WLOCK(object);
 			goto again;
 		}
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index f166649..054148b 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -601,21 +601,20 @@ vm_page_sunbusy(vm_page_t m)
  *	This is used to implement the hard-path of busying mechanism.
  *
  *	The given page must be locked.
+ *
+ *	If nonshared is true, sleep only if the page is xbusy.
  */
 void
-vm_page_busy_sleep(vm_page_t m, const char *wmesg)
+vm_page_busy_sleep(vm_page_t m, const char *wmesg, bool nonshared)
 {
 	u_int x;
 
-	vm_page_lock_assert(m, MA_OWNED);
+	vm_page_assert_locked(m);
 
 	x = m->busy_lock;
-	if (x == VPB_UNBUSIED) {
-		vm_page_unlock(m);
-		return;
-	}
-	if ((x & VPB_BIT_WAITERS) == 0 &&
-	    !atomic_cmpset_int(&m->busy_lock, x, x | VPB_BIT_WAITERS)) {
+	if (x == VPB_UNBUSIED || (nonshared && (x & VPB_BIT_SHARED) != 0) ||
+	    ((x & VPB_BIT_WAITERS) == 0 &&
+	    !atomic_cmpset_int(&m->busy_lock, x, x | VPB_BIT_WAITERS))) {
 		vm_page_unlock(m);
 		return;
 	}
@@ -928,7 +927,7 @@ vm_page_sleep_if_busy(vm_page_t m, const char *msg)
 		obj = m->object;
 		vm_page_lock(m);
 		VM_OBJECT_WUNLOCK(obj);
-		vm_page_busy_sleep(m, msg);
+		vm_page_busy_sleep(m, msg, false);
 		VM_OBJECT_WLOCK(obj);
 		return (TRUE);
 	}
@@ -2717,7 +2716,8 @@ retrylookup:
 			vm_page_aflag_set(m, PGA_REFERENCED);
 			vm_page_lock(m);
 			VM_OBJECT_WUNLOCK(object);
-			vm_page_busy_sleep(m, "pgrbwt");
+			vm_page_busy_sleep(m, "pgrbwt", (allocflags &
+			    VM_ALLOC_IGN_SBUSY) != 0);
 			VM_OBJECT_WLOCK(object);
 			goto retrylookup;
 		} else {
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 3ab4c24..7cf949e 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -420,7 +420,7 @@ malloc2vm_flags(int malloc_flags)
 #endif
 
 void vm_page_busy_downgrade(vm_page_t m);
-void vm_page_busy_sleep(vm_page_t m, const char *msg);
+void vm_page_busy_sleep(vm_page_t m, const char *msg, bool nonshared);
 void vm_page_flash(vm_page_t m);
 void vm_page_hold(vm_page_t mem);
 void vm_page_unhold(vm_page_t mem);