Revamp VM map wiring.

* Allow no-fault wiring/unwiring to succeed for consistency;
  however, the wired count remains at zero, so it's a special case.

* Fix issues inside vm_map_wire() and vm_map_unwire() where the
  exact state of user wiring (one or zero) and system wiring
  (zero or more) could be confused; for example, system unwiring
  could succeed in removing a user wire, instead of being an
  error.

* Require all mappings to be unwired before they are deleted.
  When VM space is still wired upon deletion, it will be waited
  upon for the following unwire.  This makes vslock(9) work
  rather than allowing kernel-locked memory to be deleted
  out from underneath of its consumer as it would before.

1 files changed, 32 insertions, 14 deletions

diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c
index c354655..e4dfccc 100644
--- a/sys/vm/vm_map.c
+++ b/sys/vm/vm_map.c
@@ -1606,6 +1606,8 @@ vm_map_inherit(vm_map_t map, vm_offset_t start, vm_offset_t end,
  *	vm_map_unwire:
  *
  *	Implements both kernel and user unwiring.
+ *
+ *	Ignores MAP_NOFAULT (wired_count == 0 is okay) for kernel unwiring.
  */
 int
 vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end,
@@ -1698,8 +1700,9 @@ vm_map_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 		/*
 		 * If system unwiring, require that the entry is system wired.
 		 */
-		if (!user_unwire && entry->wired_count < ((entry->eflags &
-		    MAP_ENTRY_USER_WIRED) ? 2 : 1)) {
+		if (user_unwire ? vm_map_entry_user_wired_count(entry) == 0 :
+		    (vm_map_entry_system_wired_count(entry) == 0 &&
+		    (entry->eflags & MAP_ENTRY_NOFAULT) == 0)) {
 			end = entry->end;
 			rv = KERN_INVALID_ARGUMENT;
 			goto done;
@@ -1722,8 +1725,11 @@ done:
 		    (entry->eflags & MAP_ENTRY_USER_WIRED))) {
 			if (user_unwire)
 				entry->eflags &= ~MAP_ENTRY_USER_WIRED;
-			entry->wired_count--;
-			if (entry->wired_count == 0) {
+			if (user_unwire ||
+			    (entry->eflags & MAP_ENTRY_NOFAULT) == 0)
+				entry->wired_count--;
+			if (entry->wired_count == 0 &&
+			    (entry->eflags & MAP_ENTRY_NOFAULT) == 0) {
 				/*
 				 * Retain the map lock.
 				 */
@@ -1831,10 +1837,14 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 		 */
 		entry->eflags |= MAP_ENTRY_IN_TRANSITION;
 		/*
-		 *
+		 * The wired_count keeps track of zero or one user
+		 * wirings.  It also keeps track of system wirings if
+		 * MAP_ENTRY_NOFAULT is not set.
 		 */
-		if (entry->wired_count == 0) {
-			entry->wired_count++;
+		if ((user_wire ? vm_map_entry_user_wired_count(entry) == 0 :
+		    (entry->eflags & MAP_ENTRY_NOFAULT) == 0) &&
+		    entry->wired_count++ == 0 &&
+		    (entry->eflags & MAP_ENTRY_NOFAULT) == 0) {
 			saved_start = entry->start;
 			saved_end = entry->end;
 			fictitious = entry->object.vm_object != NULL &&
@@ -1883,9 +1893,6 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end,
 				end = entry->end;
 				goto done;
 			}
-		} else if (!user_wire ||
-			   (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) {
-			entry->wired_count++;
 		}
 		/*
 		 * Check the map for holes in the specified region.
@@ -1913,6 +1920,11 @@ done:
 	entry = first_entry;
 	while (entry != &map->header && entry->start < end) {
 		if (rv == KERN_SUCCESS) {
+			/*
+			 * The MAP_ENTRY_USER_WIRED may already have been
+			 * set.  If so, this statement has no effect
+			 * (and wired_count will not have changed).
+			 */
 			if (user_wire)
 				entry->eflags |= MAP_ENTRY_USER_WIRED;
 		} else if (entry->wired_count == -1) {
@@ -1922,10 +1934,11 @@ done:
 			 */
 			entry->wired_count = 0;
 		} else {
-			if (!user_wire ||
-			    (entry->eflags & MAP_ENTRY_USER_WIRED) == 0)
+			if (user_wire ||
+			    (entry->eflags & MAP_ENTRY_NOFAULT) == 0)
 				entry->wired_count--;
-			if (entry->wired_count == 0) {
+			if (entry->wired_count == 0 &&
+			    (entry->eflags & MAP_ENTRY_NOFAULT) == 0) {
 				/*
 				 * Retain the map lock.
 				 */
@@ -2053,6 +2066,9 @@ vm_map_sync(
 static void
 vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry)
 {
+	KASSERT(vm_map_entry_user_wired_count(entry) == 1 &&
+	    vm_map_entry_system_wired_count(entry) == 0,
+	    ("system/user wiring mistake"));
 	vm_fault_unwire(map, entry->start, entry->end,
 	    entry->object.vm_object != NULL &&
 	    entry->object.vm_object->type == OBJT_DEVICE);
@@ -2137,8 +2153,10 @@ vm_map_delete(vm_map_t map, vm_offset_t start, vm_offset_t end)
 
 		/*
 		 * Wait for wiring or unwiring of an entry to complete.
+		 * Also wait for any system wirings to disappear.
 		 */
-		if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0) {
+		if ((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 ||
+		    vm_map_entry_system_wired_count(entry) != 0) {
 			unsigned int last_timestamp;
 			vm_offset_t saved_start;
 			vm_map_entry_t tmp_entry;