On all the architectures, avoid to preallocate the physical memory

for nodes used in vm_radix.
On architectures supporting direct mapping, also avoid to pre-allocate
the KVA for such nodes.

In order to do so make the operations derived from vm_radix_insert()
to fail and handle all the deriving failure of those.

vm_radix-wise introduce a new function called vm_radix_replace(),
which can replace a leaf node, already present, with a new one,
and take into account the possibility, during vm_radix_insert()
allocation, that the operations on the radix trie can recurse.
This means that if operations in vm_radix_insert() recursed
vm_radix_insert() will start from scratch again.

Sponsored by:	EMC / Isilon storage division
Reviewed by:	alc (older version)
Reviewed by:	jeff
Tested by:	pho, scottl

1 files changed, 49 insertions, 27 deletions

diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 74e580f..1e22cd3 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -201,10 +201,12 @@ vm_object_zinit(void *mem, int size, int flags)
 
 	/* These are true for any object that has been freed */
 	object->rtree.rt_root = 0;
+	object->rtree.rt_flags = 0;
 	object->paging_in_progress = 0;
 	object->resident_page_count = 0;
 	object->shadow_count = 0;
 	object->cache.rt_root = 0;
+	object->cache.rt_flags = 0;
 	return (0);
 }
 
@@ -1351,6 +1353,16 @@ retry:
 			VM_OBJECT_WLOCK(new_object);
 			goto retry;
 		}
+
+		/* vm_page_rename() will handle dirty and cache. */
+		if (vm_page_rename(m, new_object, idx)) {
+			VM_OBJECT_WUNLOCK(new_object);
+			VM_OBJECT_WUNLOCK(orig_object);
+			VM_WAIT;
+			VM_OBJECT_WLOCK(orig_object);
+			VM_OBJECT_WLOCK(new_object);
+			goto retry;
+		}
 #if VM_NRESERVLEVEL > 0
 		/*
 		 * If some of the reservation's allocated pages remain with
@@ -1366,10 +1378,6 @@ retry:
 		 */
 		vm_reserv_rename(m, new_object, orig_object, offidxstart);
 #endif
-		vm_page_lock(m);
-		vm_page_rename(m, new_object, idx);
-		vm_page_unlock(m);
-		/* page automatically made dirty by rename and cache handled */
 		if (orig_object->type == OBJT_SWAP)
 			vm_page_xbusy(m);
 	}
@@ -1525,21 +1533,14 @@ vm_object_backing_scan(vm_object_t object, int op)
 			    ("vm_object_backing_scan: object mismatch")
 			);
 
-			/*
-			 * Destroy any associated swap
-			 */
-			if (backing_object->type == OBJT_SWAP) {
-				swap_pager_freespace(
-				    backing_object, 
-				    p->pindex,
-				    1
-				);
-			}
-
 			if (
 			    p->pindex < backing_offset_index ||
 			    new_pindex >= object->size
 			) {
+				if (backing_object->type == OBJT_SWAP)
+					swap_pager_freespace(backing_object, 
+					    p->pindex, 1);
+
 				/*
 				 * Page is out of the parent object's range, we 
 				 * can simply destroy it. 
@@ -1561,6 +1562,10 @@ vm_object_backing_scan(vm_object_t object, int op)
 			    (op & OBSC_COLLAPSE_NOWAIT) != 0 &&
 			    (pp != NULL && pp->valid == 0)
 			) {
+				if (backing_object->type == OBJT_SWAP)
+					swap_pager_freespace(backing_object, 
+					    p->pindex, 1);
+
 				/*
 				 * The page in the parent is not (yet) valid.
 				 * We don't know anything about the state of
@@ -1579,6 +1584,10 @@ vm_object_backing_scan(vm_object_t object, int op)
 			    pp != NULL ||
 			    vm_pager_has_page(object, new_pindex, NULL, NULL)
 			) {
+				if (backing_object->type == OBJT_SWAP)
+					swap_pager_freespace(backing_object, 
+					    p->pindex, 1);
+
 				/*
 				 * page already exists in parent OR swap exists
 				 * for this location in the parent.  Destroy 
@@ -1598,25 +1607,38 @@ vm_object_backing_scan(vm_object_t object, int op)
 				continue;
 			}
 
-#if VM_NRESERVLEVEL > 0
-			/*
-			 * Rename the reservation.
-			 */
-			vm_reserv_rename(p, object, backing_object,
-			    backing_offset_index);
-#endif
-
 			/*
 			 * Page does not exist in parent, rename the
 			 * page from the backing object to the main object. 
 			 *
 			 * If the page was mapped to a process, it can remain 
 			 * mapped through the rename.
+			 * vm_page_rename() will handle dirty and cache.
+			 */
+			if (vm_page_rename(p, object, new_pindex)) {
+				if (op & OBSC_COLLAPSE_NOWAIT) {
+					p = next;
+					continue;
+				}
+				VM_OBJECT_WLOCK(backing_object);
+				VM_OBJECT_WUNLOCK(object);
+				VM_WAIT;
+				VM_OBJECT_WLOCK(object);
+				VM_OBJECT_WLOCK(backing_object);
+				p = TAILQ_FIRST(&backing_object->memq);
+				continue;
+			}
+			if (backing_object->type == OBJT_SWAP)
+				swap_pager_freespace(backing_object, p->pindex,
+				    1);
+
+#if VM_NRESERVLEVEL > 0
+			/*
+			 * Rename the reservation.
 			 */
-			vm_page_lock(p);
-			vm_page_rename(p, object, new_pindex);
-			vm_page_unlock(p);
-			/* page automatically made dirty by rename */
+			vm_reserv_rename(p, object, backing_object,
+			    backing_offset_index);
+#endif
 		}
 		p = next;
 	}