Introduce a global lock for the vm subsystem (vm_mtx).

vm_mtx does not recurse and is required for most low level
vm operations.

faults can not be taken without holding Giant.

Memory subsystems can now call the base page allocators safely.

Almost all atomic ops were removed as they are covered under the
vm mutex.

Alpha and ia64 now need to catch up to i386's trap handlers.

FFS and NFS have been tested, other filesystems will need minor
changes (grabbing the vm lock when twiddling page properties).

Reviewed (partially) by: jake, jhb

1 files changed, 24 insertions, 8 deletions

diff --git a/sys/vm/vm_zone.c b/sys/vm/vm_zone.c
index 4cddadc..30fadbe 100644
--- a/sys/vm/vm_zone.c
+++ b/sys/vm/vm_zone.c
@@ -137,6 +137,7 @@ zinitna(vm_zone_t z, vm_object_t obj, char *name, int size,
 	 * in pages as needed.
 	 */
 	if (z->zflags & ZONE_INTERRUPT) {
+		int hadvmlock;
 
 		totsize = round_page(z->zsize * nentries);
 		atomic_add_int(&zone_kmem_kvaspace, totsize);
@@ -145,12 +146,17 @@ zinitna(vm_zone_t z, vm_object_t obj, char *name, int size,
 			return 0;
 
 		z->zpagemax = totsize / PAGE_SIZE;
+		hadvmlock = mtx_owned(&vm_mtx);
+		if (!hadvmlock)
+			mtx_lock(&vm_mtx);
 		if (obj == NULL) {
 			z->zobj = vm_object_allocate(OBJT_DEFAULT, z->zpagemax);
 		} else {
 			z->zobj = obj;
 			_vm_object_allocate(OBJT_DEFAULT, z->zpagemax, obj);
 		}
+		if (!hadvmlock)
+			mtx_unlock(&vm_mtx);
 		z->zallocflag = VM_ALLOC_INTERRUPT;
 		z->zmax += nentries;
 	} else {
@@ -262,7 +268,6 @@ _zget(vm_zone_t z)
 	void *item;
 
 	KASSERT(z != NULL, ("invalid zone"));
-	mtx_assert(&z->zmtx, MA_OWNED);
 
 	if (z->zflags & ZONE_INTERRUPT) {
 		item = (char *) z->zkva + z->zpagecount * PAGE_SIZE;
@@ -299,16 +304,13 @@ _zget(vm_zone_t z)
 		 * We can wait, so just do normal map allocation in the appropriate
 		 * map.
 		 */
+		mtx_unlock(&z->zmtx);
 		if (lockstatus(&kernel_map->lock, NULL)) {
-			mtx_unlock(&z->zmtx);
 			item = (void *) kmem_malloc(kmem_map, nbytes, M_WAITOK);
-			mtx_lock(&z->zmtx);
 			if (item != NULL)
 				atomic_add_int(&zone_kmem_pages, z->zalloc);
 		} else {
-			mtx_unlock(&z->zmtx);
 			item = (void *) kmem_alloc(kernel_map, nbytes);
-			mtx_lock(&z->zmtx);
 			if (item != NULL)
 				atomic_add_int(&zone_kern_pages, z->zalloc);
 		}
@@ -318,6 +320,7 @@ _zget(vm_zone_t z)
 			nbytes = 0;
 		}
 		nitems = nbytes / z->zsize;
+		mtx_lock(&z->zmtx);
 	}
 	z->ztotal += nitems;
 
@@ -361,14 +364,17 @@ void *
 zalloc(vm_zone_t z)
 {
 	void *item;
+	int hadvmlock;
 
 	KASSERT(z != NULL, ("invalid zone"));
+	hadvmlock = mtx_owned(&vm_mtx);
+	if (!hadvmlock)
+		mtx_lock(&vm_mtx);
 	mtx_lock(&z->zmtx);
 	
 	if (z->zfreecnt <= z->zfreemin) {
 		item = _zget(z);
-		mtx_unlock(&z->zmtx);
-		return item;
+		goto out;
 	}
 
 	item = z->zitems;
@@ -381,8 +387,11 @@ zalloc(vm_zone_t z)
 
 	z->zfreecnt--;
 	z->znalloc++;
-	
+
+out:	
 	mtx_unlock(&z->zmtx);
+	if (!hadvmlock)
+		mtx_unlock(&vm_mtx);
 	return item;
 }
 
@@ -392,8 +401,13 @@ zalloc(vm_zone_t z)
 void
 zfree(vm_zone_t z, void *item)
 {
+	int hadvmlock;
+
 	KASSERT(z != NULL, ("invalid zone"));
 	KASSERT(item != NULL, ("invalid item"));
+	hadvmlock = mtx_owned(&vm_mtx);
+	if (!hadvmlock)
+		mtx_lock(&vm_mtx);
 	mtx_lock(&z->zmtx);
 	
 	((void **) item)[0] = z->zitems;
@@ -405,6 +419,8 @@ zfree(vm_zone_t z, void *item)
 	z->zitems = item;
 	z->zfreecnt++;
 
+	if (!hadvmlock)
+		mtx_unlock(&vm_mtx);
 	mtx_unlock(&z->zmtx);
 }