Try to stabilize ZFS with regard to memory consumption:

- Allow to shrink ARC down to 16MB (instead of 64MB).
- Set arc_max to 1/2 of kmem_map by default.
- Start freeing things earlier when low memory situation is detected.
- Serialize execution of arc_lowmem().

I decided to setup minimum ZFS memory requirements to 512MB of RAM and 256MB of
kmem_map size. If there is less RAM or kmem_map, a warning will be printed.
World is cruel, be no better. In other words: modern file system requires
modern hardware:)

From ZFS administration guide:

"Currently the minimum amount of memory recommended to install a Solaris
 system is 512 Mbytes. However, for good ZFS performance, at least one
 Gbyte or more of memory is recommended."

8 files changed, 122 insertions, 48 deletions

diff --git a/sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c b/sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c
index 2d6f3e5..a73349f 100644
--- a/sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c
+++ b/sys/cddl/compat/opensolaris/kern/opensolaris_kmem.c
@@ -31,12 +31,19 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
-#include <vm/uma.h>
 #include <sys/kmem.h>
 #include <sys/debug.h>
 #include <sys/mutex.h>
+
+#include <vm/vm_page.h>
+#include <vm/vm_object.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_map.h>
+
+#ifdef KMEM_DEBUG
 #include <sys/queue.h>
 #include <sys/stack.h>
+#endif
 
 #ifdef _KERNEL
 static MALLOC_DEFINE(M_SOLARIS, "solaris", "Solaris");
@@ -82,12 +89,6 @@ zfs_kmem_alloc(size_t size, int kmflags)
 	return (p);
 }
 
-void *
-kmem_zalloc(size_t size, int kmflags)
-{
-	return (kmem_alloc(size, kmflags | M_ZERO));
-}
-
 void
 zfs_kmem_free(void *buf, size_t size __unused)
 {
@@ -107,6 +108,20 @@ zfs_kmem_free(void *buf, size_t size __unused)
 	free(buf, M_SOLARIS);
 }
 
+u_long
+kmem_size(void)
+{
+
+	return ((u_long)vm_kmem_size);
+}
+
+u_long
+kmem_used(void)
+{
+
+	return ((u_long)kmem_map->size);
+}
+
 static int
 kmem_std_constructor(void *mem, int size __unused, void *private, int flags)
 {
diff --git a/sys/cddl/compat/opensolaris/sys/kmem.h b/sys/cddl/compat/opensolaris/sys/kmem.h
index 288ab7a..89dfac9 100644
--- a/sys/cddl/compat/opensolaris/sys/kmem.h
+++ b/sys/cddl/compat/opensolaris/sys/kmem.h
@@ -32,6 +32,7 @@
 #include <sys/param.h>
 #include <sys/proc.h>
 #include <sys/malloc.h>
+
 #include <vm/uma.h>
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
@@ -55,8 +56,9 @@ typedef struct kmem_cache {
 } kmem_cache_t;
 
 void *zfs_kmem_alloc(size_t size, int kmflags);
-void *kmem_zalloc(size_t size, int kmflags);
 void zfs_kmem_free(void *buf, size_t size);
+u_long kmem_size(void);
+u_long kmem_used(void);
 kmem_cache_t *kmem_cache_create(char *name, size_t bufsize, size_t align,
     int (*constructor)(void *, void *, int), void (*destructor)(void *, void *),
     void (*reclaim)(void *) __unused, void *private, vmem_t *vmp, int cflags);
@@ -69,6 +71,7 @@ int kmem_debugging(void);
 void *calloc(size_t n, size_t s);
 
 #define	kmem_alloc(size, kmflags)	zfs_kmem_alloc((size), (kmflags))
+#define	kmem_zalloc(size, kmflags)	zfs_kmem_alloc((size), (kmflags) | M_ZERO)
 #define	kmem_free(buf, size)		zfs_kmem_free((buf), (size))
 
 #endif	/* _OPENSOLARIS_SYS_KMEM_H_ */
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
index 7ec366b..5b37d08 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/arc.c
@@ -1441,7 +1441,7 @@ arc_reclaim_needed(void)
 		return (1);
 #endif
 #else
-	if (kmem_map->size > (vm_kmem_size * 3) / 4)
+	if (kmem_used() > kmem_size() / 2)
 		return (1);
 #endif
 
@@ -2685,16 +2685,20 @@ arc_tempreserve_space(uint64_t tempreserve)
 }
 
 #ifdef _KERNEL
-static eventhandler_tag zfs_event_lowmem = NULL;
+static eventhandler_tag arc_event_lowmem = NULL;
+static kmutex_t arc_lowmem_lock;
 
 static void
-zfs_lowmem(void *arg __unused, int howto __unused)
+arc_lowmem(void *arg __unused, int howto __unused)
 {
 
+	/* Serialize access via arc_lowmem_lock. */
+	mutex_enter(&arc_lowmem_lock);
 	zfs_needfree = 1;
 	cv_signal(&arc_reclaim_thr_cv);
 	while (zfs_needfree)
 		tsleep(&zfs_needfree, 0, "zfs:lowmem", hz / 5);
+	mutex_exit(&arc_lowmem_lock);
 }
 #endif
 
@@ -2703,12 +2707,15 @@ arc_init(void)
 {
 	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
+#ifdef _KERNEL
+	mutex_init(&arc_lowmem_lock, NULL, MUTEX_DEFAULT, NULL);
+#endif
 
 	/* Convert seconds to clock ticks */
 	arc_min_prefetch_lifespan = 1 * hz;
 
 	/* Start out with 1/8 of all memory */
-	arc_c = physmem * PAGESIZE / 8;
+	arc_c = kmem_size() / 8;
 #if 0
 #ifdef _KERNEL
 	/*
@@ -2719,22 +2726,22 @@ arc_init(void)
 	arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
 #endif
 #endif
-	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
-	arc_c_min = MAX(arc_c / 4, 64<<20);
-	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
+	/* set min cache to 1/32 of all memory, or 16MB, whichever is more */
+	arc_c_min = MAX(arc_c / 4, 64<<18);
+	/* set max to 1/2 of all memory, or all but 1GB, whichever is more */
 	if (arc_c * 8 >= 1<<30)
 		arc_c_max = (arc_c * 8) - (1<<30);
 	else
 		arc_c_max = arc_c_min;
-	arc_c_max = MAX(arc_c * 6, arc_c_max);
+	arc_c_max = MAX(arc_c * 4, arc_c_max);
 #ifdef _KERNEL
 	/*
 	 * Allow the tunables to override our calculations if they are
-	 * reasonable (ie. over 64MB)
+	 * reasonable (ie. over 16MB)
 	 */
-	if (zfs_arc_max > 64<<20 && zfs_arc_max < vm_kmem_size)
+	if (zfs_arc_max >= 64<<18 && zfs_arc_max < kmem_size())
 		arc_c_max = zfs_arc_max;
-	if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
+	if (zfs_arc_min >= 64<<18 && zfs_arc_min <= arc_c_max)
 		arc_c_min = zfs_arc_min;
 #endif
 	arc_c = arc_c_max;
@@ -2790,11 +2797,24 @@ arc_init(void)
 	    TS_RUN, minclsyspri);
 
 #ifdef _KERNEL
-	zfs_event_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, zfs_lowmem, NULL,
+	arc_event_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, arc_lowmem, NULL,
 	    EVENTHANDLER_PRI_FIRST);
 #endif
 
 	arc_dead = FALSE;
+
+#ifdef _KERNEL
+	/* Warn about ZFS memory requirements. */
+	if ((physmem * PAGESIZE) < (256 + 128 + 64) * (1 << 20)) {
+		printf("ZFS WARNING: Recomended minimum of RAM size is 512MB, "
+		    "expect unstable behaviour.\n");
+	} else if (kmem_size() < 256 * (1 << 20)) {
+		printf("ZFS WARNING: Recomended minimum of kmem_map size is "
+		    "256MB, expect unstable behaviour.\n");
+		printf("             Consider tunning vm.kmem_size and "
+		    "vm.kmem_size_max in /boot/loader.conf.\n");
+	}
+#endif
 }
 
 void
@@ -2834,7 +2854,8 @@ arc_fini(void)
 	buf_fini();
 
 #ifdef _KERNEL
-	if (zfs_event_lowmem != NULL)
-		EVENTHANDLER_DEREGISTER(vm_lowmem, zfs_event_lowmem);
+	if (arc_event_lowmem != NULL)
+		EVENTHANDLER_DEREGISTER(vm_lowmem, arc_event_lowmem);
+	mutex_destroy(&arc_lowmem_lock);
 #endif
 }
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_context.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_context.h
index 454126f..4deeb3c 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_context.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_context.h
@@ -103,8 +103,6 @@ extern "C" {
 }
 #endif
 
-#define	physmem	(vm_kmem_size / PAGE_SIZE)
-
 extern int zfs_debug_level;
 extern struct mtx zfs_debug_mtx;
 #define	ZFS_LOG(lvl, ...)	do {					\
diff --git a/sys/compat/opensolaris/kern/opensolaris_kmem.c b/sys/compat/opensolaris/kern/opensolaris_kmem.c
index 2d6f3e5..a73349f 100644
--- a/sys/compat/opensolaris/kern/opensolaris_kmem.c
+++ b/sys/compat/opensolaris/kern/opensolaris_kmem.c
@@ -31,12 +31,19 @@ __FBSDID("$FreeBSD$");
 #include <sys/kernel.h>
 #include <sys/systm.h>
 #include <sys/malloc.h>
-#include <vm/uma.h>
 #include <sys/kmem.h>
 #include <sys/debug.h>
 #include <sys/mutex.h>
+
+#include <vm/vm_page.h>
+#include <vm/vm_object.h>
+#include <vm/vm_kern.h>
+#include <vm/vm_map.h>
+
+#ifdef KMEM_DEBUG
 #include <sys/queue.h>
 #include <sys/stack.h>
+#endif
 
 #ifdef _KERNEL
 static MALLOC_DEFINE(M_SOLARIS, "solaris", "Solaris");
@@ -82,12 +89,6 @@ zfs_kmem_alloc(size_t size, int kmflags)
 	return (p);
 }
 
-void *
-kmem_zalloc(size_t size, int kmflags)
-{
-	return (kmem_alloc(size, kmflags | M_ZERO));
-}
-
 void
 zfs_kmem_free(void *buf, size_t size __unused)
 {
@@ -107,6 +108,20 @@ zfs_kmem_free(void *buf, size_t size __unused)
 	free(buf, M_SOLARIS);
 }
 
+u_long
+kmem_size(void)
+{
+
+	return ((u_long)vm_kmem_size);
+}
+
+u_long
+kmem_used(void)
+{
+
+	return ((u_long)kmem_map->size);
+}
+
 static int
 kmem_std_constructor(void *mem, int size __unused, void *private, int flags)
 {
diff --git a/sys/compat/opensolaris/sys/kmem.h b/sys/compat/opensolaris/sys/kmem.h
index 288ab7a..89dfac9 100644
--- a/sys/compat/opensolaris/sys/kmem.h
+++ b/sys/compat/opensolaris/sys/kmem.h
@@ -32,6 +32,7 @@
 #include <sys/param.h>
 #include <sys/proc.h>
 #include <sys/malloc.h>
+
 #include <vm/uma.h>
 #include <vm/vm.h>
 #include <vm/vm_extern.h>
@@ -55,8 +56,9 @@ typedef struct kmem_cache {
 } kmem_cache_t;
 
 void *zfs_kmem_alloc(size_t size, int kmflags);
-void *kmem_zalloc(size_t size, int kmflags);
 void zfs_kmem_free(void *buf, size_t size);
+u_long kmem_size(void);
+u_long kmem_used(void);
 kmem_cache_t *kmem_cache_create(char *name, size_t bufsize, size_t align,
     int (*constructor)(void *, void *, int), void (*destructor)(void *, void *),
     void (*reclaim)(void *) __unused, void *private, vmem_t *vmp, int cflags);
@@ -69,6 +71,7 @@ int kmem_debugging(void);
 void *calloc(size_t n, size_t s);
 
 #define	kmem_alloc(size, kmflags)	zfs_kmem_alloc((size), (kmflags))
+#define	kmem_zalloc(size, kmflags)	zfs_kmem_alloc((size), (kmflags) | M_ZERO)
 #define	kmem_free(buf, size)		zfs_kmem_free((buf), (size))
 
 #endif	/* _OPENSOLARIS_SYS_KMEM_H_ */
diff --git a/sys/contrib/opensolaris/uts/common/fs/zfs/arc.c b/sys/contrib/opensolaris/uts/common/fs/zfs/arc.c
index 7ec366b..5b37d08 100644
--- a/sys/contrib/opensolaris/uts/common/fs/zfs/arc.c
+++ b/sys/contrib/opensolaris/uts/common/fs/zfs/arc.c
@@ -1441,7 +1441,7 @@ arc_reclaim_needed(void)
 		return (1);
 #endif
 #else
-	if (kmem_map->size > (vm_kmem_size * 3) / 4)
+	if (kmem_used() > kmem_size() / 2)
 		return (1);
 #endif
 
@@ -2685,16 +2685,20 @@ arc_tempreserve_space(uint64_t tempreserve)
 }
 
 #ifdef _KERNEL
-static eventhandler_tag zfs_event_lowmem = NULL;
+static eventhandler_tag arc_event_lowmem = NULL;
+static kmutex_t arc_lowmem_lock;
 
 static void
-zfs_lowmem(void *arg __unused, int howto __unused)
+arc_lowmem(void *arg __unused, int howto __unused)
 {
 
+	/* Serialize access via arc_lowmem_lock. */
+	mutex_enter(&arc_lowmem_lock);
 	zfs_needfree = 1;
 	cv_signal(&arc_reclaim_thr_cv);
 	while (zfs_needfree)
 		tsleep(&zfs_needfree, 0, "zfs:lowmem", hz / 5);
+	mutex_exit(&arc_lowmem_lock);
 }
 #endif
 
@@ -2703,12 +2707,15 @@ arc_init(void)
 {
 	mutex_init(&arc_reclaim_thr_lock, NULL, MUTEX_DEFAULT, NULL);
 	cv_init(&arc_reclaim_thr_cv, NULL, CV_DEFAULT, NULL);
+#ifdef _KERNEL
+	mutex_init(&arc_lowmem_lock, NULL, MUTEX_DEFAULT, NULL);
+#endif
 
 	/* Convert seconds to clock ticks */
 	arc_min_prefetch_lifespan = 1 * hz;
 
 	/* Start out with 1/8 of all memory */
-	arc_c = physmem * PAGESIZE / 8;
+	arc_c = kmem_size() / 8;
 #if 0
 #ifdef _KERNEL
 	/*
@@ -2719,22 +2726,22 @@ arc_init(void)
 	arc_c = MIN(arc_c, vmem_size(heap_arena, VMEM_ALLOC | VMEM_FREE) / 8);
 #endif
 #endif
-	/* set min cache to 1/32 of all memory, or 64MB, whichever is more */
-	arc_c_min = MAX(arc_c / 4, 64<<20);
-	/* set max to 3/4 of all memory, or all but 1GB, whichever is more */
+	/* set min cache to 1/32 of all memory, or 16MB, whichever is more */
+	arc_c_min = MAX(arc_c / 4, 64<<18);
+	/* set max to 1/2 of all memory, or all but 1GB, whichever is more */
 	if (arc_c * 8 >= 1<<30)
 		arc_c_max = (arc_c * 8) - (1<<30);
 	else
 		arc_c_max = arc_c_min;
-	arc_c_max = MAX(arc_c * 6, arc_c_max);
+	arc_c_max = MAX(arc_c * 4, arc_c_max);
 #ifdef _KERNEL
 	/*
 	 * Allow the tunables to override our calculations if they are
-	 * reasonable (ie. over 64MB)
+	 * reasonable (ie. over 16MB)
 	 */
-	if (zfs_arc_max > 64<<20 && zfs_arc_max < vm_kmem_size)
+	if (zfs_arc_max >= 64<<18 && zfs_arc_max < kmem_size())
 		arc_c_max = zfs_arc_max;
-	if (zfs_arc_min > 64<<20 && zfs_arc_min <= arc_c_max)
+	if (zfs_arc_min >= 64<<18 && zfs_arc_min <= arc_c_max)
 		arc_c_min = zfs_arc_min;
 #endif
 	arc_c = arc_c_max;
@@ -2790,11 +2797,24 @@ arc_init(void)
 	    TS_RUN, minclsyspri);
 
 #ifdef _KERNEL
-	zfs_event_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, zfs_lowmem, NULL,
+	arc_event_lowmem = EVENTHANDLER_REGISTER(vm_lowmem, arc_lowmem, NULL,
 	    EVENTHANDLER_PRI_FIRST);
 #endif
 
 	arc_dead = FALSE;
+
+#ifdef _KERNEL
+	/* Warn about ZFS memory requirements. */
+	if ((physmem * PAGESIZE) < (256 + 128 + 64) * (1 << 20)) {
+		printf("ZFS WARNING: Recomended minimum of RAM size is 512MB, "
+		    "expect unstable behaviour.\n");
+	} else if (kmem_size() < 256 * (1 << 20)) {
+		printf("ZFS WARNING: Recomended minimum of kmem_map size is "
+		    "256MB, expect unstable behaviour.\n");
+		printf("             Consider tunning vm.kmem_size and "
+		    "vm.kmem_size_max in /boot/loader.conf.\n");
+	}
+#endif
 }
 
 void
@@ -2834,7 +2854,8 @@ arc_fini(void)
 	buf_fini();
 
 #ifdef _KERNEL
-	if (zfs_event_lowmem != NULL)
-		EVENTHANDLER_DEREGISTER(vm_lowmem, zfs_event_lowmem);
+	if (arc_event_lowmem != NULL)
+		EVENTHANDLER_DEREGISTER(vm_lowmem, arc_event_lowmem);
+	mutex_destroy(&arc_lowmem_lock);
 #endif
 }
diff --git a/sys/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_context.h b/sys/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_context.h
index 454126f..4deeb3c 100644
--- a/sys/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_context.h
+++ b/sys/contrib/opensolaris/uts/common/fs/zfs/sys/zfs_context.h
@@ -103,8 +103,6 @@ extern "C" {
 }
 #endif
 
-#define	physmem	(vm_kmem_size / PAGE_SIZE)
-
 extern int zfs_debug_level;
 extern struct mtx zfs_debug_mtx;
 #define	ZFS_LOG(lvl, ...)	do {					\