Redo r292484. Embed task(9) into zone, so that uz_maxaction is called

in a context that can sleep, allowing consumers of the KPI to run their
drain routines without any extra measures.

Discussed with:	jtl

9 files changed, 22 insertions, 88 deletions

diff --git a/sys/amd64/amd64/uma_machdep.c b/sys/amd64/amd64/uma_machdep.c
index db566ae..fc343ca 100644
--- a/sys/amd64/amd64/uma_machdep.c
+++ b/sys/amd64/amd64/uma_machdep.c
@@ -32,6 +32,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/malloc.h>
 #include <sys/mutex.h>
 #include <sys/systm.h>
+#include <sys/taskqueue.h>
 #include <vm/vm.h>
 #include <vm/vm_page.h>
 #include <vm/vm_pageout.h>
diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c
index e7c81d6..f7c67f2 100644
--- a/sys/kern/kern_malloc.c
+++ b/sys/kern/kern_malloc.c
@@ -59,6 +59,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/sbuf.h>
 #include <sys/sysctl.h>
+#include <sys/taskqueue.h>
 #include <sys/time.h>
 #include <sys/vmem.h>
 
diff --git a/sys/kern/kern_mbuf.c b/sys/kern/kern_mbuf.c
index 779c62a..5596439 100644
--- a/sys/kern/kern_mbuf.c
+++ b/sys/kern/kern_mbuf.c
@@ -274,12 +274,6 @@ uma_zone_t	zone_jumbo16;
 uma_zone_t	zone_ext_refcnt;
 
 /*
- * Callout to assist us in freeing mbufs.
- */
-static struct callout	mb_reclaim_callout;
-static struct mtx	mb_reclaim_callout_mtx;
-
-/*
  * Local prototypes.
  */
 static int	mb_ctor_mbuf(void *, int, void *, int);
@@ -291,9 +285,8 @@ static void	mb_dtor_pack(void *, int, void *);
 static int	mb_zinit_pack(void *, int, int);
 static void	mb_zfini_pack(void *, int);
 
-static void	mb_reclaim(void *);
+static void	mb_reclaim(uma_zone_t, int);
 static void    *mbuf_jumbo_alloc(uma_zone_t, vm_size_t, uint8_t *, int);
-static void	mb_maxaction(uma_zone_t);
 
 /* Ensure that MSIZE is a power of 2. */
 CTASSERT((((MSIZE - 1) ^ MSIZE) + 1) >> 1 == MSIZE);
@@ -319,7 +312,7 @@ mbuf_init(void *dummy)
 	if (nmbufs > 0)
 		nmbufs = uma_zone_set_max(zone_mbuf, nmbufs);
 	uma_zone_set_warning(zone_mbuf, "kern.ipc.nmbufs limit reached");
-	uma_zone_set_maxaction(zone_mbuf, mb_maxaction);
+	uma_zone_set_maxaction(zone_mbuf, mb_reclaim);
 
 	zone_clust = uma_zcreate(MBUF_CLUSTER_MEM_NAME, MCLBYTES,
 	    mb_ctor_clust, mb_dtor_clust,
@@ -332,7 +325,7 @@ mbuf_init(void *dummy)
 	if (nmbclusters > 0)
 		nmbclusters = uma_zone_set_max(zone_clust, nmbclusters);
 	uma_zone_set_warning(zone_clust, "kern.ipc.nmbclusters limit reached");
-	uma_zone_set_maxaction(zone_clust, mb_maxaction);
+	uma_zone_set_maxaction(zone_clust, mb_reclaim);
 
 	zone_pack = uma_zsecond_create(MBUF_PACKET_MEM_NAME, mb_ctor_pack,
 	    mb_dtor_pack, mb_zinit_pack, mb_zfini_pack, zone_mbuf);
@@ -349,7 +342,7 @@ mbuf_init(void *dummy)
 	if (nmbjumbop > 0)
 		nmbjumbop = uma_zone_set_max(zone_jumbop, nmbjumbop);
 	uma_zone_set_warning(zone_jumbop, "kern.ipc.nmbjumbop limit reached");
-	uma_zone_set_maxaction(zone_jumbop, mb_maxaction);
+	uma_zone_set_maxaction(zone_jumbop, mb_reclaim);
 
 	zone_jumbo9 = uma_zcreate(MBUF_JUMBO9_MEM_NAME, MJUM9BYTES,
 	    mb_ctor_clust, mb_dtor_clust,
@@ -363,7 +356,7 @@ mbuf_init(void *dummy)
 	if (nmbjumbo9 > 0)
 		nmbjumbo9 = uma_zone_set_max(zone_jumbo9, nmbjumbo9);
 	uma_zone_set_warning(zone_jumbo9, "kern.ipc.nmbjumbo9 limit reached");
-	uma_zone_set_maxaction(zone_jumbo9, mb_maxaction);
+	uma_zone_set_maxaction(zone_jumbo9, mb_reclaim);
 
 	zone_jumbo16 = uma_zcreate(MBUF_JUMBO16_MEM_NAME, MJUM16BYTES,
 	    mb_ctor_clust, mb_dtor_clust,
@@ -377,20 +370,13 @@ mbuf_init(void *dummy)
 	if (nmbjumbo16 > 0)
 		nmbjumbo16 = uma_zone_set_max(zone_jumbo16, nmbjumbo16);
 	uma_zone_set_warning(zone_jumbo16, "kern.ipc.nmbjumbo16 limit reached");
-	uma_zone_set_maxaction(zone_jumbo16, mb_maxaction);
+	uma_zone_set_maxaction(zone_jumbo16, mb_reclaim);
 
 	zone_ext_refcnt = uma_zcreate(MBUF_EXTREFCNT_MEM_NAME, sizeof(u_int),
 	    NULL, NULL,
 	    NULL, NULL,
 	    UMA_ALIGN_PTR, UMA_ZONE_ZINIT);
 
-	/* uma_prealloc() goes here... */
-
-	/* Initialize the mb_reclaim() callout. */
-	mtx_init(&mb_reclaim_callout_mtx, "mb_reclaim_callout_mtx", NULL,
-	    MTX_DEF);
-	callout_init(&mb_reclaim_callout, 1);
-
 	/*
 	 * Hook event handler for low-memory situation, used to
 	 * drain protocols and push data back to the caches (UMA
@@ -677,81 +663,23 @@ m_pkthdr_init(struct mbuf *m, int how)
 }
 
 /*
- * This is the protocol drain routine.
+ * This is the protocol drain routine.  Called by UMA whenever any of the
+ * mbuf zones is closed to its limit.
  *
  * No locks should be held when this is called.  The drain routines have to
  * presently acquire some locks which raises the possibility of lock order
  * reversal.
  */
 static void
-mb_reclaim(void *junk)
+mb_reclaim(uma_zone_t zone __unused, int pending __unused)
 {
 	struct domain *dp;
 	struct protosw *pr;
 
-	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL,
-	    "mb_reclaim()");
+	WITNESS_WARN(WARN_GIANTOK | WARN_SLEEPOK | WARN_PANIC, NULL, __func__);
 
 	for (dp = domains; dp != NULL; dp = dp->dom_next)
 		for (pr = dp->dom_protosw; pr < dp->dom_protoswNPROTOSW; pr++)
 			if (pr->pr_drain != NULL)
 				(*pr->pr_drain)();
 }
-
-/*
- * This is the function called by the mb_reclaim_callout, which is
- * used when we hit the maximum for a zone.
- *
- * (See mb_maxaction() below.)
- */
-static void
-mb_reclaim_timer(void *junk __unused)
-{
-
-	mtx_lock(&mb_reclaim_callout_mtx);
-
-	/*
-	 * Avoid running this function extra times by skipping this invocation
-	 * if the callout has already been rescheduled.
-	 */
-	if (callout_pending(&mb_reclaim_callout) ||
-	    !callout_active(&mb_reclaim_callout)) {
-		mtx_unlock(&mb_reclaim_callout_mtx);
-		return;
-	}
-	mtx_unlock(&mb_reclaim_callout_mtx);
-
-	mb_reclaim(NULL);
-
-	mtx_lock(&mb_reclaim_callout_mtx);
-	callout_deactivate(&mb_reclaim_callout);
-	mtx_unlock(&mb_reclaim_callout_mtx);
-}
-
-/*
- * This function is called when we hit the maximum for a zone.
- *
- * At that point, we want to call the protocol drain routine to free up some
- * mbufs. However, we will use the callout routines to schedule this to
- * occur in another thread. (The thread calling this function holds the
- * zone lock.)
- */
-static void
-mb_maxaction(uma_zone_t zone __unused)
-{
-
-	/*
-	 * If we can't immediately obtain the lock, either the callout
-	 * is currently running, or another thread is scheduling the
-	 * callout.
-	 */
-	if (!mtx_trylock(&mb_reclaim_callout_mtx))
-		return;
-
-	/* If not already scheduled/running, schedule the callout. */
-	if (!callout_active(&mb_reclaim_callout)) {
-		callout_reset(&mb_reclaim_callout, 1, mb_reclaim_timer, NULL);
-	}
-
-	mtx_unlock(&mb_reclaim_callout_mtx);
-}
diff --git a/sys/vm/memguard.c b/sys/vm/memguard.c
index d4efc2b..fb70a41 100644
--- a/sys/vm/memguard.c
+++ b/sys/vm/memguard.c
@@ -48,6 +48,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/mutex.h>
 #include <sys/malloc.h>
 #include <sys/sysctl.h>
+#include <sys/taskqueue.h>
 #include <sys/vmem.h>
 
 #include <vm/vm.h>
diff --git a/sys/vm/uma.h b/sys/vm/uma.h
index d218e60..21c6121 100644
--- a/sys/vm/uma.h
+++ b/sys/vm/uma.h
@@ -530,7 +530,7 @@ void uma_zone_set_warning(uma_zone_t zone, const char *warning);
  * Returns:
  *	Nothing
  */
-typedef void (*uma_maxaction_t)(uma_zone_t);
+typedef void (*uma_maxaction_t)(uma_zone_t, int);
 void uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t);
 
 /*
diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index 89ed213..0d45046 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -78,6 +78,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sbuf.h>
 #include <sys/sched.h>
 #include <sys/smp.h>
+#include <sys/taskqueue.h>
 #include <sys/vmmeter.h>
 
 #include <vm/vm.h>
@@ -439,8 +440,9 @@ zone_log_warning(uma_zone_t zone)
 static inline void
 zone_maxaction(uma_zone_t zone)
 {
-	if (zone->uz_maxaction)
-		(*zone->uz_maxaction)(zone);
+
+	if (zone->uz_maxaction.ta_func != NULL)
+		taskqueue_enqueue(taskqueue_thread, &zone->uz_maxaction);
 }
 
 static void
@@ -1590,7 +1592,6 @@ zone_ctor(void *mem, int size, void *udata, int flags)
 	zone->uz_flags = 0;
 	zone->uz_warning = NULL;
 	timevalclear(&zone->uz_ratecheck);
-	zone->uz_maxaction = NULL;
 	keg = arg->keg;
 
 	ZONE_LOCK_INIT(zone, (arg->flags & UMA_ZONE_MTXCLASS));
@@ -3027,7 +3028,7 @@ uma_zone_set_maxaction(uma_zone_t zone, uma_maxaction_t maxaction)
 {
 
 	ZONE_LOCK(zone);
-	zone->uz_maxaction = maxaction;
+	TASK_INIT(&zone->uz_maxaction, 0, (task_fn_t *)maxaction, zone);
 	ZONE_UNLOCK(zone);
 }
 
diff --git a/sys/vm/uma_dbg.c b/sys/vm/uma_dbg.c
index dcef5c9..7bf06d3 100644
--- a/sys/vm/uma_dbg.c
+++ b/sys/vm/uma_dbg.c
@@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/lock.h>
 #include <sys/mutex.h>
 #include <sys/malloc.h>
+#include <sys/taskqueue.h>
 
 #include <vm/vm.h>
 #include <vm/vm_object.h>
diff --git a/sys/vm/uma_int.h b/sys/vm/uma_int.h
index 5d7ecd3..1b0d5d5 100644
--- a/sys/vm/uma_int.h
+++ b/sys/vm/uma_int.h
@@ -307,7 +307,7 @@ struct uma_zone {
 	const char	*uz_warning;	/* Warning to print on failure */
 	struct timeval	uz_ratecheck;	/* Warnings rate-limiting */
 
-	uma_maxaction_t	uz_maxaction;	/* Function to run when at limit */
+	struct task	uz_maxaction;	/* Task to run when at limit */
 
 	/*
 	 * This HAS to be the last item because we adjust the zone size
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index d27eb2d..b510696 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -102,6 +102,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysctl.h>
 #include <sys/vmmeter.h>
 #include <sys/vnode.h>
+#include <sys/taskqueue.h>
 
 #include <vm/vm.h>
 #include <vm/pmap.h>