Defer the vrele() on a jail's root vnode reference from prison_free()

to a new prison_complete() task run by a task queue.  This removes
a requirement for grabbing Giant in crfree().  Embed the 'struct task'
in 'struct prison' so that we don't have to allocate memory from
prison_free() (which means we also defer the FREE()).

With this change, I believe grabbing Giant from crfree() can now be
removed, but need to check the uidinfo code paths.

To avoid header pollution, move the definition of 'struct task'
to _task.h, and recursively include from taskqueue.h and jail.h; much
preferably to all files including jail.h picking up a requirement to
include taskqueue.h.

Bumped into by:	sam
Reviewed by:	bde, tjr

4 files changed, 80 insertions, 23 deletions

diff --git a/sys/kern/kern_jail.c b/sys/kern/kern_jail.c
index 2241dd6..79be249 100644
--- a/sys/kern/kern_jail.c
+++ b/sys/kern/kern_jail.c
@@ -18,6 +18,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/sysproto.h>
 #include <sys/malloc.h>
 #include <sys/proc.h>
+#include <sys/taskqueue.h>
 #include <sys/jail.h>
 #include <sys/lock.h>
 #include <sys/mutex.h>
@@ -60,6 +61,7 @@ int	lastprid = 0;
 int	prisoncount = 0;
 
 static void		 init_prison(void *);
+static void		 prison_complete(void *context, int pending);
 static struct prison	*prison_find(int);
 static int		 sysctl_jail_list(SYSCTL_HANDLER_ARGS);
 
@@ -255,7 +257,6 @@ void
 prison_free(struct prison *pr)
 {
 
-	mtx_assert(&Giant, MA_OWNED);
 	mtx_lock(&allprison_mtx);
 	mtx_lock(&pr->pr_mtx);
 	pr->pr_ref--;
@@ -264,17 +265,32 @@ prison_free(struct prison *pr)
 		mtx_unlock(&pr->pr_mtx);
 		prisoncount--;
 		mtx_unlock(&allprison_mtx);
-		vrele(pr->pr_root);
-		mtx_destroy(&pr->pr_mtx);
-		if (pr->pr_linux != NULL)
-			FREE(pr->pr_linux, M_PRISON);
-		FREE(pr, M_PRISON);
+
+		TASK_INIT(&pr->pr_task, 0, prison_complete, pr);
+		taskqueue_enqueue(taskqueue_swi, &pr->pr_task);
 		return;
 	}
 	mtx_unlock(&pr->pr_mtx);
 	mtx_unlock(&allprison_mtx);
 }
 
+static void
+prison_complete(void *context, int pending)
+{
+	struct prison *pr;
+
+	pr = (struct prison *)context;
+
+	mtx_lock(&Giant);
+	vrele(pr->pr_root);
+	mtx_unlock(&Giant);
+
+	mtx_destroy(&pr->pr_mtx);
+	if (pr->pr_linux != NULL)
+		FREE(pr->pr_linux, M_PRISON);
+	FREE(pr, M_PRISON);
+}
+
 void
 prison_hold(struct prison *pr)
 {
diff --git a/sys/sys/_task.h b/sys/sys/_task.h
new file mode 100644
index 0000000..c5c5a68
--- /dev/null
+++ b/sys/sys/_task.h
@@ -0,0 +1,54 @@
+/*-
+ * Copyright (c) 2000 Doug Rabson
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#ifndef _SYS__TASK_H_
+#define _SYS__TASK_H_
+
+#ifndef _KERNEL
+#error "no user-servicable parts inside"
+#endif
+
+#include <sys/queue.h>
+
+/*
+ * Each task includes a function which is called from
+ * taskqueue_run().  The first argument is taken from the 'ta_context'
+ * field of struct task and the second argument is a count of how many
+ * times the task was enqueued before the call to taskqueue_run().
+ */
+typedef void task_fn_t(void *context, int pending);
+
+struct task {
+	STAILQ_ENTRY(task) ta_link;	/* link for queue */
+	int	ta_pending;		/* count times queued */
+	int	ta_priority;		/* priority of task in queue */
+	task_fn_t *ta_func;		/* task handler */
+	void	*ta_context;		/* argument for handler */
+};
+
+#endif /* !_SYS__TASK_H_ */
diff --git a/sys/sys/jail.h b/sys/sys/jail.h
index fbc8ba93..9779363 100644
--- a/sys/sys/jail.h
+++ b/sys/sys/jail.h
@@ -39,6 +39,7 @@ int jail_attach(int);
 #include <sys/queue.h>
 #include <sys/_lock.h>
 #include <sys/_mutex.h>
+#include <sys/_task.h>
 
 #define JAIL_MAX	999999
 
@@ -56,8 +57,8 @@ MALLOC_DECLARE(M_PRISON);
  *   (p) locked by pr_mutex
  *   (c) set only during creation before the structure is shared, no mutex
  *       required to read
+ *   (d) set only during destruction of jail, no mutex needed
  */
-struct mtx;
 struct prison {
 	LIST_ENTRY(prison) pr_list;			/* (a) all prisons */
 	int		 pr_id;				/* (c) prison id */
@@ -68,6 +69,7 @@ struct prison {
 	u_int32_t	 pr_ip;				/* (c) ip addr host */
 	void		*pr_linux;			/* (p) linux abi */
 	int		 pr_securelevel;		/* (p) securelevel */
+	struct task	 pr_task;			/* (d) destroy task */
 	struct mtx	 pr_mtx;
 };
 
diff --git a/sys/sys/taskqueue.h b/sys/sys/taskqueue.h
index 9695b35..d35eb10 100644
--- a/sys/sys/taskqueue.h
+++ b/sys/sys/taskqueue.h
@@ -34,18 +34,11 @@
 #endif
 
 #include <sys/queue.h>
+#include <sys/_task.h>
 
 struct taskqueue;
 
 /*
- * Each task includes a function which is called from
- * taskqueue_run().  The first argument is taken from the 'ta_context'
- * field of struct task and the second argument is a count of how many
- * times the task was enqueued before the call to taskqueue_run().
- */
-typedef void task_fn_t(void *context, int pending);
-
-/*
  * A notification callback function which is called from
  * taskqueue_enqueue().  The context argument is given in the call to
  * taskqueue_create().  This function would normally be used to allow the
@@ -54,14 +47,6 @@ typedef void task_fn_t(void *context, int pending);
  */
 typedef void (*taskqueue_enqueue_fn)(void *context);
 
-struct task {
-	STAILQ_ENTRY(task) ta_link;	/* link for queue */
-	int	ta_pending;		/* count times queued */
-	int	ta_priority;		/* priority of task in queue */
-	task_fn_t *ta_func;		/* task handler */
-	void	*ta_context;		/* argument for handler */
-};
-
 struct taskqueue *taskqueue_create(const char *name, int mflags,
 				    taskqueue_enqueue_fn enqueue,
 				    void *context);