Add locking to taskqueues. There is one mutex per task, one mutex per

queue, and a mutex to protect the global list of taskqueues.  The only
visible change is that a TASK_DESTROY() macro has been added to mirror
the TASK_INIT() macro to destroy a task before it is free'd.

Submitted by:	Andrew Reiter <awr@watson.org>

6 files changed, 110 insertions, 44 deletions

diff --git a/sys/dev/aac/aac.c b/sys/dev/aac/aac.c
index 4eef4e7..535a783 100644
--- a/sys/dev/aac/aac.c
+++ b/sys/dev/aac/aac.c
@@ -434,6 +434,9 @@ aac_free(struct aac_softc *sc)
 	if (sc->aac_regs_resource != NULL)
 		bus_release_resource(sc->aac_dev, SYS_RES_MEMORY,
 				     sc->aac_regs_rid, sc->aac_regs_resource);
+#if __FreeBSD_version >= 500005
+	TASK_DESTROY(&sc->aac_task_complete);
+#endif
 }
 
 /*
diff --git a/sys/dev/acpica/Osd/OsdSchedule.c b/sys/dev/acpica/Osd/OsdSchedule.c
index 8a952df..6dcc521 100644
--- a/sys/dev/acpica/Osd/OsdSchedule.c
+++ b/sys/dev/acpica/Osd/OsdSchedule.c
@@ -113,6 +113,7 @@ AcpiOsExecuteQueue(void *arg, int pending)
     Function = (OSD_EXECUTION_CALLBACK)at->at_function;
     Context = at->at_context;
 
+    TASK_DESTROY(at);
     free(at, M_ACPITASK);
 
     Function(Context);
diff --git a/sys/dev/amr/amr.c b/sys/dev/amr/amr.c
index 2b5eca1..1e928c1 100644
--- a/sys/dev/amr/amr.c
+++ b/sys/dev/amr/amr.c
@@ -320,6 +320,10 @@ amr_free(struct amr_softc *sc)
 	TAILQ_REMOVE(&sc->amr_cmd_clusters, acc, acc_link);
 	amr_freecmd_cluster(acc);
     }
+
+#if __FreeBSD_version >= 500005
+    TASK_DESTROY(&sc->amr_task_complete);
+#endif
 }
 
 /*******************************************************************************
diff --git a/sys/dev/mly/mly.c b/sys/dev/mly/mly.c
index e7b7a5c..06115f3 100644
--- a/sys/dev/mly/mly.c
+++ b/sys/dev/mly/mly.c
@@ -732,6 +732,10 @@ mly_free(struct mly_softc *sc)
     /* release the register window mapping */
     if (sc->mly_regs_resource != NULL)
 	bus_release_resource(sc->mly_dev, SYS_RES_MEMORY, sc->mly_regs_rid, sc->mly_regs_resource);
+
+#if __FreeBSD_version >= 500005
+    TASK_DESTROY(&sc->mly_task_complete);
+#endif
 }
 
 /********************************************************************************
diff --git a/sys/kern/subr_taskqueue.c b/sys/kern/subr_taskqueue.c
index 2c05cde..e75a0c2 100644
--- a/sys/kern/subr_taskqueue.c
+++ b/sys/kern/subr_taskqueue.c
@@ -27,19 +27,21 @@
  */
 
 #include <sys/param.h>
-#include <sys/bus.h>
-#include <sys/queue.h>
 #include <sys/systm.h>
+#include <sys/bus.h>
 #include <sys/kernel.h>
-#include <sys/taskqueue.h>
+#include <sys/lock.h>
 #include <sys/interrupt.h>
 #include <sys/malloc.h>
+#include <sys/mutex.h>
+#include <sys/taskqueue.h>
 
 static MALLOC_DEFINE(M_TASKQUEUE, "taskqueue", "Task Queues");
 
 static STAILQ_HEAD(taskqueue_list, taskqueue) taskqueue_queues;
 
 static void	*taskqueue_ih;
+static struct mtx taskqueue_queues_mutex;
 
 struct taskqueue {
 	STAILQ_ENTRY(taskqueue)	tq_link;
@@ -48,32 +50,63 @@ struct taskqueue {
 	taskqueue_enqueue_fn	tq_enqueue;
 	void			*tq_context;
 	int			tq_draining;
+	struct mtx		tq_mutex;
 };
 
+static void	init_taskqueue_list(void *data);
+
+static void
+init_taskqueue_list(void *data __unused)
+{
+
+	mtx_init(&taskqueue_queues_mutex, "taskqueue list", MTX_DEF);
+	STAILQ_INIT(&taskqueue_queues);
+}
+SYSINIT(taskqueue_list, SI_SUB_INTRINSIC, SI_ORDER_ANY, init_taskqueue_list,
+    NULL);
+
+void
+task_init(struct task *task, int priority, task_fn_t *func, void *context)
+{
+
+	KASSERT(task != NULL, ("task == NULL"));
+
+	mtx_init(&task->ta_mutex, "task", MTX_DEF);
+	mtx_lock(&task->ta_mutex);
+	task->ta_pending = 0;
+	task->ta_priority = priority;
+	task->ta_func = func;
+	task->ta_context = context;
+	mtx_unlock(&task->ta_mutex);
+}
+
+void
+task_destroy(struct task *task)
+{
+
+	mtx_destroy(&task->ta_mutex);
+}
+
 struct taskqueue *
 taskqueue_create(const char *name, int mflags,
 		 taskqueue_enqueue_fn enqueue, void *context)
 {
 	struct taskqueue *queue;
-	static int once = 1;
-	int s;
 
-	queue = malloc(sizeof(struct taskqueue), M_TASKQUEUE, mflags);
+	queue = malloc(sizeof(struct taskqueue), M_TASKQUEUE, mflags | M_ZERO);
 	if (!queue)
 		return 0;
+
 	STAILQ_INIT(&queue->tq_queue);
 	queue->tq_name = name;
 	queue->tq_enqueue = enqueue;
 	queue->tq_context = context;
 	queue->tq_draining = 0;
+	mtx_init(&queue->tq_mutex, "taskqueue", MTX_DEF);
 
-	s = splhigh();
-	if (once) {
-		STAILQ_INIT(&taskqueue_queues);
-		once = 0;
-	}
+	mtx_lock(&taskqueue_queues_mutex);
 	STAILQ_INSERT_TAIL(&taskqueue_queues, queue, tq_link);
-	splx(s);
+	mtx_unlock(&taskqueue_queues_mutex);
 
 	return queue;
 }
@@ -81,32 +114,39 @@ taskqueue_create(const char *name, int mflags,
 void
 taskqueue_free(struct taskqueue *queue)
 {
-	int s = splhigh();
+
+	mtx_lock(&queue->tq_mutex);
 	queue->tq_draining = 1;
-	splx(s);
+	mtx_unlock(&queue->tq_mutex);
 
 	taskqueue_run(queue);
 
-	s = splhigh();
+	mtx_lock(&taskqueue_queues_mutex);
 	STAILQ_REMOVE(&taskqueue_queues, queue, taskqueue, tq_link);
-	splx(s);
+	mtx_unlock(&taskqueue_queues_mutex);
 
+	mtx_destroy(&queue->tq_mutex);
 	free(queue, M_TASKQUEUE);
 }
 
+/*
+ * Returns with the taskqueue locked.
+ */
 struct taskqueue *
 taskqueue_find(const char *name)
 {
 	struct taskqueue *queue;
-	int s;
 
-	s = splhigh();
-	STAILQ_FOREACH(queue, &taskqueue_queues, tq_link)
+	mtx_lock(&taskqueue_queues_mutex);
+	STAILQ_FOREACH(queue, &taskqueue_queues, tq_link) {
+		mtx_lock(&queue->tq_mutex);
 		if (!strcmp(queue->tq_name, name)) {
-			splx(s);
+			mtx_unlock(&taskqueue_queues_mutex);
 			return queue;
 		}
-	splx(s);
+		mtx_unlock(&queue->tq_mutex);
+	}
+	mtx_unlock(&taskqueue_queues_mutex);
 	return 0;
 }
 
@@ -116,22 +156,23 @@ taskqueue_enqueue(struct taskqueue *queue, struct task *task)
 	struct task *ins;
 	struct task *prev;
 
-	int s = splhigh();
-
 	/*
 	 * Don't allow new tasks on a queue which is being freed.
 	 */
+	mtx_lock(&queue->tq_mutex);
 	if (queue->tq_draining) {
-		splx(s);
+		mtx_unlock(&queue->tq_mutex);
 		return EPIPE;
 	}
 
 	/*
 	 * Count multiple enqueues.
 	 */
+	mtx_lock(&task->ta_mutex);
 	if (task->ta_pending) {
 		task->ta_pending++;
-		splx(s);
+		mtx_unlock(&task->ta_mutex);
+		mtx_unlock(&queue->tq_mutex);
 		return 0;
 	}
 
@@ -155,38 +196,43 @@ taskqueue_enqueue(struct taskqueue *queue, struct task *task)
 	}
 
 	task->ta_pending = 1;
+	mtx_unlock(&task->ta_mutex);
+
 	if (queue->tq_enqueue)
 		queue->tq_enqueue(queue->tq_context);
-
-	splx(s);
-
+	mtx_unlock(&queue->tq_mutex);
 	return 0;
 }
 
 void
 taskqueue_run(struct taskqueue *queue)
 {
-	int s;
 	struct task *task;
+	task_fn_t *saved_func;
+	void *arg;
 	int pending;
 
-	s = splhigh();
+	mtx_lock(&queue->tq_mutex);
 	while (STAILQ_FIRST(&queue->tq_queue)) {
 		/*
 		 * Carefully remove the first task from the queue and
 		 * zero its pending count.
 		 */
 		task = STAILQ_FIRST(&queue->tq_queue);
+		mtx_lock(&task->ta_mutex);
 		STAILQ_REMOVE_HEAD(&queue->tq_queue, ta_link);
+		mtx_unlock(&queue->tq_mutex);
 		pending = task->ta_pending;
 		task->ta_pending = 0;
-		splx(s);
+		saved_func = task->ta_func;
+		arg = task->ta_context;
+		mtx_unlock(&task->ta_mutex);
 
-		task->ta_func(task->ta_context, pending);
+		saved_func(arg, pending);
 
-		s = splhigh();
+		mtx_lock(&queue->tq_mutex);
 	}
-	splx(s);
+	mtx_unlock(&queue->tq_mutex);
 }
 
 static void
diff --git a/sys/sys/taskqueue.h b/sys/sys/taskqueue.h
index 4cad2dc..dc940c9 100644
--- a/sys/sys/taskqueue.h
+++ b/sys/sys/taskqueue.h
@@ -34,6 +34,8 @@
 #endif
 
 #include <sys/queue.h>
+#include <sys/_lock.h>
+#include <sys/_mutex.h>
 
 struct taskqueue;
 
@@ -56,12 +58,16 @@ typedef void (*taskqueue_enqueue_fn)(void *context);
 
 struct task {
 	STAILQ_ENTRY(task) ta_link;	/* link for queue */
-	int	ta_pending;		/* count times queued */
-	int	ta_priority;		/* priority of task in queue */
-	task_fn_t *ta_func;		/* task handler */
-	void	*ta_context;		/* argument for handler */
+	int		ta_pending;	/* count times queued */
+	int		ta_priority;	/* priority of task in queue */
+	task_fn_t	*ta_func;	/* task handler */
+	void		*ta_context;	/* argument for handler */
+	struct mtx	ta_mutex;	/* lock for each task */
 };
 
+void	task_init(struct task *task, int priority, task_fn_t *func,
+	    void *context);
+void	task_destroy(struct task *task);
 struct taskqueue *taskqueue_create(const char *name, int mflags,
 				    taskqueue_enqueue_fn enqueue,
 				    void *context);
@@ -73,12 +79,14 @@ void	taskqueue_run(struct taskqueue *queue);
 /*
  * Initialise a task structure.
  */
-#define TASK_INIT(task, priority, func, context) do {	\
-	(task)->ta_pending = 0;				\
-	(task)->ta_priority = (priority);		\
-	(task)->ta_func = (func);			\
-	(task)->ta_context = (context);			\
-} while (0)
+#define	TASK_INIT(task, priority, func, context) \
+	task_init((task), (priority), (func), (context))
+
+/*
+ * Destroy a task structure.
+ */
+#define	TASK_DESTROY(task)			\
+	task_destroy((task))
 
 /*
  * Declare a reference to a taskqueue.