MFC r258630: 734 taskq_dispatch_prealloc() desired

6 files changed, 51 insertions, 82 deletions

diff --git a/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c b/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c
index f8f06d8..d5766c7 100644
--- a/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c
+++ b/sys/cddl/compat/opensolaris/kern/opensolaris_taskq.c
@@ -46,7 +46,7 @@ static void
 system_taskq_init(void *arg)
 {
 
-	taskq_zone = uma_zcreate("taskq_zone", sizeof(struct ostask),
+	taskq_zone = uma_zcreate("taskq_zone", sizeof(taskq_ent_t),
 	    NULL, NULL, NULL, NULL, 0, 0);
 	system_taskq = taskq_create("system_taskq", mp_ncpus, 0, 0, 0, 0);
 }
@@ -104,9 +104,9 @@ taskq_member(taskq_t *tq, kthread_t *thread)
 static void
 taskq_run(void *arg, int pending __unused)
 {
-	struct ostask *task = arg;
+	taskq_ent_t *task = arg;
 
-	task->ost_func(task->ost_arg);
+	task->tqent_func(task->tqent_arg);
 
 	uma_zfree(taskq_zone, task);
 }
@@ -114,7 +114,7 @@ taskq_run(void *arg, int pending __unused)
 taskqid_t
 taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
 {
-	struct ostask *task;
+	taskq_ent_t *task;
 	int mflag, prio;
 
 	if ((flags & (TQ_SLEEP | TQ_NOQUEUE)) == TQ_SLEEP)
@@ -131,26 +131,26 @@ taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags)
 	if (task == NULL)
 		return (0);
 
-	task->ost_func = func;
-	task->ost_arg = arg;
+	task->tqent_func = func;
+	task->tqent_arg = arg;
 
-	TASK_INIT(&task->ost_task, prio, taskq_run, task);
-	taskqueue_enqueue(tq->tq_queue, &task->ost_task);
+	TASK_INIT(&task->tqent_task, prio, taskq_run, task);
+	taskqueue_enqueue(tq->tq_queue, &task->tqent_task);
 
 	return ((taskqid_t)(void *)task);
 }
 
 static void
-taskq_run_safe(void *arg, int pending __unused)
+taskq_run_ent(void *arg, int pending __unused)
 {
-	struct ostask *task = arg;
+	taskq_ent_t *task = arg;
 
-	task->ost_func(task->ost_arg);
+	task->tqent_func(task->tqent_arg);
 }
 
-taskqid_t
-taskq_dispatch_safe(taskq_t *tq, task_func_t func, void *arg, u_int flags,
-    struct ostask *task)
+void
+taskq_dispatch_ent(taskq_t *tq, task_func_t func, void *arg, u_int flags,
+    taskq_ent_t *task)
 {
 	int prio;
 
@@ -160,11 +160,9 @@ taskq_dispatch_safe(taskq_t *tq, task_func_t func, void *arg, u_int flags,
 	 */
 	prio = !!(flags & TQ_FRONT);
 
-	task->ost_func = func;
-	task->ost_arg = arg;
+	task->tqent_func = func;
+	task->tqent_arg = arg;
 
-	TASK_INIT(&task->ost_task, prio, taskq_run_safe, task);
-	taskqueue_enqueue(tq->tq_queue, &task->ost_task);
-
-	return ((taskqid_t)(void *)task);
+	TASK_INIT(&task->tqent_task, prio, taskq_run_ent, task);
+	taskqueue_enqueue(tq->tq_queue, &task->tqent_task);
 }
diff --git a/sys/cddl/compat/opensolaris/sys/taskq.h b/sys/cddl/compat/opensolaris/sys/taskq.h
deleted file mode 100644
index ffe70ca..0000000
--- a/sys/cddl/compat/opensolaris/sys/taskq.h
+++ /dev/null
@@ -1,43 +0,0 @@
-/*-
- * Copyright (c) 2010 Pawel Jakub Dawidek <pjd@FreeBSD.org>
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND
- * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
- * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
- * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE
- * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
- * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
- * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
- * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
- * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
- * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
- * SUCH DAMAGE.
- *
- * $FreeBSD$
- */
-
-#ifndef _OPENSOLARIS_SYS_TASKQ_H_
-#define	_OPENSOLARIS_SYS_TASKQ_H_
-
-#include_next <sys/taskq.h>
-
-struct ostask {
-	struct task	 ost_task;
-	task_func_t	*ost_func;
-	void		*ost_arg;
-};
-
-taskqid_t taskq_dispatch_safe(taskq_t *tq, task_func_t func, void *arg,
-    u_int flags, struct ostask *task);
-
-#endif	/* _OPENSOLARIS_SYS_TASKQ_H_ */
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
index 31bc32d..a5c6b16 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/spa.c
@@ -825,7 +825,7 @@ static taskq_t *
 spa_taskq_create(spa_t *spa, const char *name, enum zti_modes mode,
     uint_t value)
 {
-	uint_t flags = TASKQ_PREPOPULATE;
+	uint_t flags = 0;
 	boolean_t batch = B_FALSE;
 
 	switch (mode) {
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h
index b001e11..70a8a7e 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/sys/zio.h
@@ -24,6 +24,7 @@
  * Copyright (c) 2012 by Delphix. All rights reserved.
  * Copyright (c) 2013 by Saso Kiselkov. All rights reserved.
  * Copyright (c) 2013, Joyent, Inc. All rights reserved.
+ * Copyright 2011 Nexenta Systems, Inc.  All rights reserved.
  */
 
 #ifndef _ZIO_H
@@ -474,10 +475,9 @@ struct zio {
 	zio_cksum_report_t *io_cksum_report;
 	uint64_t	io_ena;
 
-#ifdef _KERNEL
-	/* FreeBSD only. */
-	struct ostask	io_task;
-#endif
+	/* Taskq dispatching state */
+	taskq_ent_t	io_tqent;
+
 	avl_node_t	io_trim_node;
 	list_node_t	io_trim_link;
 };
diff --git a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
index 425a8ff..c5bcc8e 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/fs/zfs/zio.c
@@ -21,6 +21,7 @@
 /*
  * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved.
  * Copyright (c) 2013 by Delphix. All rights reserved.
+ * Copyright (c) 2011 Nexenta Systems, Inc. All rights reserved.
  */
 
 #include <sys/zfs_context.h>
@@ -1224,7 +1225,7 @@ zio_taskq_dispatch(zio_t *zio, enum zio_taskq_type q, boolean_t cutinline)
 {
 	spa_t *spa = zio->io_spa;
 	zio_type_t t = zio->io_type;
-	int flags = TQ_SLEEP | (cutinline ? TQ_FRONT : 0);
+	int flags = (cutinline ? TQ_FRONT : 0);
 
 	ASSERT(q == ZIO_TASKQ_ISSUE || q == ZIO_TASKQ_INTERRUPT);
 
@@ -1250,13 +1251,19 @@ zio_taskq_dispatch(zio_t *zio, enum zio_taskq_type q, boolean_t cutinline)
 		q++;
 
 	ASSERT3U(q, <, ZIO_TASKQ_TYPES);
-#ifdef _KERNEL
-	(void) taskq_dispatch_safe(spa->spa_zio_taskq[t][q],
-	    (task_func_t *)zio_execute, zio, flags, &zio->io_task);
+
+	/*
+	 * NB: We are assuming that the zio can only be dispatched
+	 * to a single taskq at a time.  It would be a grievous error
+	 * to dispatch the zio to another taskq at the same time.
+	 */
+#if defined(illumos) || !defined(_KERNEL)
+	ASSERT(zio->io_tqent.tqent_next == NULL);
 #else
-	(void) taskq_dispatch(spa->spa_zio_taskq[t][q],
-	    (task_func_t *)zio_execute, zio, flags);
+	ASSERT(zio->io_tqent.tqent_task.ta_pending == 0);
 #endif
+	taskq_dispatch_ent(spa->spa_zio_taskq[t][q],
+	    (task_func_t *)zio_execute, zio, flags, &zio->io_tqent);
 }
 
 static boolean_t
@@ -3174,16 +3181,15 @@ zio_done(zio_t *zio)
 			 * Reexecution is potentially a huge amount of work.
 			 * Hand it off to the otherwise-unused claim taskq.
 			 */
-#ifdef _KERNEL
-			(void) taskq_dispatch_safe(
-			    spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE],
-			    (task_func_t *)zio_reexecute, zio, TQ_SLEEP,
-			    &zio->io_task);
+#if defined(illumos) || !defined(_KERNEL)
+			ASSERT(zio->io_tqent.tqent_next == NULL);
 #else
-			(void) taskq_dispatch(
-			    spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE],
-			    (task_func_t *)zio_reexecute, zio, TQ_SLEEP);
+			ASSERT(zio->io_tqent.tqent_task.ta_pending == 0);
 #endif
+			(void) taskq_dispatch_ent(
+			    spa->spa_zio_taskq[ZIO_TYPE_CLAIM][ZIO_TASKQ_ISSUE],
+			    (task_func_t *)zio_reexecute, zio, 0,
+			    &zio->io_tqent);
 		}
 		return (ZIO_PIPELINE_STOP);
 	}
diff --git a/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h b/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h
index 246e470..1d8bfd6 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/sys/taskq.h
@@ -45,6 +45,12 @@ typedef struct taskq taskq_t;
 typedef uintptr_t taskqid_t;
 typedef void (task_func_t)(void *);
 
+typedef struct taskq_ent {
+	struct task	 tqent_task;
+	task_func_t	*tqent_func;
+	void		*tqent_arg;
+} taskq_ent_t;
+
 struct proc;
 
 /*
@@ -80,6 +86,8 @@ taskq_t	*taskq_create_proc(const char *, int, pri_t, int, int,
 taskq_t	*taskq_create_sysdc(const char *, int, int, int,
     struct proc *, uint_t, uint_t);
 taskqid_t taskq_dispatch(taskq_t *, task_func_t, void *, uint_t);
+void	taskq_dispatch_ent(taskq_t *, task_func_t, void *, uint_t,
+    taskq_ent_t *);
 void	nulltask(void *);
 void	taskq_destroy(taskq_t *);
 void	taskq_wait(taskq_t *);