diff options
Diffstat (limited to 'sys/contrib/opensolaris/uts/common/os')
| -rw-r--r-- | sys/contrib/opensolaris/uts/common/os/callb.c | 363 | ||||
| -rw-r--r-- | sys/contrib/opensolaris/uts/common/os/list.c | 193 | ||||
| -rw-r--r-- | sys/contrib/opensolaris/uts/common/os/nvpair_alloc_system.c | 63 | ||||
| -rw-r--r-- | sys/contrib/opensolaris/uts/common/os/taskq.c | 1020 |
4 files changed, 0 insertions, 1639 deletions
diff --git a/sys/contrib/opensolaris/uts/common/os/callb.c b/sys/contrib/opensolaris/uts/common/os/callb.c deleted file mode 100644 index c6e357e..0000000 --- a/sys/contrib/opensolaris/uts/common/os/callb.c +++ /dev/null @@ -1,363 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2004 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#pragma ident "%Z%%M% %I% %E% SMI" - -#include <sys/param.h> -#include <sys/types.h> -#include <sys/time.h> -#include <sys/sysmacros.h> -#include <sys/systm.h> -#include <sys/proc.h> -#include <sys/mutex.h> -#include <sys/condvar.h> -#include <sys/callb.h> -#include <sys/kmem.h> -#include <sys/cmn_err.h> -#include <sys/debug.h> -#include <sys/kobj.h> -#include <sys/systm.h> /* for delay() */ -#include <sys/taskq.h> /* For TASKQ_NAMELEN */ -#include <sys/kernel.h> - -#define CB_MAXNAME TASKQ_NAMELEN - -/* - * The callb mechanism provides generic event scheduling/echoing. - * A callb function is registered and called on behalf of the event. - */ -typedef struct callb { - struct callb *c_next; /* next in class or on freelist */ - kthread_id_t c_thread; /* ptr to caller's thread struct */ - char c_flag; /* info about the callb state */ - uchar_t c_class; /* this callb's class */ - kcondvar_t c_done_cv; /* signal callb completion */ - boolean_t (*c_func)(); /* cb function: returns true if ok */ - void *c_arg; /* arg to c_func */ - char c_name[CB_MAXNAME+1]; /* debug:max func name length */ -} callb_t; - -/* - * callb c_flag bitmap definitions - */ -#define CALLB_FREE 0x0 -#define CALLB_TAKEN 0x1 -#define CALLB_EXECUTING 0x2 - -/* - * Basic structure for a callb table. - * All callbs are organized into different class groups described - * by ct_class array. - * The callbs within a class are single-linked and normally run by a - * serial execution. - */ -typedef struct callb_table { - kmutex_t ct_lock; /* protect all callb states */ - callb_t *ct_freelist; /* free callb structures */ - int ct_busy; /* != 0 prevents additions */ - kcondvar_t ct_busy_cv; /* to wait for not busy */ - int ct_ncallb; /* num of callbs allocated */ - callb_t *ct_first_cb[NCBCLASS]; /* ptr to 1st callb in a class */ -} callb_table_t; - -int callb_timeout_sec = CPR_KTHREAD_TIMEOUT_SEC; - -static callb_id_t callb_add_common(boolean_t (*)(void *, int), - void *, int, char *, kthread_id_t); - -static callb_table_t callb_table; /* system level callback table */ -static callb_table_t *ct = &callb_table; -static kmutex_t callb_safe_mutex; -callb_cpr_t callb_cprinfo_safe = { - &callb_safe_mutex, CALLB_CPR_ALWAYS_SAFE, 0, 0, 0 }; - -/* - * Init all callb tables in the system. - */ -void -callb_init(void *dummy __unused) -{ - callb_table.ct_busy = 0; /* mark table open for additions */ - mutex_init(&callb_safe_mutex, NULL, MUTEX_DEFAULT, NULL); - mutex_init(&callb_table.ct_lock, NULL, MUTEX_DEFAULT, NULL); -} - -void -callb_fini(void *dummy __unused) -{ - callb_t *cp; - - mutex_enter(&ct->ct_lock); - while ((cp = ct->ct_freelist) != NULL) { - ct->ct_freelist = cp->c_next; - ct->ct_ncallb--; - kmem_free(cp, sizeof (callb_t)); - } - ASSERT(ct->ct_ncallb == 0); - mutex_exit(&ct->ct_lock); - mutex_destroy(&callb_safe_mutex); - mutex_destroy(&callb_table.ct_lock); -} - -/* - * callout_add() is called to register func() be called later. - */ -static callb_id_t -callb_add_common(boolean_t (*func)(void *arg, int code), - void *arg, int class, char *name, kthread_id_t t) -{ - callb_t *cp; - - ASSERT(class < NCBCLASS); - - mutex_enter(&ct->ct_lock); - while (ct->ct_busy) - cv_wait(&ct->ct_busy_cv, &ct->ct_lock); - if ((cp = ct->ct_freelist) == NULL) { - ct->ct_ncallb++; - cp = (callb_t *)kmem_zalloc(sizeof (callb_t), KM_SLEEP); - } - ct->ct_freelist = cp->c_next; - cp->c_thread = t; - cp->c_func = func; - cp->c_arg = arg; - cp->c_class = (uchar_t)class; - cp->c_flag |= CALLB_TAKEN; -#ifdef DEBUG - if (strlen(name) > CB_MAXNAME) - cmn_err(CE_WARN, "callb_add: name of callback function '%s' " - "too long -- truncated to %d chars", - name, CB_MAXNAME); -#endif - (void) strncpy(cp->c_name, name, CB_MAXNAME); - cp->c_name[CB_MAXNAME] = '\0'; - - /* - * Insert the new callb at the head of its class list. - */ - cp->c_next = ct->ct_first_cb[class]; - ct->ct_first_cb[class] = cp; - - mutex_exit(&ct->ct_lock); - return ((callb_id_t)cp); -} - -/* - * The default function to add an entry to the callback table. Since - * it uses curthread as the thread identifier to store in the table, - * it should be used for the normal case of a thread which is calling - * to add ITSELF to the table. - */ -callb_id_t -callb_add(boolean_t (*func)(void *arg, int code), - void *arg, int class, char *name) -{ - return (callb_add_common(func, arg, class, name, curthread)); -} - -/* - * A special version of callb_add() above for use by threads which - * might be adding an entry to the table on behalf of some other - * thread (for example, one which is constructed but not yet running). - * In this version the thread id is an argument. - */ -callb_id_t -callb_add_thread(boolean_t (*func)(void *arg, int code), - void *arg, int class, char *name, kthread_id_t t) -{ - return (callb_add_common(func, arg, class, name, t)); -} - -/* - * callout_delete() is called to remove an entry identified by id - * that was originally placed there by a call to callout_add(). - * return -1 if fail to delete a callb entry otherwise return 0. - */ -int -callb_delete(callb_id_t id) -{ - callb_t **pp; - callb_t *me = (callb_t *)id; - - mutex_enter(&ct->ct_lock); - - for (;;) { - pp = &ct->ct_first_cb[me->c_class]; - while (*pp != NULL && *pp != me) - pp = &(*pp)->c_next; - -#ifdef DEBUG - if (*pp != me) { - cmn_err(CE_WARN, "callb delete bogus entry 0x%p", - (void *)me); - mutex_exit(&ct->ct_lock); - return (-1); - } -#endif /* DEBUG */ - - /* - * It is not allowed to delete a callb in the middle of - * executing otherwise, the callb_execute() will be confused. - */ - if (!(me->c_flag & CALLB_EXECUTING)) - break; - - cv_wait(&me->c_done_cv, &ct->ct_lock); - } - /* relink the class list */ - *pp = me->c_next; - - /* clean up myself and return the free callb to the head of freelist */ - me->c_flag = CALLB_FREE; - me->c_next = ct->ct_freelist; - ct->ct_freelist = me; - - mutex_exit(&ct->ct_lock); - return (0); -} - -/* - * class: indicates to execute all callbs in the same class; - * code: optional argument for the callb functions. - * return: = 0: success - * != 0: ptr to string supplied when callback was registered - */ -void * -callb_execute_class(int class, int code) -{ - callb_t *cp; - void *ret = NULL; - - ASSERT(class < NCBCLASS); - - mutex_enter(&ct->ct_lock); - - for (cp = ct->ct_first_cb[class]; - cp != NULL && ret == 0; cp = cp->c_next) { - while (cp->c_flag & CALLB_EXECUTING) - cv_wait(&cp->c_done_cv, &ct->ct_lock); - /* - * cont if the callb is deleted while we're sleeping - */ - if (cp->c_flag == CALLB_FREE) - continue; - cp->c_flag |= CALLB_EXECUTING; - -#ifdef CALLB_DEBUG - printf("callb_execute: name=%s func=%p arg=%p\n", - cp->c_name, (void *)cp->c_func, (void *)cp->c_arg); -#endif /* CALLB_DEBUG */ - - mutex_exit(&ct->ct_lock); - /* If callback function fails, pass back client's name */ - if (!(*cp->c_func)(cp->c_arg, code)) - ret = cp->c_name; - mutex_enter(&ct->ct_lock); - - cp->c_flag &= ~CALLB_EXECUTING; - cv_broadcast(&cp->c_done_cv); - } - mutex_exit(&ct->ct_lock); - return (ret); -} - -/* - * callers make sure no recursive entries to this func. - * dp->cc_lockp is registered by callb_add to protect callb_cpr_t structure. - * - * When calling to stop a kernel thread (code == CB_CODE_CPR_CHKPT) we - * use a cv_timedwait() in case the kernel thread is blocked. - * - * Note that this is a generic callback handler for daemon CPR and - * should NOT be changed to accommodate any specific requirement in a daemon. - * Individual daemons that require changes to the handler shall write - * callback routines in their own daemon modules. - */ -boolean_t -callb_generic_cpr(void *arg, int code) -{ - callb_cpr_t *cp = (callb_cpr_t *)arg; - clock_t ret = 0; /* assume success */ - - mutex_enter(cp->cc_lockp); - - switch (code) { - case CB_CODE_CPR_CHKPT: - cp->cc_events |= CALLB_CPR_START; - while (!(cp->cc_events & CALLB_CPR_SAFE)) - /* cv_timedwait() returns -1 if it times out. */ - if ((ret = cv_timedwait(&cp->cc_callb_cv, - cp->cc_lockp, - callb_timeout_sec * hz)) == -1) - break; - break; - - case CB_CODE_CPR_RESUME: - cp->cc_events &= ~CALLB_CPR_START; - cv_signal(&cp->cc_stop_cv); - break; - } - mutex_exit(cp->cc_lockp); - return (ret != -1); -} - -/* - * The generic callback function associated with kernel threads which - * are always considered safe. - */ -/* ARGSUSED */ -boolean_t -callb_generic_cpr_safe(void *arg, int code) -{ - return (B_TRUE); -} -/* - * Prevent additions to callback table. - */ -void -callb_lock_table(void) -{ - mutex_enter(&ct->ct_lock); - ASSERT(ct->ct_busy == 0); - ct->ct_busy = 1; - mutex_exit(&ct->ct_lock); -} - -/* - * Allow additions to callback table. - */ -void -callb_unlock_table(void) -{ - mutex_enter(&ct->ct_lock); - ASSERT(ct->ct_busy != 0); - ct->ct_busy = 0; - cv_broadcast(&ct->ct_busy_cv); - mutex_exit(&ct->ct_lock); -} - -SYSINIT(sol_callb, SI_SUB_DRIVERS, SI_ORDER_FIRST, callb_init, NULL); -SYSUNINIT(sol_callb, SI_SUB_DRIVERS, SI_ORDER_FIRST, callb_fini, NULL); diff --git a/sys/contrib/opensolaris/uts/common/os/list.c b/sys/contrib/opensolaris/uts/common/os/list.c deleted file mode 100644 index f9b6fcb..0000000 --- a/sys/contrib/opensolaris/uts/common/os/list.c +++ /dev/null @@ -1,193 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2006 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#pragma ident "%Z%%M% %I% %E% SMI" - -/* - * Generic doubly-linked list implementation - */ - -#include <sys/list.h> -#include <sys/list_impl.h> -#include <sys/types.h> -#include <sys/sysmacros.h> -#include <sys/debug.h> - -#define list_d2l(a, obj) ((list_node_t *)(((char *)obj) + (a)->list_offset)) -#define list_object(a, node) ((void *)(((char *)node) - (a)->list_offset)) -#define list_empty(a) ((a)->list_head.list_next == &(a)->list_head) - -#define list_insert_after_node(list, node, object) { \ - list_node_t *lnew = list_d2l(list, object); \ - lnew->list_prev = node; \ - lnew->list_next = node->list_next; \ - node->list_next->list_prev = lnew; \ - node->list_next = lnew; \ -} - -#define list_insert_before_node(list, node, object) { \ - list_node_t *lnew = list_d2l(list, object); \ - lnew->list_next = node; \ - lnew->list_prev = node->list_prev; \ - node->list_prev->list_next = lnew; \ - node->list_prev = lnew; \ -} - -void -list_create(list_t *list, size_t size, size_t offset) -{ - ASSERT(list); - ASSERT(size > 0); - ASSERT(size >= offset + sizeof (list_node_t)); - - list->list_size = size; - list->list_offset = offset; - list->list_head.list_next = list->list_head.list_prev = - &list->list_head; -} - -void -list_destroy(list_t *list) -{ - list_node_t *node = &list->list_head; - - ASSERT(list); - ASSERT(list->list_head.list_next == node); - ASSERT(list->list_head.list_prev == node); - - node->list_next = node->list_prev = NULL; -} - -void -list_insert_after(list_t *list, void *object, void *nobject) -{ - list_node_t *lold = list_d2l(list, object); - list_insert_after_node(list, lold, nobject); -} - -void -list_insert_before(list_t *list, void *object, void *nobject) -{ - list_node_t *lold = list_d2l(list, object); - list_insert_before_node(list, lold, nobject) -} - -void -list_insert_head(list_t *list, void *object) -{ - list_node_t *lold = &list->list_head; - list_insert_after_node(list, lold, object); -} - -void -list_insert_tail(list_t *list, void *object) -{ - list_node_t *lold = &list->list_head; - list_insert_before_node(list, lold, object); -} - -void -list_remove(list_t *list, void *object) -{ - list_node_t *lold = list_d2l(list, object); - ASSERT(!list_empty(list)); - lold->list_prev->list_next = lold->list_next; - lold->list_next->list_prev = lold->list_prev; - lold->list_next = lold->list_prev = NULL; -} - -void * -list_head(list_t *list) -{ - if (list_empty(list)) - return (NULL); - return (list_object(list, list->list_head.list_next)); -} - -void * -list_tail(list_t *list) -{ - if (list_empty(list)) - return (NULL); - return (list_object(list, list->list_head.list_prev)); -} - -void * -list_next(list_t *list, void *object) -{ - list_node_t *node = list_d2l(list, object); - - if (node->list_next != &list->list_head) - return (list_object(list, node->list_next)); - - return (NULL); -} - -void * -list_prev(list_t *list, void *object) -{ - list_node_t *node = list_d2l(list, object); - - if (node->list_prev != &list->list_head) - return (list_object(list, node->list_prev)); - - return (NULL); -} - -/* - * Insert src list after dst list. Empty src list thereafter. - */ -void -list_move_tail(list_t *dst, list_t *src) -{ - list_node_t *dstnode = &dst->list_head; - list_node_t *srcnode = &src->list_head; - - ASSERT(dst->list_size == src->list_size); - ASSERT(dst->list_offset == src->list_offset); - - if (list_empty(src)) - return; - - dstnode->list_prev->list_next = srcnode->list_next; - srcnode->list_next->list_prev = dstnode->list_prev; - dstnode->list_prev = srcnode->list_prev; - srcnode->list_prev->list_next = dstnode; - - /* empty src list */ - srcnode->list_next = srcnode->list_prev = srcnode; -} - -int -list_link_active(list_node_t *link) -{ - return (link->list_next != NULL); -} - -int -list_is_empty(list_t *list) -{ - return (list_empty(list)); -} diff --git a/sys/contrib/opensolaris/uts/common/os/nvpair_alloc_system.c b/sys/contrib/opensolaris/uts/common/os/nvpair_alloc_system.c deleted file mode 100644 index 3682853..0000000 --- a/sys/contrib/opensolaris/uts/common/os/nvpair_alloc_system.c +++ /dev/null @@ -1,63 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2004 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#pragma ident "%Z%%M% %I% %E% SMI" - -#include <sys/nvpair.h> - -static void * -nv_alloc_sys(nv_alloc_t *nva, size_t size) -{ - return (kmem_alloc(size, (int)(uintptr_t)nva->nva_arg)); -} - -/*ARGSUSED*/ -static void -nv_free_sys(nv_alloc_t *nva, void *buf, size_t size) -{ - kmem_free(buf, size); -} - -static const nv_alloc_ops_t system_ops = { - NULL, /* nv_ao_init() */ - NULL, /* nv_ao_fini() */ - nv_alloc_sys, /* nv_ao_alloc() */ - nv_free_sys, /* nv_ao_free() */ - NULL /* nv_ao_reset() */ -}; - -nv_alloc_t nv_alloc_sleep_def = { - &system_ops, - (void *)KM_SLEEP -}; - -nv_alloc_t nv_alloc_nosleep_def = { - &system_ops, - (void *)KM_NOSLEEP -}; - -nv_alloc_t *nv_alloc_sleep = &nv_alloc_sleep_def; -nv_alloc_t *nv_alloc_nosleep = &nv_alloc_nosleep_def; diff --git a/sys/contrib/opensolaris/uts/common/os/taskq.c b/sys/contrib/opensolaris/uts/common/os/taskq.c deleted file mode 100644 index 1558c1f..0000000 --- a/sys/contrib/opensolaris/uts/common/os/taskq.c +++ /dev/null @@ -1,1020 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License, Version 1.0 only - * (the "License"). You may not use this file except in compliance - * with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ -/* - * Copyright 2005 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#pragma ident "%Z%%M% %I% %E% SMI" - -/* - * Kernel task queues: general-purpose asynchronous task scheduling. - * - * A common problem in kernel programming is the need to schedule tasks - * to be performed later, by another thread. There are several reasons - * you may want or need to do this: - * - * (1) The task isn't time-critical, but your current code path is. - * - * (2) The task may require grabbing locks that you already hold. - * - * (3) The task may need to block (e.g. to wait for memory), but you - * cannot block in your current context. - * - * (4) Your code path can't complete because of some condition, but you can't - * sleep or fail, so you queue the task for later execution when condition - * disappears. - * - * (5) You just want a simple way to launch multiple tasks in parallel. - * - * Task queues provide such a facility. In its simplest form (used when - * performance is not a critical consideration) a task queue consists of a - * single list of tasks, together with one or more threads to service the - * list. There are some cases when this simple queue is not sufficient: - * - * (1) The task queues are very hot and there is a need to avoid data and lock - * contention over global resources. - * - * (2) Some tasks may depend on other tasks to complete, so they can't be put in - * the same list managed by the same thread. - * - * (3) Some tasks may block for a long time, and this should not block other - * tasks in the queue. - * - * To provide useful service in such cases we define a "dynamic task queue" - * which has an individual thread for each of the tasks. These threads are - * dynamically created as they are needed and destroyed when they are not in - * use. The API for managing task pools is the same as for managing task queues - * with the exception of a taskq creation flag TASKQ_DYNAMIC which tells that - * dynamic task pool behavior is desired. - * - * Dynamic task queues may also place tasks in the normal queue (called "backing - * queue") when task pool runs out of resources. Users of task queues may - * disallow such queued scheduling by specifying TQ_NOQUEUE in the dispatch - * flags. - * - * The backing task queue is also used for scheduling internal tasks needed for - * dynamic task queue maintenance. - * - * INTERFACES: - * - * taskq_t *taskq_create(name, nthreads, pri_t pri, minalloc, maxall, flags); - * - * Create a taskq with specified properties. - * Possible 'flags': - * - * TASKQ_DYNAMIC: Create task pool for task management. If this flag is - * specified, 'nthreads' specifies the maximum number of threads in - * the task queue. Task execution order for dynamic task queues is - * not predictable. - * - * If this flag is not specified (default case) a - * single-list task queue is created with 'nthreads' threads - * servicing it. Entries in this queue are managed by - * taskq_ent_alloc() and taskq_ent_free() which try to keep the - * task population between 'minalloc' and 'maxalloc', but the - * latter limit is only advisory for TQ_SLEEP dispatches and the - * former limit is only advisory for TQ_NOALLOC dispatches. If - * TASKQ_PREPOPULATE is set in 'flags', the taskq will be - * prepopulated with 'minalloc' task structures. - * - * Since non-DYNAMIC taskqs are queues, tasks are guaranteed to be - * executed in the order they are scheduled if nthreads == 1. - * If nthreads > 1, task execution order is not predictable. - * - * TASKQ_PREPOPULATE: Prepopulate task queue with threads. - * Also prepopulate the task queue with 'minalloc' task structures. - * - * TASKQ_CPR_SAFE: This flag specifies that users of the task queue will - * use their own protocol for handling CPR issues. This flag is not - * supported for DYNAMIC task queues. - * - * The 'pri' field specifies the default priority for the threads that - * service all scheduled tasks. - * - * void taskq_destroy(tap): - * - * Waits for any scheduled tasks to complete, then destroys the taskq. - * Caller should guarantee that no new tasks are scheduled in the closing - * taskq. - * - * taskqid_t taskq_dispatch(tq, func, arg, flags): - * - * Dispatches the task "func(arg)" to taskq. The 'flags' indicates whether - * the caller is willing to block for memory. The function returns an - * opaque value which is zero iff dispatch fails. If flags is TQ_NOSLEEP - * or TQ_NOALLOC and the task can't be dispatched, taskq_dispatch() fails - * and returns (taskqid_t)0. - * - * ASSUMES: func != NULL. - * - * Possible flags: - * TQ_NOSLEEP: Do not wait for resources; may fail. - * - * TQ_NOALLOC: Do not allocate memory; may fail. May only be used with - * non-dynamic task queues. - * - * TQ_NOQUEUE: Do not enqueue a task if it can't dispatch it due to - * lack of available resources and fail. If this flag is not - * set, and the task pool is exhausted, the task may be scheduled - * in the backing queue. This flag may ONLY be used with dynamic - * task queues. - * - * NOTE: This flag should always be used when a task queue is used - * for tasks that may depend on each other for completion. - * Enqueueing dependent tasks may create deadlocks. - * - * TQ_SLEEP: May block waiting for resources. May still fail for - * dynamic task queues if TQ_NOQUEUE is also specified, otherwise - * always succeed. - * - * NOTE: Dynamic task queues are much more likely to fail in - * taskq_dispatch() (especially if TQ_NOQUEUE was specified), so it - * is important to have backup strategies handling such failures. - * - * void taskq_wait(tq): - * - * Waits for all previously scheduled tasks to complete. - * - * NOTE: It does not stop any new task dispatches. - * Do NOT call taskq_wait() from a task: it will cause deadlock. - * - * void taskq_suspend(tq) - * - * Suspend all task execution. Tasks already scheduled for a dynamic task - * queue will still be executed, but all new scheduled tasks will be - * suspended until taskq_resume() is called. - * - * int taskq_suspended(tq) - * - * Returns 1 if taskq is suspended and 0 otherwise. It is intended to - * ASSERT that the task queue is suspended. - * - * void taskq_resume(tq) - * - * Resume task queue execution. - * - * int taskq_member(tq, thread) - * - * Returns 1 if 'thread' belongs to taskq 'tq' and 0 otherwise. The - * intended use is to ASSERT that a given function is called in taskq - * context only. - * - * system_taskq - * - * Global system-wide dynamic task queue for common uses. It may be used by - * any subsystem that needs to schedule tasks and does not need to manage - * its own task queues. It is initialized quite early during system boot. - * - * IMPLEMENTATION. - * - * This is schematic representation of the task queue structures. - * - * taskq: - * +-------------+ - * |tq_lock | +---< taskq_ent_free() - * +-------------+ | - * |... | | tqent: tqent: - * +-------------+ | +------------+ +------------+ - * | tq_freelist |-->| tqent_next |--> ... ->| tqent_next | - * +-------------+ +------------+ +------------+ - * |... | | ... | | ... | - * +-------------+ +------------+ +------------+ - * | tq_task | | - * | | +-------------->taskq_ent_alloc() - * +--------------------------------------------------------------------------+ - * | | | tqent tqent | - * | +---------------------+ +--> +------------+ +--> +------------+ | - * | | ... | | | func, arg | | | func, arg | | - * +>+---------------------+ <---|-+ +------------+ <---|-+ +------------+ | - * | tq_taskq.tqent_next | ----+ | | tqent_next | --->+ | | tqent_next |--+ - * +---------------------+ | +------------+ ^ | +------------+ - * +-| tq_task.tqent_prev | +--| tqent_prev | | +--| tqent_prev | ^ - * | +---------------------+ +------------+ | +------------+ | - * | |... | | ... | | | ... | | - * | +---------------------+ +------------+ | +------------+ | - * | ^ | | - * | | | | - * +--------------------------------------+--------------+ TQ_APPEND() -+ - * | | | - * |... | taskq_thread()-----+ - * +-------------+ - * | tq_buckets |--+-------> [ NULL ] (for regular task queues) - * +-------------+ | - * | DYNAMIC TASK QUEUES: - * | - * +-> taskq_bucket[nCPU] taskq_bucket_dispatch() - * +-------------------+ ^ - * +--->| tqbucket_lock | | - * | +-------------------+ +--------+ +--------+ - * | | tqbucket_freelist |-->| tqent |-->...| tqent | ^ - * | +-------------------+<--+--------+<--...+--------+ | - * | | ... | | thread | | thread | | - * | +-------------------+ +--------+ +--------+ | - * | +-------------------+ | - * taskq_dispatch()--+--->| tqbucket_lock | TQ_APPEND()------+ - * TQ_HASH() | +-------------------+ +--------+ +--------+ - * | | tqbucket_freelist |-->| tqent |-->...| tqent | - * | +-------------------+<--+--------+<--...+--------+ - * | | ... | | thread | | thread | - * | +-------------------+ +--------+ +--------+ - * +---> ... - * - * - * Task queues use tq_task field to link new entry in the queue. The queue is a - * circular doubly-linked list. Entries are put in the end of the list with - * TQ_APPEND() and processed from the front of the list by taskq_thread() in - * FIFO order. Task queue entries are cached in the free list managed by - * taskq_ent_alloc() and taskq_ent_free() functions. - * - * All threads used by task queues mark t_taskq field of the thread to - * point to the task queue. - * - * Dynamic Task Queues Implementation. - * - * For a dynamic task queues there is a 1-to-1 mapping between a thread and - * taskq_ent_structure. Each entry is serviced by its own thread and each thread - * is controlled by a single entry. - * - * Entries are distributed over a set of buckets. To avoid using modulo - * arithmetics the number of buckets is 2^n and is determined as the nearest - * power of two roundown of the number of CPUs in the system. Tunable - * variable 'taskq_maxbuckets' limits the maximum number of buckets. Each entry - * is attached to a bucket for its lifetime and can't migrate to other buckets. - * - * Entries that have scheduled tasks are not placed in any list. The dispatch - * function sets their "func" and "arg" fields and signals the corresponding - * thread to execute the task. Once the thread executes the task it clears the - * "func" field and places an entry on the bucket cache of free entries pointed - * by "tqbucket_freelist" field. ALL entries on the free list should have "func" - * field equal to NULL. The free list is a circular doubly-linked list identical - * in structure to the tq_task list above, but entries are taken from it in LIFO - * order - the last freed entry is the first to be allocated. The - * taskq_bucket_dispatch() function gets the most recently used entry from the - * free list, sets its "func" and "arg" fields and signals a worker thread. - * - * After executing each task a per-entry thread taskq_d_thread() places its - * entry on the bucket free list and goes to a timed sleep. If it wakes up - * without getting new task it removes the entry from the free list and destroys - * itself. The thread sleep time is controlled by a tunable variable - * `taskq_thread_timeout'. - * - * There is various statistics kept in the bucket which allows for later - * analysis of taskq usage patterns. Also, a global copy of taskq creation and - * death statistics is kept in the global taskq data structure. Since thread - * creation and death happen rarely, updating such global data does not present - * a performance problem. - * - * NOTE: Threads are not bound to any CPU and there is absolutely no association - * between the bucket and actual thread CPU, so buckets are used only to - * split resources and reduce resource contention. Having threads attached - * to the CPU denoted by a bucket may reduce number of times the job - * switches between CPUs. - * - * Current algorithm creates a thread whenever a bucket has no free - * entries. It would be nice to know how many threads are in the running - * state and don't create threads if all CPUs are busy with existing - * tasks, but it is unclear how such strategy can be implemented. - * - * Currently buckets are created statically as an array attached to task - * queue. On some system with nCPUs < max_ncpus it may waste system - * memory. One solution may be allocation of buckets when they are first - * touched, but it is not clear how useful it is. - * - * SUSPEND/RESUME implementation. - * - * Before executing a task taskq_thread() (executing non-dynamic task - * queues) obtains taskq's thread lock as a reader. The taskq_suspend() - * function gets the same lock as a writer blocking all non-dynamic task - * execution. The taskq_resume() function releases the lock allowing - * taskq_thread to continue execution. - * - * For dynamic task queues, each bucket is marked as TQBUCKET_SUSPEND by - * taskq_suspend() function. After that taskq_bucket_dispatch() always - * fails, so that taskq_dispatch() will either enqueue tasks for a - * suspended backing queue or fail if TQ_NOQUEUE is specified in dispatch - * flags. - * - * NOTE: taskq_suspend() does not immediately block any tasks already - * scheduled for dynamic task queues. It only suspends new tasks - * scheduled after taskq_suspend() was called. - * - * taskq_member() function works by comparing a thread t_taskq pointer with - * the passed thread pointer. - * - * LOCKS and LOCK Hierarchy: - * - * There are two locks used in task queues. - * - * 1) Task queue structure has a lock, protecting global task queue state. - * - * 2) Each per-CPU bucket has a lock for bucket management. - * - * If both locks are needed, task queue lock should be taken only after bucket - * lock. - * - * DEBUG FACILITIES. - * - * For DEBUG kernels it is possible to induce random failures to - * taskq_dispatch() function when it is given TQ_NOSLEEP argument. The value of - * taskq_dmtbf and taskq_smtbf tunables control the mean time between induced - * failures for dynamic and static task queues respectively. - * - * Setting TASKQ_STATISTIC to 0 will disable per-bucket statistics. - * - * TUNABLES - * - * system_taskq_size - Size of the global system_taskq. - * This value is multiplied by nCPUs to determine - * actual size. - * Default value: 64 - * - * taskq_thread_timeout - Maximum idle time for taskq_d_thread() - * Default value: 5 minutes - * - * taskq_maxbuckets - Maximum number of buckets in any task queue - * Default value: 128 - * - * taskq_search_depth - Maximum # of buckets searched for a free entry - * Default value: 4 - * - * taskq_dmtbf - Mean time between induced dispatch failures - * for dynamic task queues. - * Default value: UINT_MAX (no induced failures) - * - * taskq_smtbf - Mean time between induced dispatch failures - * for static task queues. - * Default value: UINT_MAX (no induced failures) - * - * CONDITIONAL compilation. - * - * TASKQ_STATISTIC - If set will enable bucket statistic (default). - * - */ - -#include <sys/taskq_impl.h> -#include <sys/proc.h> -#include <sys/kmem.h> -#include <sys/callb.h> -#include <sys/systm.h> -#include <sys/cmn_err.h> -#include <sys/debug.h> -#include <sys/sysmacros.h> -#include <sys/sdt.h> -#include <sys/mutex.h> -#include <sys/kernel.h> -#include <sys/limits.h> - -static kmem_cache_t *taskq_ent_cache, *taskq_cache; - -/* Global system task queue for common use */ -taskq_t *system_taskq; - -/* - * Maxmimum number of entries in global system taskq is - * system_taskq_size * max_ncpus - */ -#define SYSTEM_TASKQ_SIZE 1 -int system_taskq_size = SYSTEM_TASKQ_SIZE; - -/* - * Dynamic task queue threads that don't get any work within - * taskq_thread_timeout destroy themselves - */ -#define TASKQ_THREAD_TIMEOUT (60 * 5) -int taskq_thread_timeout = TASKQ_THREAD_TIMEOUT; - -#define TASKQ_MAXBUCKETS 128 -int taskq_maxbuckets = TASKQ_MAXBUCKETS; - -/* - * When a bucket has no available entries another buckets are tried. - * taskq_search_depth parameter limits the amount of buckets that we search - * before failing. This is mostly useful in systems with many CPUs where we may - * spend too much time scanning busy buckets. - */ -#define TASKQ_SEARCH_DEPTH 4 -int taskq_search_depth = TASKQ_SEARCH_DEPTH; - -/* - * Hashing function: mix various bits of x. May be pretty much anything. - */ -#define TQ_HASH(x) ((x) ^ ((x) >> 11) ^ ((x) >> 17) ^ ((x) ^ 27)) - -/* - * We do not create any new threads when the system is low on memory and start - * throttling memory allocations. The following macro tries to estimate such - * condition. - */ -#define ENOUGH_MEMORY() (freemem > throttlefree) - -/* - * Static functions. - */ -static taskq_t *taskq_create_common(const char *, int, int, pri_t, int, - int, uint_t); -static void taskq_thread(void *); -static int taskq_constructor(void *, void *, int); -static void taskq_destructor(void *, void *); -static int taskq_ent_constructor(void *, void *, int); -static void taskq_ent_destructor(void *, void *); -static taskq_ent_t *taskq_ent_alloc(taskq_t *, int); -static void taskq_ent_free(taskq_t *, taskq_ent_t *); - -/* - * Collect per-bucket statistic when TASKQ_STATISTIC is defined. - */ -#define TASKQ_STATISTIC 1 - -#if TASKQ_STATISTIC -#define TQ_STAT(b, x) b->tqbucket_stat.x++ -#else -#define TQ_STAT(b, x) -#endif - -/* - * Random fault injection. - */ -uint_t taskq_random; -uint_t taskq_dmtbf = UINT_MAX; /* mean time between injected failures */ -uint_t taskq_smtbf = UINT_MAX; /* mean time between injected failures */ - -/* - * TQ_NOSLEEP dispatches on dynamic task queues are always allowed to fail. - * - * TQ_NOSLEEP dispatches on static task queues can't arbitrarily fail because - * they could prepopulate the cache and make sure that they do not use more - * then minalloc entries. So, fault injection in this case insures that - * either TASKQ_PREPOPULATE is not set or there are more entries allocated - * than is specified by minalloc. TQ_NOALLOC dispatches are always allowed - * to fail, but for simplicity we treat them identically to TQ_NOSLEEP - * dispatches. - */ -#ifdef DEBUG -#define TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag) \ - taskq_random = (taskq_random * 2416 + 374441) % 1771875;\ - if ((flag & TQ_NOSLEEP) && \ - taskq_random < 1771875 / taskq_dmtbf) { \ - return (NULL); \ - } - -#define TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag) \ - taskq_random = (taskq_random * 2416 + 374441) % 1771875;\ - if ((flag & (TQ_NOSLEEP | TQ_NOALLOC)) && \ - (!(tq->tq_flags & TASKQ_PREPOPULATE) || \ - (tq->tq_nalloc > tq->tq_minalloc)) && \ - (taskq_random < (1771875 / taskq_smtbf))) { \ - mutex_exit(&tq->tq_lock); \ - return ((taskqid_t)0); \ - } -#else -#define TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flag) -#define TASKQ_D_RANDOM_DISPATCH_FAILURE(tq, flag) -#endif - -#define IS_EMPTY(l) (((l).tqent_prev == (l).tqent_next) && \ - ((l).tqent_prev == &(l))) - -/* - * Append `tqe' in the end of the doubly-linked list denoted by l. - */ -#define TQ_APPEND(l, tqe) { \ - tqe->tqent_next = &l; \ - tqe->tqent_prev = l.tqent_prev; \ - tqe->tqent_next->tqent_prev = tqe; \ - tqe->tqent_prev->tqent_next = tqe; \ -} - -/* - * Schedule a task specified by func and arg into the task queue entry tqe. - */ -#define TQ_ENQUEUE(tq, tqe, func, arg) { \ - ASSERT(MUTEX_HELD(&tq->tq_lock)); \ - TQ_APPEND(tq->tq_task, tqe); \ - tqe->tqent_func = (func); \ - tqe->tqent_arg = (arg); \ - tq->tq_tasks++; \ - if (tq->tq_tasks - tq->tq_executed > tq->tq_maxtasks) \ - tq->tq_maxtasks = tq->tq_tasks - tq->tq_executed; \ - cv_signal(&tq->tq_dispatch_cv); \ - DTRACE_PROBE2(taskq__enqueue, taskq_t *, tq, taskq_ent_t *, tqe); \ -} - -/* - * Do-nothing task which may be used to prepopulate thread caches. - */ -/*ARGSUSED*/ -void -nulltask(void *unused) -{ -} - - -/*ARGSUSED*/ -static int -taskq_constructor(void *buf, void *cdrarg, int kmflags) -{ - taskq_t *tq = buf; - - bzero(tq, sizeof (taskq_t)); - - mutex_init(&tq->tq_lock, NULL, MUTEX_DEFAULT, NULL); - rw_init(&tq->tq_threadlock, NULL, RW_DEFAULT, NULL); - cv_init(&tq->tq_dispatch_cv, NULL, CV_DEFAULT, NULL); - cv_init(&tq->tq_wait_cv, NULL, CV_DEFAULT, NULL); - - tq->tq_task.tqent_next = &tq->tq_task; - tq->tq_task.tqent_prev = &tq->tq_task; - - return (0); -} - -/*ARGSUSED*/ -static void -taskq_destructor(void *buf, void *cdrarg) -{ - taskq_t *tq = buf; - - mutex_destroy(&tq->tq_lock); - rw_destroy(&tq->tq_threadlock); - cv_destroy(&tq->tq_dispatch_cv); - cv_destroy(&tq->tq_wait_cv); -} - -/*ARGSUSED*/ -static int -taskq_ent_constructor(void *buf, void *cdrarg, int kmflags) -{ - taskq_ent_t *tqe = buf; - - tqe->tqent_thread = NULL; - cv_init(&tqe->tqent_cv, NULL, CV_DEFAULT, NULL); - - return (0); -} - -/*ARGSUSED*/ -static void -taskq_ent_destructor(void *buf, void *cdrarg) -{ - taskq_ent_t *tqe = buf; - - ASSERT(tqe->tqent_thread == NULL); - cv_destroy(&tqe->tqent_cv); -} - -/* - * Create global system dynamic task queue. - */ -void -system_taskq_init(void) -{ - system_taskq = taskq_create_common("system_taskq", 0, - system_taskq_size * max_ncpus, minclsyspri, 4, 512, - TASKQ_PREPOPULATE); -} - -void -system_taskq_fini(void) -{ - taskq_destroy(system_taskq); -} - -static void -taskq_init(void *dummy __unused) -{ - taskq_ent_cache = kmem_cache_create("taskq_ent_cache", - sizeof (taskq_ent_t), 0, taskq_ent_constructor, - taskq_ent_destructor, NULL, NULL, NULL, 0); - taskq_cache = kmem_cache_create("taskq_cache", sizeof (taskq_t), - 0, taskq_constructor, taskq_destructor, NULL, NULL, NULL, 0); - system_taskq_init(); -} - -static void -taskq_fini(void *dummy __unused) -{ - system_taskq_fini(); - kmem_cache_destroy(taskq_cache); - kmem_cache_destroy(taskq_ent_cache); -} - -/* - * taskq_ent_alloc() - * - * Allocates a new taskq_ent_t structure either from the free list or from the - * cache. Returns NULL if it can't be allocated. - * - * Assumes: tq->tq_lock is held. - */ -static taskq_ent_t * -taskq_ent_alloc(taskq_t *tq, int flags) -{ - int kmflags = (flags & TQ_NOSLEEP) ? KM_NOSLEEP : KM_SLEEP; - - taskq_ent_t *tqe; - - ASSERT(MUTEX_HELD(&tq->tq_lock)); - - /* - * TQ_NOALLOC allocations are allowed to use the freelist, even if - * we are below tq_minalloc. - */ - if ((tqe = tq->tq_freelist) != NULL && - ((flags & TQ_NOALLOC) || tq->tq_nalloc >= tq->tq_minalloc)) { - tq->tq_freelist = tqe->tqent_next; - } else { - if (flags & TQ_NOALLOC) - return (NULL); - - mutex_exit(&tq->tq_lock); - if (tq->tq_nalloc >= tq->tq_maxalloc) { - if (kmflags & KM_NOSLEEP) { - mutex_enter(&tq->tq_lock); - return (NULL); - } - /* - * We don't want to exceed tq_maxalloc, but we can't - * wait for other tasks to complete (and thus free up - * task structures) without risking deadlock with - * the caller. So, we just delay for one second - * to throttle the allocation rate. - */ - delay(hz); - } - tqe = kmem_cache_alloc(taskq_ent_cache, kmflags); - mutex_enter(&tq->tq_lock); - if (tqe != NULL) - tq->tq_nalloc++; - } - return (tqe); -} - -/* - * taskq_ent_free() - * - * Free taskq_ent_t structure by either putting it on the free list or freeing - * it to the cache. - * - * Assumes: tq->tq_lock is held. - */ -static void -taskq_ent_free(taskq_t *tq, taskq_ent_t *tqe) -{ - ASSERT(MUTEX_HELD(&tq->tq_lock)); - - if (tq->tq_nalloc <= tq->tq_minalloc) { - tqe->tqent_next = tq->tq_freelist; - tq->tq_freelist = tqe; - } else { - tq->tq_nalloc--; - mutex_exit(&tq->tq_lock); - kmem_cache_free(taskq_ent_cache, tqe); - mutex_enter(&tq->tq_lock); - } -} - -/* - * Dispatch a task. - * - * Assumes: func != NULL - * - * Returns: NULL if dispatch failed. - * non-NULL if task dispatched successfully. - * Actual return value is the pointer to taskq entry that was used to - * dispatch a task. This is useful for debugging. - */ -/* ARGSUSED */ -taskqid_t -taskq_dispatch(taskq_t *tq, task_func_t func, void *arg, uint_t flags) -{ - taskq_ent_t *tqe = NULL; - - ASSERT(tq != NULL); - ASSERT(func != NULL); - ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC)); - - /* - * TQ_NOQUEUE flag can't be used with non-dynamic task queues. - */ - ASSERT(! (flags & TQ_NOQUEUE)); - - /* - * Enqueue the task to the underlying queue. - */ - mutex_enter(&tq->tq_lock); - - TASKQ_S_RANDOM_DISPATCH_FAILURE(tq, flags); - - if ((tqe = taskq_ent_alloc(tq, flags)) == NULL) { - mutex_exit(&tq->tq_lock); - return ((taskqid_t)NULL); - } - TQ_ENQUEUE(tq, tqe, func, arg); - mutex_exit(&tq->tq_lock); - return ((taskqid_t)tqe); -} - -/* - * Wait for all pending tasks to complete. - * Calling taskq_wait from a task will cause deadlock. - */ -void -taskq_wait(taskq_t *tq) -{ - - mutex_enter(&tq->tq_lock); - while (tq->tq_task.tqent_next != &tq->tq_task || tq->tq_active != 0) - cv_wait(&tq->tq_wait_cv, &tq->tq_lock); - mutex_exit(&tq->tq_lock); -} - -/* - * Suspend execution of tasks. - * - * Tasks in the queue part will be suspended immediately upon return from this - * function. Pending tasks in the dynamic part will continue to execute, but all - * new tasks will be suspended. - */ -void -taskq_suspend(taskq_t *tq) -{ - rw_enter(&tq->tq_threadlock, RW_WRITER); - - /* - * Mark task queue as being suspended. Needed for taskq_suspended(). - */ - mutex_enter(&tq->tq_lock); - ASSERT(!(tq->tq_flags & TASKQ_SUSPENDED)); - tq->tq_flags |= TASKQ_SUSPENDED; - mutex_exit(&tq->tq_lock); -} - -/* - * returns: 1 if tq is suspended, 0 otherwise. - */ -int -taskq_suspended(taskq_t *tq) -{ - return ((tq->tq_flags & TASKQ_SUSPENDED) != 0); -} - -/* - * Resume taskq execution. - */ -void -taskq_resume(taskq_t *tq) -{ - ASSERT(RW_WRITE_HELD(&tq->tq_threadlock)); - - mutex_enter(&tq->tq_lock); - ASSERT(tq->tq_flags & TASKQ_SUSPENDED); - tq->tq_flags &= ~TASKQ_SUSPENDED; - mutex_exit(&tq->tq_lock); - - rw_exit(&tq->tq_threadlock); -} - -/* - * Worker thread for processing task queue. - */ -static void -taskq_thread(void *arg) -{ - taskq_t *tq = arg; - taskq_ent_t *tqe; - callb_cpr_t cprinfo; - hrtime_t start, end; - - CALLB_CPR_INIT(&cprinfo, &tq->tq_lock, callb_generic_cpr, tq->tq_name); - - mutex_enter(&tq->tq_lock); - while (tq->tq_flags & TASKQ_ACTIVE) { - if ((tqe = tq->tq_task.tqent_next) == &tq->tq_task) { - if (--tq->tq_active == 0) - cv_broadcast(&tq->tq_wait_cv); - if (tq->tq_flags & TASKQ_CPR_SAFE) { - cv_wait(&tq->tq_dispatch_cv, &tq->tq_lock); - } else { - CALLB_CPR_SAFE_BEGIN(&cprinfo); - cv_wait(&tq->tq_dispatch_cv, &tq->tq_lock); - CALLB_CPR_SAFE_END(&cprinfo, &tq->tq_lock); - } - tq->tq_active++; - continue; - } - tqe->tqent_prev->tqent_next = tqe->tqent_next; - tqe->tqent_next->tqent_prev = tqe->tqent_prev; - mutex_exit(&tq->tq_lock); - - rw_enter(&tq->tq_threadlock, RW_READER); - start = gethrtime(); - DTRACE_PROBE2(taskq__exec__start, taskq_t *, tq, - taskq_ent_t *, tqe); - tqe->tqent_func(tqe->tqent_arg); - DTRACE_PROBE2(taskq__exec__end, taskq_t *, tq, - taskq_ent_t *, tqe); - end = gethrtime(); - rw_exit(&tq->tq_threadlock); - - mutex_enter(&tq->tq_lock); - tq->tq_totaltime += end - start; - tq->tq_executed++; - - taskq_ent_free(tq, tqe); - } - tq->tq_nthreads--; - cv_broadcast(&tq->tq_wait_cv); - ASSERT(!(tq->tq_flags & TASKQ_CPR_SAFE)); - CALLB_CPR_EXIT(&cprinfo); - thread_exit(); -} - -/* - * Taskq creation. May sleep for memory. - * Always use automatically generated instances to avoid kstat name space - * collisions. - */ - -taskq_t * -taskq_create(const char *name, int nthreads, pri_t pri, int minalloc, - int maxalloc, uint_t flags) -{ - return taskq_create_common(name, 0, nthreads, pri, minalloc, - maxalloc, flags | TASKQ_NOINSTANCE); -} - -static taskq_t * -taskq_create_common(const char *name, int instance, int nthreads, pri_t pri, - int minalloc, int maxalloc, uint_t flags) -{ - taskq_t *tq = kmem_cache_alloc(taskq_cache, KM_SLEEP); - uint_t ncpus = ((boot_max_ncpus == -1) ? max_ncpus : boot_max_ncpus); - uint_t bsize; /* # of buckets - always power of 2 */ - - ASSERT(instance == 0); - ASSERT(flags == TASKQ_PREPOPULATE | TASKQ_NOINSTANCE); - - /* - * TASKQ_CPR_SAFE and TASKQ_DYNAMIC flags are mutually exclusive. - */ - ASSERT((flags & (TASKQ_DYNAMIC | TASKQ_CPR_SAFE)) != - ((TASKQ_DYNAMIC | TASKQ_CPR_SAFE))); - - ASSERT(tq->tq_buckets == NULL); - - bsize = 1 << (highbit(ncpus) - 1); - ASSERT(bsize >= 1); - bsize = MIN(bsize, taskq_maxbuckets); - - tq->tq_maxsize = nthreads; - - (void) strncpy(tq->tq_name, name, TASKQ_NAMELEN + 1); - tq->tq_name[TASKQ_NAMELEN] = '\0'; - /* Make sure the name conforms to the rules for C indentifiers */ - strident_canon(tq->tq_name, TASKQ_NAMELEN); - - tq->tq_flags = flags | TASKQ_ACTIVE; - tq->tq_active = nthreads; - tq->tq_nthreads = nthreads; - tq->tq_minalloc = minalloc; - tq->tq_maxalloc = maxalloc; - tq->tq_nbuckets = bsize; - tq->tq_pri = pri; - - if (flags & TASKQ_PREPOPULATE) { - mutex_enter(&tq->tq_lock); - while (minalloc-- > 0) - taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP)); - mutex_exit(&tq->tq_lock); - } - - if (nthreads == 1) { - tq->tq_thread = thread_create(NULL, 0, taskq_thread, tq, - 0, NULL, TS_RUN, pri); - } else { - kthread_t **tpp = kmem_alloc(sizeof (kthread_t *) * nthreads, - KM_SLEEP); - - tq->tq_threadlist = tpp; - - mutex_enter(&tq->tq_lock); - while (nthreads-- > 0) { - *tpp = thread_create(NULL, 0, taskq_thread, tq, - 0, NULL, TS_RUN, pri); - tpp++; - } - mutex_exit(&tq->tq_lock); - } - - return (tq); -} - -/* - * taskq_destroy(). - * - * Assumes: by the time taskq_destroy is called no one will use this task queue - * in any way and no one will try to dispatch entries in it. - */ -void -taskq_destroy(taskq_t *tq) -{ - taskq_bucket_t *b = tq->tq_buckets; - int bid = 0; - - ASSERT(! (tq->tq_flags & TASKQ_CPR_SAFE)); - - /* - * Wait for any pending entries to complete. - */ - taskq_wait(tq); - - mutex_enter(&tq->tq_lock); - ASSERT((tq->tq_task.tqent_next == &tq->tq_task) && - (tq->tq_active == 0)); - - if ((tq->tq_nthreads > 1) && (tq->tq_threadlist != NULL)) - kmem_free(tq->tq_threadlist, sizeof (kthread_t *) * - tq->tq_nthreads); - - tq->tq_flags &= ~TASKQ_ACTIVE; - cv_broadcast(&tq->tq_dispatch_cv); - while (tq->tq_nthreads != 0) - cv_wait(&tq->tq_wait_cv, &tq->tq_lock); - - tq->tq_minalloc = 0; - while (tq->tq_nalloc != 0) - taskq_ent_free(tq, taskq_ent_alloc(tq, TQ_SLEEP)); - - mutex_exit(&tq->tq_lock); - - /* - * Mark each bucket as closing and wakeup all sleeping threads. - */ - for (; (b != NULL) && (bid < tq->tq_nbuckets); b++, bid++) { - taskq_ent_t *tqe; - - mutex_enter(&b->tqbucket_lock); - - b->tqbucket_flags |= TQBUCKET_CLOSE; - /* Wakeup all sleeping threads */ - - for (tqe = b->tqbucket_freelist.tqent_next; - tqe != &b->tqbucket_freelist; tqe = tqe->tqent_next) - cv_signal(&tqe->tqent_cv); - - ASSERT(b->tqbucket_nalloc == 0); - - /* - * At this point we waited for all pending jobs to complete (in - * both the task queue and the bucket and no new jobs should - * arrive. Wait for all threads to die. - */ - while (b->tqbucket_nfree > 0) - cv_wait(&b->tqbucket_cv, &b->tqbucket_lock); - mutex_exit(&b->tqbucket_lock); - mutex_destroy(&b->tqbucket_lock); - cv_destroy(&b->tqbucket_cv); - } - - if (tq->tq_buckets != NULL) { - ASSERT(tq->tq_flags & TASKQ_DYNAMIC); - kmem_free(tq->tq_buckets, - sizeof (taskq_bucket_t) * tq->tq_nbuckets); - - /* Cleanup fields before returning tq to the cache */ - tq->tq_buckets = NULL; - tq->tq_tcreates = 0; - tq->tq_tdeaths = 0; - } else { - ASSERT(!(tq->tq_flags & TASKQ_DYNAMIC)); - } - - tq->tq_totaltime = 0; - tq->tq_tasks = 0; - tq->tq_maxtasks = 0; - tq->tq_executed = 0; - kmem_cache_free(taskq_cache, tq); -} - -SYSINIT(sol_taskq, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, taskq_init, NULL); -SYSUNINIT(sol_taskq, SI_SUB_DRIVERS, SI_ORDER_MIDDLE, taskq_fini, NULL); |
