diff options
32 files changed, 1363 insertions, 474 deletions
diff --git a/include/pthread.h b/include/pthread.h index 8078bcb..40dca7a 100644 --- a/include/pthread.h +++ b/include/pthread.h @@ -135,6 +135,9 @@ enum pthread_mutextype { #define PTHREAD_MUTEX_DEFAULT PTHREAD_MUTEX_ERRORCHECK +#define PTHREAD_MUTEX_STALLED 0 +#define PTHREAD_MUTEX_ROBUST 1 + struct _pthread_cleanup_info { __uintptr_t pthread_cleanup_pad[8]; }; @@ -229,6 +232,8 @@ int pthread_mutexattr_settype(pthread_mutexattr_t *, int) __nonnull(1); int pthread_mutexattr_setpshared(pthread_mutexattr_t *, int) __nonnull(1); +int pthread_mutex_consistent(pthread_mutex_t *__mutex) + __nonnull(1) __requires_exclusive(*__mutex); int pthread_mutex_destroy(pthread_mutex_t *__mutex) __nonnull(1) __requires_unlocked(*__mutex); int pthread_mutex_init(pthread_mutex_t *__mutex, @@ -310,6 +315,11 @@ int pthread_mutex_setprioceiling(pthread_mutex_t *, int, int *); int pthread_mutexattr_getprotocol(pthread_mutexattr_t *, int *); int pthread_mutexattr_setprotocol(pthread_mutexattr_t *, int); +int pthread_mutexattr_getrobust(pthread_mutexattr_t *__restrict, + int *__restrict) __nonnull_all; +int pthread_mutexattr_setrobust(pthread_mutexattr_t *, int) + __nonnull(1); + int pthread_attr_getinheritsched(const pthread_attr_t *, int *); int pthread_attr_getschedparam(const pthread_attr_t *, struct sched_param *) __nonnull_all; diff --git a/lib/libc/gen/Symbol.map b/lib/libc/gen/Symbol.map index ee4d619..d28991d 100644 --- a/lib/libc/gen/Symbol.map +++ b/lib/libc/gen/Symbol.map @@ -410,6 +410,9 @@ FBSD_1.3 { }; FBSD_1.4 { + pthread_mutex_consistent; + pthread_mutexattr_getrobust; + pthread_mutexattr_setrobust; scandir_b; }; diff --git a/lib/libc/gen/_pthread_stubs.c b/lib/libc/gen/_pthread_stubs.c index bd35bd2..18a2321 100644 --- a/lib/libc/gen/_pthread_stubs.c +++ b/lib/libc/gen/_pthread_stubs.c @@ -125,6 +125,9 @@ pthread_func_entry_t __thr_jtable[PJT_MAX] = { {PJT_DUAL_ENTRY(stub_zero)}, /* PJT_CLEANUP_PUSH_IMP */ {PJT_DUAL_ENTRY(stub_zero)}, /* PJT_CANCEL_ENTER */ {PJT_DUAL_ENTRY(stub_zero)}, /* PJT_CANCEL_LEAVE */ + {PJT_DUAL_ENTRY(stub_zero)}, /* PJT_MUTEX_CONSISTENT */ + {PJT_DUAL_ENTRY(stub_zero)}, /* PJT_MUTEXATTR_GETROBUST */ + {PJT_DUAL_ENTRY(stub_zero)}, /* PJT_MUTEXATTR_SETROBUST */ }; /* @@ -226,9 +229,14 @@ STUB_FUNC2(pthread_mutex_init, PJT_MUTEX_INIT, int, void *, void *) STUB_FUNC1(pthread_mutex_lock, PJT_MUTEX_LOCK, int, void *) STUB_FUNC1(pthread_mutex_trylock, PJT_MUTEX_TRYLOCK, int, void *) STUB_FUNC1(pthread_mutex_unlock, PJT_MUTEX_UNLOCK, int, void *) +STUB_FUNC1(pthread_mutex_consistent, PJT_MUTEX_CONSISTENT, int, void *) STUB_FUNC1(pthread_mutexattr_destroy, PJT_MUTEXATTR_DESTROY, int, void *) STUB_FUNC1(pthread_mutexattr_init, PJT_MUTEXATTR_INIT, int, void *) STUB_FUNC2(pthread_mutexattr_settype, PJT_MUTEXATTR_SETTYPE, int, void *, int) +STUB_FUNC2(pthread_mutexattr_getrobust, PJT_MUTEXATTR_GETROBUST, int, void *, + int *) +STUB_FUNC2(pthread_mutexattr_setrobust, PJT_MUTEXATTR_SETROBUST, int, void *, + int) STUB_FUNC2(pthread_once, PJT_ONCE, int, void *, void *) STUB_FUNC1(pthread_rwlock_destroy, PJT_RWLOCK_DESTROY, int, void *) STUB_FUNC2(pthread_rwlock_init, PJT_RWLOCK_INIT, int, void *, void *) diff --git a/lib/libc/include/libc_private.h b/lib/libc/include/libc_private.h index 8ee77d9..8fdf4a9 100644 --- a/lib/libc/include/libc_private.h +++ b/lib/libc/include/libc_private.h @@ -168,6 +168,9 @@ typedef enum { PJT_CLEANUP_PUSH_IMP, PJT_CANCEL_ENTER, PJT_CANCEL_LEAVE, + PJT_MUTEX_CONSISTENT, + PJT_MUTEXATTR_GETROBUST, + PJT_MUTEXATTR_SETROBUST, PJT_MAX } pjt_index_t; diff --git a/lib/libthr/pthread.map b/lib/libthr/pthread.map index 9fb72eb..e568393 100644 --- a/lib/libthr/pthread.map +++ b/lib/libthr/pthread.map @@ -315,3 +315,9 @@ FBSD_1.1 { FBSD_1.2 { pthread_getthreadid_np; }; + +FBSD_1.4 { + pthread_mutex_consistent; + pthread_mutexattr_getrobust; + pthread_mutexattr_setrobust; +}; diff --git a/lib/libthr/thread/thr_cond.c b/lib/libthr/thread/thr_cond.c index 0e37b70..4d9356a 100644 --- a/lib/libthr/thread/thr_cond.c +++ b/lib/libthr/thread/thr_cond.c @@ -188,46 +188,57 @@ _pthread_cond_destroy(pthread_cond_t *cond) */ static int cond_wait_kernel(struct pthread_cond *cvp, struct pthread_mutex *mp, - const struct timespec *abstime, int cancel) + const struct timespec *abstime, int cancel) { - struct pthread *curthread = _get_curthread(); - int recurse; - int error, error2 = 0; + struct pthread *curthread; + int error, error2, recurse, robust; + + curthread = _get_curthread(); + robust = _mutex_enter_robust(curthread, mp); error = _mutex_cv_detach(mp, &recurse); - if (error != 0) + if (error != 0) { + if (robust) + _mutex_leave_robust(curthread, mp); return (error); + } - if (cancel) { + if (cancel) _thr_cancel_enter2(curthread, 0); - error = _thr_ucond_wait((struct ucond *)&cvp->__has_kern_waiters, - (struct umutex *)&mp->m_lock, abstime, - CVWAIT_ABSTIME|CVWAIT_CLOCKID); + error = _thr_ucond_wait((struct ucond *)&cvp->__has_kern_waiters, + (struct umutex *)&mp->m_lock, abstime, CVWAIT_ABSTIME | + CVWAIT_CLOCKID); + if (cancel) _thr_cancel_leave(curthread, 0); - } else { - error = _thr_ucond_wait((struct ucond *)&cvp->__has_kern_waiters, - (struct umutex *)&mp->m_lock, abstime, - CVWAIT_ABSTIME|CVWAIT_CLOCKID); - } /* * Note that PP mutex and ROBUST mutex may return * interesting error codes. */ if (error == 0) { - error2 = _mutex_cv_lock(mp, recurse); + error2 = _mutex_cv_lock(mp, recurse, true); } else if (error == EINTR || error == ETIMEDOUT) { - error2 = _mutex_cv_lock(mp, recurse); + error2 = _mutex_cv_lock(mp, recurse, true); + /* + * Do not do cancellation on EOWNERDEAD there. The + * cancellation cleanup handler will use the protected + * state and unlock the mutex without making the state + * consistent and the state will be unrecoverable. + */ if (error2 == 0 && cancel) _thr_testcancel(curthread); + if (error == EINTR) error = 0; } else { /* We know that it didn't unlock the mutex. */ - error2 = _mutex_cv_attach(mp, recurse); - if (error2 == 0 && cancel) + _mutex_cv_attach(mp, recurse); + if (cancel) _thr_testcancel(curthread); + error2 = 0; } + if (robust) + _mutex_leave_robust(curthread, mp); return (error2 != 0 ? error2 : error); } @@ -240,14 +251,13 @@ cond_wait_kernel(struct pthread_cond *cvp, struct pthread_mutex *mp, static int cond_wait_user(struct pthread_cond *cvp, struct pthread_mutex *mp, - const struct timespec *abstime, int cancel) + const struct timespec *abstime, int cancel) { - struct pthread *curthread = _get_curthread(); + struct pthread *curthread; struct sleepqueue *sq; - int recurse; - int error; - int defered; + int deferred, error, error2, recurse; + curthread = _get_curthread(); if (curthread->wchan != NULL) PANIC("thread was already on queue."); @@ -260,32 +270,31 @@ cond_wait_user(struct pthread_cond *cvp, struct pthread_mutex *mp, * us to check it without locking in pthread_cond_signal(). */ cvp->__has_user_waiters = 1; - defered = 0; - (void)_mutex_cv_unlock(mp, &recurse, &defered); + deferred = 0; + (void)_mutex_cv_unlock(mp, &recurse, &deferred); curthread->mutex_obj = mp; _sleepq_add(cvp, curthread); for(;;) { _thr_clear_wake(curthread); _sleepq_unlock(cvp); - if (defered) { - defered = 0; + if (deferred) { + deferred = 0; if ((mp->m_lock.m_owner & UMUTEX_CONTESTED) == 0) - (void)_umtx_op_err(&mp->m_lock, UMTX_OP_MUTEX_WAKE2, - mp->m_lock.m_flags, 0, 0); + (void)_umtx_op_err(&mp->m_lock, + UMTX_OP_MUTEX_WAKE2, mp->m_lock.m_flags, + 0, 0); } if (curthread->nwaiter_defer > 0) { _thr_wake_all(curthread->defer_waiters, - curthread->nwaiter_defer); + curthread->nwaiter_defer); curthread->nwaiter_defer = 0; } - if (cancel) { + if (cancel) _thr_cancel_enter2(curthread, 0); - error = _thr_sleep(curthread, cvp->__clock_id, abstime); + error = _thr_sleep(curthread, cvp->__clock_id, abstime); + if (cancel) _thr_cancel_leave(curthread, 0); - } else { - error = _thr_sleep(curthread, cvp->__clock_id, abstime); - } _sleepq_lock(cvp); if (curthread->wchan == NULL) { @@ -293,25 +302,26 @@ cond_wait_user(struct pthread_cond *cvp, struct pthread_mutex *mp, break; } else if (cancel && SHOULD_CANCEL(curthread)) { sq = _sleepq_lookup(cvp); - cvp->__has_user_waiters = - _sleepq_remove(sq, curthread); + cvp->__has_user_waiters = _sleepq_remove(sq, curthread); _sleepq_unlock(cvp); curthread->mutex_obj = NULL; - _mutex_cv_lock(mp, recurse); + error2 = _mutex_cv_lock(mp, recurse, false); if (!THR_IN_CRITICAL(curthread)) _pthread_exit(PTHREAD_CANCELED); else /* this should not happen */ - return (0); + return (error2); } else if (error == ETIMEDOUT) { sq = _sleepq_lookup(cvp); cvp->__has_user_waiters = - _sleepq_remove(sq, curthread); + _sleepq_remove(sq, curthread); break; } } _sleepq_unlock(cvp); curthread->mutex_obj = NULL; - _mutex_cv_lock(mp, recurse); + error2 = _mutex_cv_lock(mp, recurse, false); + if (error == 0) + error = error2; return (error); } @@ -338,12 +348,12 @@ cond_wait_common(pthread_cond_t *cond, pthread_mutex_t *mutex, return (error); if (curthread->attr.sched_policy != SCHED_OTHER || - (mp->m_lock.m_flags & (UMUTEX_PRIO_PROTECT|UMUTEX_PRIO_INHERIT| - USYNC_PROCESS_SHARED)) != 0 || + (mp->m_lock.m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT | + USYNC_PROCESS_SHARED)) != 0 || (cvp->__flags & USYNC_PROCESS_SHARED) != 0) - return cond_wait_kernel(cvp, mp, abstime, cancel); + return (cond_wait_kernel(cvp, mp, abstime, cancel)); else - return cond_wait_user(cvp, mp, abstime, cancel); + return (cond_wait_user(cvp, mp, abstime, cancel)); } int @@ -420,15 +430,15 @@ cond_signal_common(pthread_cond_t *cond) td = _sleepq_first(sq); mp = td->mutex_obj; cvp->__has_user_waiters = _sleepq_remove(sq, td); - if (mp->m_owner == TID(curthread)) { + if (PMUTEX_OWNER_ID(mp) == TID(curthread)) { if (curthread->nwaiter_defer >= MAX_DEFER_WAITERS) { _thr_wake_all(curthread->defer_waiters, - curthread->nwaiter_defer); + curthread->nwaiter_defer); curthread->nwaiter_defer = 0; } curthread->defer_waiters[curthread->nwaiter_defer++] = - &td->wake_addr->value; - mp->m_flags |= PMUTEX_FLAG_DEFERED; + &td->wake_addr->value; + mp->m_flags |= PMUTEX_FLAG_DEFERRED; } else { waddr = &td->wake_addr->value; } @@ -452,15 +462,15 @@ drop_cb(struct pthread *td, void *arg) struct pthread *curthread = ba->curthread; mp = td->mutex_obj; - if (mp->m_owner == TID(curthread)) { + if (PMUTEX_OWNER_ID(mp) == TID(curthread)) { if (curthread->nwaiter_defer >= MAX_DEFER_WAITERS) { _thr_wake_all(curthread->defer_waiters, - curthread->nwaiter_defer); + curthread->nwaiter_defer); curthread->nwaiter_defer = 0; } curthread->defer_waiters[curthread->nwaiter_defer++] = - &td->wake_addr->value; - mp->m_flags |= PMUTEX_FLAG_DEFERED; + &td->wake_addr->value; + mp->m_flags |= PMUTEX_FLAG_DEFERRED; } else { if (ba->count >= MAX_DEFER_WAITERS) { _thr_wake_all(ba->waddrs, ba->count); diff --git a/lib/libthr/thread/thr_init.c b/lib/libthr/thread/thr_init.c index c852406..0927e15 100644 --- a/lib/libthr/thread/thr_init.c +++ b/lib/libthr/thread/thr_init.c @@ -94,6 +94,7 @@ struct pthread_mutex_attr _pthread_mutexattr_default = { .m_protocol = PTHREAD_PRIO_NONE, .m_ceiling = 0, .m_pshared = PTHREAD_PROCESS_PRIVATE, + .m_robust = PTHREAD_MUTEX_STALLED, }; struct pthread_mutex_attr _pthread_mutexattr_adaptive_default = { @@ -101,6 +102,7 @@ struct pthread_mutex_attr _pthread_mutexattr_adaptive_default = { .m_protocol = PTHREAD_PRIO_NONE, .m_ceiling = 0, .m_pshared = PTHREAD_PROCESS_PRIVATE, + .m_robust = PTHREAD_MUTEX_STALLED, }; /* Default condition variable attributes: */ @@ -265,7 +267,10 @@ static pthread_func_t jmp_table[][2] = { {DUAL_ENTRY(__pthread_cleanup_pop_imp)},/* PJT_CLEANUP_POP_IMP */ {DUAL_ENTRY(__pthread_cleanup_push_imp)},/* PJT_CLEANUP_PUSH_IMP */ {DUAL_ENTRY(_pthread_cancel_enter)}, /* PJT_CANCEL_ENTER */ - {DUAL_ENTRY(_pthread_cancel_leave)} /* PJT_CANCEL_LEAVE */ + {DUAL_ENTRY(_pthread_cancel_leave)}, /* PJT_CANCEL_LEAVE */ + {DUAL_ENTRY(_pthread_mutex_consistent)},/* PJT_MUTEX_CONSISTENT */ + {DUAL_ENTRY(_pthread_mutexattr_getrobust)},/* PJT_MUTEXATTR_GETROBUST */ + {DUAL_ENTRY(_pthread_mutexattr_setrobust)},/* PJT_MUTEXATTR_SETROBUST */ }; static int init_once = 0; @@ -308,7 +313,7 @@ _libpthread_init(struct pthread *curthread) int first, dlopened; /* Check if this function has already been called: */ - if ((_thr_initial != NULL) && (curthread == NULL)) + if (_thr_initial != NULL && curthread == NULL) /* Only initialize the threaded application once. */ return; @@ -316,7 +321,7 @@ _libpthread_init(struct pthread *curthread) * Check the size of the jump table to make sure it is preset * with the correct number of entries. */ - if (sizeof(jmp_table) != (sizeof(pthread_func_t) * PJT_MAX * 2)) + if (sizeof(jmp_table) != sizeof(pthread_func_t) * PJT_MAX * 2) PANIC("Thread jump table not properly initialized"); memcpy(__thr_jtable, jmp_table, sizeof(jmp_table)); __thr_interpose_libc(); diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c index f75ea6f..2d507e7 100644 --- a/lib/libthr/thread/thr_mutex.c +++ b/lib/libthr/thread/thr_mutex.c @@ -1,7 +1,7 @@ /* * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>. * Copyright (c) 2006 David Xu <davidxu@freebsd.org>. - * Copyright (c) 2015 The FreeBSD Foundation + * Copyright (c) 2015, 2016 The FreeBSD Foundation * * All rights reserved. * @@ -39,7 +39,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include <stdbool.h> #include "namespace.h" #include <stdlib.h> #include <errno.h> @@ -64,6 +63,7 @@ _Static_assert(sizeof(struct pthread_mutex) <= PAGE_SIZE, /* * Prototypes */ +int __pthread_mutex_consistent(pthread_mutex_t *mutex); int __pthread_mutex_init(pthread_mutex_t *mutex, const pthread_mutexattr_t *mutex_attr); int __pthread_mutex_trylock(pthread_mutex_t *mutex); @@ -82,9 +82,13 @@ int __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count); static int mutex_self_trylock(pthread_mutex_t); static int mutex_self_lock(pthread_mutex_t, const struct timespec *abstime); -static int mutex_unlock_common(struct pthread_mutex *, int, int *); +static int mutex_unlock_common(struct pthread_mutex *, bool, int *); static int mutex_lock_sleep(struct pthread *, pthread_mutex_t, const struct timespec *); +static void mutex_init_robust(struct pthread *curthread); +static int mutex_qidx(struct pthread_mutex *m); +static bool is_robust_mutex(struct pthread_mutex *m); +static bool is_pshared_mutex(struct pthread_mutex *m); __weak_reference(__pthread_mutex_init, pthread_mutex_init); __strong_reference(__pthread_mutex_init, _pthread_mutex_init); @@ -94,6 +98,8 @@ __weak_reference(__pthread_mutex_timedlock, pthread_mutex_timedlock); __strong_reference(__pthread_mutex_timedlock, _pthread_mutex_timedlock); __weak_reference(__pthread_mutex_trylock, pthread_mutex_trylock); __strong_reference(__pthread_mutex_trylock, _pthread_mutex_trylock); +__weak_reference(_pthread_mutex_consistent, pthread_mutex_consistent); +__strong_reference(_pthread_mutex_consistent, __pthread_mutex_consistent); /* Single underscore versions provided for libc internal usage: */ /* No difference between libc and application usage of these: */ @@ -125,23 +131,23 @@ mutex_init_link(struct pthread_mutex *m) } static void -mutex_assert_is_owned(struct pthread_mutex *m) +mutex_assert_is_owned(struct pthread_mutex *m __unused) { #if defined(_PTHREADS_INVARIANTS) if (__predict_false(m->m_qe.tqe_prev == NULL)) { char msg[128]; snprintf(msg, sizeof(msg), - "mutex %p own %#x %#x is not on list %p %p", - m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev, - m->m_qe.tqe_next); + "mutex %p own %#x is not on list %p %p", + m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next); PANIC(msg); } #endif } static void -mutex_assert_not_owned(struct pthread_mutex *m) +mutex_assert_not_owned(struct pthread *curthread __unused, + struct pthread_mutex *m __unused) { #if defined(_PTHREADS_INVARIANTS) @@ -149,21 +155,68 @@ mutex_assert_not_owned(struct pthread_mutex *m) m->m_qe.tqe_next != NULL)) { char msg[128]; snprintf(msg, sizeof(msg), - "mutex %p own %#x %#x is on list %p %p", - m, m->m_lock.m_owner, m->m_owner, m->m_qe.tqe_prev, - m->m_qe.tqe_next); + "mutex %p own %#x is on list %p %p", + m, m->m_lock.m_owner, m->m_qe.tqe_prev, m->m_qe.tqe_next); + PANIC(msg); + } + if (__predict_false(is_robust_mutex(m) && + (m->m_lock.m_rb_lnk != 0 || m->m_rb_prev != NULL || + (is_pshared_mutex(m) && curthread->robust_list == + (uintptr_t)&m->m_lock) || + (!is_pshared_mutex(m) && curthread->priv_robust_list == + (uintptr_t)&m->m_lock)))) { + char msg[128]; + snprintf(msg, sizeof(msg), + "mutex %p own %#x is on robust linkage %p %p head %p phead %p", + m, m->m_lock.m_owner, (void *)m->m_lock.m_rb_lnk, + m->m_rb_prev, (void *)curthread->robust_list, + (void *)curthread->priv_robust_list); PANIC(msg); } #endif } -static int +static bool is_pshared_mutex(struct pthread_mutex *m) { return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0); } +static bool +is_robust_mutex(struct pthread_mutex *m) +{ + + return ((m->m_lock.m_flags & UMUTEX_ROBUST) != 0); +} + +int +_mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m) +{ + +#if defined(_PTHREADS_INVARIANTS) + if (__predict_false(curthread->inact_mtx != 0)) + PANIC("inact_mtx enter"); +#endif + if (!is_robust_mutex(m)) + return (0); + + mutex_init_robust(curthread); + curthread->inact_mtx = (uintptr_t)&m->m_lock; + return (1); +} + +void +_mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m __unused) +{ + +#if defined(_PTHREADS_INVARIANTS) + if (__predict_false(curthread->inact_mtx != (uintptr_t)&m->m_lock)) + PANIC("inact_mtx leave"); +#endif + curthread->inact_mtx = 0; +} + static int mutex_check_attr(const struct pthread_mutex_attr *attr) { @@ -178,12 +231,27 @@ mutex_check_attr(const struct pthread_mutex_attr *attr) } static void +mutex_init_robust(struct pthread *curthread) +{ + struct umtx_robust_lists_params rb; + + if (curthread == NULL) + curthread = _get_curthread(); + if (curthread->robust_inited) + return; + rb.robust_list_offset = (uintptr_t)&curthread->robust_list; + rb.robust_priv_list_offset = (uintptr_t)&curthread->priv_robust_list; + rb.robust_inact_offset = (uintptr_t)&curthread->inact_mtx; + _umtx_op(NULL, UMTX_OP_ROBUST_LISTS, sizeof(rb), &rb, NULL); + curthread->robust_inited = 1; +} + +static void mutex_init_body(struct pthread_mutex *pmutex, const struct pthread_mutex_attr *attr) { pmutex->m_flags = attr->m_type; - pmutex->m_owner = 0; pmutex->m_count = 0; pmutex->m_spinloops = 0; pmutex->m_yieldloops = 0; @@ -205,7 +273,10 @@ mutex_init_body(struct pthread_mutex *pmutex, } if (attr->m_pshared == PTHREAD_PROCESS_SHARED) pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED; - + if (attr->m_robust == PTHREAD_MUTEX_ROBUST) { + mutex_init_robust(NULL); + pmutex->m_lock.m_flags |= UMUTEX_ROBUST; + } if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) { pmutex->m_spinloops = _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS; @@ -262,7 +333,7 @@ set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m) { struct pthread_mutex *m2; - m2 = TAILQ_LAST(&curthread->mq[TMQ_NORM_PP], mutex_queue); + m2 = TAILQ_LAST(&curthread->mq[mutex_qidx(m)], mutex_queue); if (m2 != NULL) m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0]; else @@ -277,7 +348,8 @@ shared_mutex_init(struct pthread_mutex *pmtx, const struct .m_type = PTHREAD_MUTEX_DEFAULT, .m_protocol = PTHREAD_PRIO_NONE, .m_ceiling = 0, - .m_pshared = PTHREAD_PROCESS_SHARED + .m_pshared = PTHREAD_PROCESS_SHARED, + .m_robust = PTHREAD_MUTEX_STALLED, }; bool done; @@ -329,7 +401,7 @@ __pthread_mutex_init(pthread_mutex_t *mutex, if (mutex_attr == NULL || (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) { return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, - calloc)); + calloc)); } pmtx = __thr_pshared_offpage(mutex, 1); if (pmtx == NULL) @@ -349,6 +421,7 @@ _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex, .m_protocol = PTHREAD_PRIO_NONE, .m_ceiling = 0, .m_pshared = PTHREAD_PROCESS_PRIVATE, + .m_robust = PTHREAD_MUTEX_STALLED, }; int ret; @@ -378,7 +451,6 @@ queue_fork(struct pthread *curthread, struct mutex_queue *q, TAILQ_FOREACH(m, qp, m_pqe) { TAILQ_INSERT_TAIL(q, m, m_qe); m->m_lock.m_owner = TID(curthread) | bit; - m->m_owner = TID(curthread); } } @@ -390,6 +462,9 @@ _mutex_fork(struct pthread *curthread) &curthread->mq[TMQ_NORM_PRIV], 0); queue_fork(curthread, &curthread->mq[TMQ_NORM_PP], &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED); + queue_fork(curthread, &curthread->mq[TMQ_ROBUST_PP], + &curthread->mq[TMQ_ROBUST_PP_PRIV], UMUTEX_CONTESTED); + curthread->robust_list = 0; } int @@ -407,17 +482,18 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex) if (m == THR_PSHARED_PTR) { m1 = __thr_pshared_offpage(mutex, 0); if (m1 != NULL) { - mutex_assert_not_owned(m1); + mutex_assert_not_owned(_get_curthread(), m1); __thr_pshared_destroy(mutex); } *mutex = THR_MUTEX_DESTROYED; return (0); } - if (m->m_owner != 0) { + if (PMUTEX_OWNER_ID(m) != 0 && + (uint32_t)m->m_lock.m_owner != UMUTEX_RB_NOTRECOV) { ret = EBUSY; } else { *mutex = THR_MUTEX_DESTROYED; - mutex_assert_not_owned(m); + mutex_assert_not_owned(_get_curthread(), m); free(m); ret = 0; } @@ -432,31 +508,81 @@ mutex_qidx(struct pthread_mutex *m) if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0) return (TMQ_NORM); - return (TMQ_NORM_PP); + return (is_robust_mutex(m) ? TMQ_ROBUST_PP : TMQ_NORM_PP); } +/* + * Both enqueue_mutex() and dequeue_mutex() operate on the + * thread-private linkage of the locked mutexes and on the robust + * linkage. + * + * Robust list, as seen by kernel, must be consistent even in the case + * of thread termination at arbitrary moment. Since either enqueue or + * dequeue for list walked by kernel consists of rewriting a single + * forward pointer, it is safe. On the other hand, rewrite of the + * back pointer is not atomic WRT the forward one, but kernel does not + * care. + */ static void -enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m) +enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m, + int error) { + struct pthread_mutex *m1; + uintptr_t *rl; int qidx; - m->m_owner = TID(curthread); /* Add to the list of owned mutexes: */ - mutex_assert_not_owned(m); + if (error != EOWNERDEAD) + mutex_assert_not_owned(curthread, m); qidx = mutex_qidx(m); TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe); if (!is_pshared_mutex(m)) TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe); + if (is_robust_mutex(m)) { + rl = is_pshared_mutex(m) ? &curthread->robust_list : + &curthread->priv_robust_list; + m->m_rb_prev = NULL; + if (*rl != 0) { + m1 = __containerof((void *)*rl, + struct pthread_mutex, m_lock); + m->m_lock.m_rb_lnk = (uintptr_t)&m1->m_lock; + m1->m_rb_prev = m; + } else { + m1 = NULL; + m->m_lock.m_rb_lnk = 0; + } + *rl = (uintptr_t)&m->m_lock; + } } static void dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m) { + struct pthread_mutex *mp, *mn; int qidx; - m->m_owner = 0; mutex_assert_is_owned(m); qidx = mutex_qidx(m); + if (is_robust_mutex(m)) { + mp = m->m_rb_prev; + if (mp == NULL) { + if (is_pshared_mutex(m)) { + curthread->robust_list = m->m_lock.m_rb_lnk; + } else { + curthread->priv_robust_list = + m->m_lock.m_rb_lnk; + } + } else { + mp->m_lock.m_rb_lnk = m->m_lock.m_rb_lnk; + } + if (m->m_lock.m_rb_lnk != 0) { + mn = __containerof((void *)m->m_lock.m_rb_lnk, + struct pthread_mutex, m_lock); + mn->m_rb_prev = m->m_rb_prev; + } + m->m_lock.m_rb_lnk = 0; + m->m_rb_prev = NULL; + } TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe); if (!is_pshared_mutex(m)) TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe); @@ -496,7 +622,7 @@ __pthread_mutex_trylock(pthread_mutex_t *mutex) struct pthread *curthread; struct pthread_mutex *m; uint32_t id; - int ret; + int ret, robust; ret = check_and_init_mutex(mutex, &m); if (ret != 0) @@ -505,27 +631,32 @@ __pthread_mutex_trylock(pthread_mutex_t *mutex) id = TID(curthread); if (m->m_flags & PMUTEX_FLAG_PRIVATE) THR_CRITICAL_ENTER(curthread); + robust = _mutex_enter_robust(curthread, m); ret = _thr_umutex_trylock(&m->m_lock, id); - if (__predict_true(ret == 0)) { - enqueue_mutex(curthread, m); - } else if (m->m_owner == id) { + if (__predict_true(ret == 0) || ret == EOWNERDEAD) { + enqueue_mutex(curthread, m, ret); + if (ret == EOWNERDEAD) + m->m_lock.m_flags |= UMUTEX_NONCONSISTENT; + } else if (PMUTEX_OWNER_ID(m) == id) { ret = mutex_self_trylock(m); } /* else {} */ - if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE)) + if (robust) + _mutex_leave_robust(curthread, m); + if ((ret == 0 || ret == EOWNERDEAD) && + (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0) THR_CRITICAL_LEAVE(curthread); return (ret); } static int mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, - const struct timespec *abstime) + const struct timespec *abstime) { - uint32_t id, owner; - int count; - int ret; + uint32_t id, owner; + int count, ret; id = TID(curthread); - if (m->m_owner == id) + if (PMUTEX_OWNER_ID(m) == id) return (mutex_self_lock(m, abstime)); /* @@ -534,10 +665,9 @@ mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, * the lock is likely to be released quickly and it is * faster than entering the kernel */ - if (__predict_false( - (m->m_lock.m_flags & - (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0)) - goto sleep_in_kernel; + if (__predict_false((m->m_lock.m_flags & (UMUTEX_PRIO_PROTECT | + UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) != 0)) + goto sleep_in_kernel; if (!_thr_is_smp) goto yield_loop; @@ -546,7 +676,8 @@ mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m, while (count--) { owner = m->m_lock.m_owner; if ((owner & ~UMUTEX_CONTESTED) == 0) { - if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { + if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, + id | owner)) { ret = 0; goto done; } @@ -560,7 +691,8 @@ yield_loop: _sched_yield(); owner = m->m_lock.m_owner; if ((owner & ~UMUTEX_CONTESTED) == 0) { - if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) { + if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, + id | owner)) { ret = 0; goto done; } @@ -568,38 +700,46 @@ yield_loop: } sleep_in_kernel: - if (abstime == NULL) { + if (abstime == NULL) ret = __thr_umutex_lock(&m->m_lock, id); - } else if (__predict_false( - abstime->tv_nsec < 0 || - abstime->tv_nsec >= 1000000000)) { + else if (__predict_false(abstime->tv_nsec < 0 || + abstime->tv_nsec >= 1000000000)) ret = EINVAL; - } else { + else ret = __thr_umutex_timedlock(&m->m_lock, id, abstime); - } done: - if (ret == 0) - enqueue_mutex(curthread, m); - + if (ret == 0 || ret == EOWNERDEAD) { + enqueue_mutex(curthread, m, ret); + if (ret == EOWNERDEAD) + m->m_lock.m_flags |= UMUTEX_NONCONSISTENT; + } return (ret); } static inline int -mutex_lock_common(struct pthread_mutex *m, - const struct timespec *abstime, int cvattach) +mutex_lock_common(struct pthread_mutex *m, const struct timespec *abstime, + bool cvattach, bool rb_onlist) { - struct pthread *curthread = _get_curthread(); - int ret; + struct pthread *curthread; + int ret, robust; + curthread = _get_curthread(); if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE) THR_CRITICAL_ENTER(curthread); - if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) { - enqueue_mutex(curthread, m); - ret = 0; + if (!rb_onlist) + robust = _mutex_enter_robust(curthread, m); + ret = _thr_umutex_trylock2(&m->m_lock, TID(curthread)); + if (ret == 0 || ret == EOWNERDEAD) { + enqueue_mutex(curthread, m, ret); + if (ret == EOWNERDEAD) + m->m_lock.m_flags |= UMUTEX_NONCONSISTENT; } else { ret = mutex_lock_sleep(curthread, m, abstime); } - if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE) && !cvattach) + if (!rb_onlist && robust) + _mutex_leave_robust(curthread, m); + if (ret != 0 && ret != EOWNERDEAD && + (m->m_flags & PMUTEX_FLAG_PRIVATE) != 0 && !cvattach) THR_CRITICAL_LEAVE(curthread); return (ret); } @@ -613,7 +753,7 @@ __pthread_mutex_lock(pthread_mutex_t *mutex) _thr_check_init(); ret = check_and_init_mutex(mutex, &m); if (ret == 0) - ret = mutex_lock_common(m, NULL, 0); + ret = mutex_lock_common(m, NULL, false, false); return (ret); } @@ -627,7 +767,7 @@ __pthread_mutex_timedlock(pthread_mutex_t *mutex, _thr_check_init(); ret = check_and_init_mutex(mutex, &m); if (ret == 0) - ret = mutex_lock_common(m, abstime, 0); + ret = mutex_lock_common(m, abstime, false, false); return (ret); } @@ -644,16 +784,16 @@ _pthread_mutex_unlock(pthread_mutex_t *mutex) } else { mp = *mutex; } - return (mutex_unlock_common(mp, 0, NULL)); + return (mutex_unlock_common(mp, false, NULL)); } int -_mutex_cv_lock(struct pthread_mutex *m, int count) +_mutex_cv_lock(struct pthread_mutex *m, int count, bool rb_onlist) { - int error; + int error; - error = mutex_lock_common(m, NULL, 1); - if (error == 0) + error = mutex_lock_common(m, NULL, true, rb_onlist); + if (error == 0 || error == EOWNERDEAD) m->m_count = count; return (error); } @@ -667,16 +807,17 @@ _mutex_cv_unlock(struct pthread_mutex *m, int *count, int *defer) */ *count = m->m_count; m->m_count = 0; - (void)mutex_unlock_common(m, 1, defer); + (void)mutex_unlock_common(m, true, defer); return (0); } int _mutex_cv_attach(struct pthread_mutex *m, int count) { - struct pthread *curthread = _get_curthread(); + struct pthread *curthread; - enqueue_mutex(curthread, m); + curthread = _get_curthread(); + enqueue_mutex(curthread, m, 0); m->m_count = count; return (0); } @@ -684,12 +825,12 @@ _mutex_cv_attach(struct pthread_mutex *m, int count) int _mutex_cv_detach(struct pthread_mutex *mp, int *recurse) { - struct pthread *curthread = _get_curthread(); - int defered; - int error; + struct pthread *curthread; + int deferred, error; + curthread = _get_curthread(); if ((error = _mutex_owned(curthread, mp)) != 0) - return (error); + return (error); /* * Clear the count in case this is a recursive mutex. @@ -699,15 +840,15 @@ _mutex_cv_detach(struct pthread_mutex *mp, int *recurse) dequeue_mutex(curthread, mp); /* Will this happen in real-world ? */ - if ((mp->m_flags & PMUTEX_FLAG_DEFERED) != 0) { - defered = 1; - mp->m_flags &= ~PMUTEX_FLAG_DEFERED; + if ((mp->m_flags & PMUTEX_FLAG_DEFERRED) != 0) { + deferred = 1; + mp->m_flags &= ~PMUTEX_FLAG_DEFERRED; } else - defered = 0; + deferred = 0; - if (defered) { + if (deferred) { _thr_wake_all(curthread->defer_waiters, - curthread->nwaiter_defer); + curthread->nwaiter_defer); curthread->nwaiter_defer = 0; } return (0); @@ -716,7 +857,7 @@ _mutex_cv_detach(struct pthread_mutex *mp, int *recurse) static int mutex_self_trylock(struct pthread_mutex *m) { - int ret; + int ret; switch (PMUTEX_TYPE(m->m_flags)) { case PTHREAD_MUTEX_ERRORCHECK: @@ -746,7 +887,7 @@ static int mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime) { struct timespec ts1, ts2; - int ret; + int ret; switch (PMUTEX_TYPE(m->m_flags)) { case PTHREAD_MUTEX_ERRORCHECK: @@ -812,11 +953,11 @@ mutex_self_lock(struct pthread_mutex *m, const struct timespec *abstime) } static int -mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer) +mutex_unlock_common(struct pthread_mutex *m, bool cv, int *mtx_defer) { - struct pthread *curthread = _get_curthread(); + struct pthread *curthread; uint32_t id; - int defered, error; + int deferred, error, robust; if (__predict_false(m <= THR_MUTEX_DESTROYED)) { if (m == THR_MUTEX_DESTROYED) @@ -824,34 +965,39 @@ mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer) return (EPERM); } + curthread = _get_curthread(); id = TID(curthread); /* * Check if the running thread is not the owner of the mutex. */ - if (__predict_false(m->m_owner != id)) + if (__predict_false(PMUTEX_OWNER_ID(m) != id)) return (EPERM); error = 0; - if (__predict_false( - PMUTEX_TYPE(m->m_flags) == PTHREAD_MUTEX_RECURSIVE && - m->m_count > 0)) { + if (__predict_false(PMUTEX_TYPE(m->m_flags) == + PTHREAD_MUTEX_RECURSIVE && m->m_count > 0)) { m->m_count--; } else { - if ((m->m_flags & PMUTEX_FLAG_DEFERED) != 0) { - defered = 1; - m->m_flags &= ~PMUTEX_FLAG_DEFERED; + if ((m->m_flags & PMUTEX_FLAG_DEFERRED) != 0) { + deferred = 1; + m->m_flags &= ~PMUTEX_FLAG_DEFERRED; } else - defered = 0; + deferred = 0; + robust = _mutex_enter_robust(curthread, m); dequeue_mutex(curthread, m); error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer); - - if (mtx_defer == NULL && defered) { - _thr_wake_all(curthread->defer_waiters, - curthread->nwaiter_defer); - curthread->nwaiter_defer = 0; + if (deferred) { + if (mtx_defer == NULL) { + _thr_wake_all(curthread->defer_waiters, + curthread->nwaiter_defer); + curthread->nwaiter_defer = 0; + } else + *mtx_defer = 1; } + if (robust) + _mutex_leave_robust(curthread, m); } if (!cv && m->m_flags & PMUTEX_FLAG_PRIVATE) THR_CRITICAL_LEAVE(curthread); @@ -887,7 +1033,7 @@ _pthread_mutex_setprioceiling(pthread_mutex_t *mutex, struct pthread *curthread; struct pthread_mutex *m, *m1, *m2; struct mutex_queue *q, *qp; - int ret; + int qidx, ret; if (*mutex == THR_PSHARED_PTR) { m = __thr_pshared_offpage(mutex, 0); @@ -907,14 +1053,15 @@ _pthread_mutex_setprioceiling(pthread_mutex_t *mutex, return (ret); curthread = _get_curthread(); - if (m->m_owner == TID(curthread)) { + if (PMUTEX_OWNER_ID(m) == TID(curthread)) { mutex_assert_is_owned(m); m1 = TAILQ_PREV(m, mutex_queue, m_qe); m2 = TAILQ_NEXT(m, m_qe); if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) || (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) { - q = &curthread->mq[TMQ_NORM_PP]; - qp = &curthread->mq[TMQ_NORM_PP_PRIV]; + qidx = mutex_qidx(m); + q = &curthread->mq[qidx]; + qp = &curthread->mq[qidx + 1]; TAILQ_REMOVE(q, m, m_qe); if (!is_pshared_mutex(m)) TAILQ_REMOVE(qp, m, m_pqe); @@ -1009,18 +1156,45 @@ _pthread_mutex_isowned_np(pthread_mutex_t *mutex) if (m <= THR_MUTEX_DESTROYED) return (0); } - return (m->m_owner == TID(_get_curthread())); + return (PMUTEX_OWNER_ID(m) == TID(_get_curthread())); } int _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp) { + if (__predict_false(mp <= THR_MUTEX_DESTROYED)) { if (mp == THR_MUTEX_DESTROYED) return (EINVAL); return (EPERM); } - if (mp->m_owner != TID(curthread)) + if (PMUTEX_OWNER_ID(mp) != TID(curthread)) return (EPERM); return (0); } + +int +_pthread_mutex_consistent(pthread_mutex_t *mutex) +{ + struct pthread_mutex *m; + struct pthread *curthread; + + if (*mutex == THR_PSHARED_PTR) { + m = __thr_pshared_offpage(mutex, 0); + if (m == NULL) + return (EINVAL); + shared_mutex_init(m, NULL); + } else { + m = *mutex; + if (m <= THR_MUTEX_DESTROYED) + return (EINVAL); + } + curthread = _get_curthread(); + if ((m->m_lock.m_flags & (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) != + (UMUTEX_ROBUST | UMUTEX_NONCONSISTENT)) + return (EINVAL); + if (PMUTEX_OWNER_ID(m) != TID(curthread)) + return (EPERM); + m->m_lock.m_flags &= ~UMUTEX_NONCONSISTENT; + return (0); +} diff --git a/lib/libthr/thread/thr_mutexattr.c b/lib/libthr/thread/thr_mutexattr.c index a9e07c2..d8a8671 100644 --- a/lib/libthr/thread/thr_mutexattr.c +++ b/lib/libthr/thread/thr_mutexattr.c @@ -80,8 +80,12 @@ __weak_reference(_pthread_mutexattr_getpshared, pthread_mutexattr_getpshared); __weak_reference(_pthread_mutexattr_setpshared, pthread_mutexattr_setpshared); __weak_reference(_pthread_mutexattr_getprotocol, pthread_mutexattr_getprotocol); __weak_reference(_pthread_mutexattr_setprotocol, pthread_mutexattr_setprotocol); -__weak_reference(_pthread_mutexattr_getprioceiling, pthread_mutexattr_getprioceiling); -__weak_reference(_pthread_mutexattr_setprioceiling, pthread_mutexattr_setprioceiling); +__weak_reference(_pthread_mutexattr_getprioceiling, + pthread_mutexattr_getprioceiling); +__weak_reference(_pthread_mutexattr_setprioceiling, + pthread_mutexattr_setprioceiling); +__weak_reference(_pthread_mutexattr_getrobust, pthread_mutexattr_getrobust); +__weak_reference(_pthread_mutexattr_setrobust, pthread_mutexattr_setrobust); int _pthread_mutexattr_init(pthread_mutexattr_t *attr) @@ -119,26 +123,28 @@ int _pthread_mutexattr_getkind_np(pthread_mutexattr_t attr) { int ret; + if (attr == NULL) { errno = EINVAL; ret = -1; } else { ret = attr->m_type; } - return(ret); + return (ret); } int _pthread_mutexattr_settype(pthread_mutexattr_t *attr, int type) { int ret; + if (attr == NULL || *attr == NULL || type >= PTHREAD_MUTEX_TYPE_MAX) { ret = EINVAL; } else { (*attr)->m_type = type; ret = 0; } - return(ret); + return (ret); } int @@ -153,7 +159,7 @@ _pthread_mutexattr_gettype(pthread_mutexattr_t *attr, int *type) *type = (*attr)->m_type; ret = 0; } - return ret; + return (ret); } int @@ -167,7 +173,7 @@ _pthread_mutexattr_destroy(pthread_mutexattr_t *attr) *attr = NULL; ret = 0; } - return(ret); + return (ret); } int @@ -198,12 +204,12 @@ _pthread_mutexattr_getprotocol(pthread_mutexattr_t *mattr, int *protocol) { int ret = 0; - if ((mattr == NULL) || (*mattr == NULL)) + if (mattr == NULL || *mattr == NULL) ret = EINVAL; else *protocol = (*mattr)->m_protocol; - return(ret); + return (ret); } int @@ -211,14 +217,14 @@ _pthread_mutexattr_setprotocol(pthread_mutexattr_t *mattr, int protocol) { int ret = 0; - if ((mattr == NULL) || (*mattr == NULL) || - (protocol < PTHREAD_PRIO_NONE) || (protocol > PTHREAD_PRIO_PROTECT)) + if (mattr == NULL || *mattr == NULL || + protocol < PTHREAD_PRIO_NONE || protocol > PTHREAD_PRIO_PROTECT) ret = EINVAL; else { (*mattr)->m_protocol = protocol; (*mattr)->m_ceiling = THR_MAX_RR_PRIORITY; } - return(ret); + return (ret); } int @@ -226,14 +232,14 @@ _pthread_mutexattr_getprioceiling(pthread_mutexattr_t *mattr, int *prioceiling) { int ret = 0; - if ((mattr == NULL) || (*mattr == NULL)) + if (mattr == NULL || *mattr == NULL) ret = EINVAL; else if ((*mattr)->m_protocol != PTHREAD_PRIO_PROTECT) ret = EINVAL; else *prioceiling = (*mattr)->m_ceiling; - return(ret); + return (ret); } int @@ -241,13 +247,44 @@ _pthread_mutexattr_setprioceiling(pthread_mutexattr_t *mattr, int prioceiling) { int ret = 0; - if ((mattr == NULL) || (*mattr == NULL)) + if (mattr == NULL || *mattr == NULL) ret = EINVAL; else if ((*mattr)->m_protocol != PTHREAD_PRIO_PROTECT) ret = EINVAL; else (*mattr)->m_ceiling = prioceiling; - return(ret); + return (ret); +} + +int +_pthread_mutexattr_getrobust(pthread_mutexattr_t *mattr, int *robust) +{ + int ret; + + if (mattr == NULL || *mattr == NULL) { + ret = EINVAL; + } else { + ret = 0; + *robust = (*mattr)->m_robust; + } + return (ret); +} + +int +_pthread_mutexattr_setrobust(pthread_mutexattr_t *mattr, int robust) +{ + int ret; + + if (mattr == NULL || *mattr == NULL) { + ret = EINVAL; + } else if (robust != PTHREAD_MUTEX_STALLED && + robust != PTHREAD_MUTEX_ROBUST) { + ret = EINVAL; + } else { + ret = 0; + (*mattr)->m_robust = robust; + } + return (ret); } diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h index f35d3cd..21399f3 100644 --- a/lib/libthr/thread/thr_private.h +++ b/lib/libthr/thread/thr_private.h @@ -45,6 +45,7 @@ #include <errno.h> #include <limits.h> #include <signal.h> +#include <stdbool.h> #include <stddef.h> #include <stdio.h> #include <unistd.h> @@ -141,9 +142,11 @@ TAILQ_HEAD(mutex_queue, pthread_mutex); #define PMUTEX_FLAG_TYPE_MASK 0x0ff #define PMUTEX_FLAG_PRIVATE 0x100 -#define PMUTEX_FLAG_DEFERED 0x200 +#define PMUTEX_FLAG_DEFERRED 0x200 #define PMUTEX_TYPE(mtxflags) ((mtxflags) & PMUTEX_FLAG_TYPE_MASK) +#define PMUTEX_OWNER_ID(m) ((m)->m_lock.m_owner & ~UMUTEX_CONTESTED) + #define MAX_DEFER_WAITERS 50 /* @@ -159,7 +162,6 @@ struct pthread_mutex { */ struct umutex m_lock; int m_flags; - uint32_t m_owner; int m_count; int m_spinloops; int m_yieldloops; @@ -171,6 +173,7 @@ struct pthread_mutex { TAILQ_ENTRY(pthread_mutex) m_qe; /* Link for all private mutexes a thread currently owns. */ TAILQ_ENTRY(pthread_mutex) m_pqe; + struct pthread_mutex *m_rb_prev; }; struct pthread_mutex_attr { @@ -178,10 +181,12 @@ struct pthread_mutex_attr { int m_protocol; int m_ceiling; int m_pshared; + int m_robust; }; #define PTHREAD_MUTEXATTR_STATIC_INITIALIZER \ - { PTHREAD_MUTEX_DEFAULT, PTHREAD_PRIO_NONE, 0, MUTEX_FLAGS_PRIVATE } + { PTHREAD_MUTEX_DEFAULT, PTHREAD_PRIO_NONE, 0, MUTEX_FLAGS_PRIVATE, \ + PTHREAD_MUTEX_STALLED } struct pthread_cond { __uint32_t __has_user_waiters; @@ -491,7 +496,9 @@ struct pthread { #define TMQ_NORM_PRIV 1 /* NORMAL or PRIO_INHERIT normal priv */ #define TMQ_NORM_PP 2 /* PRIO_PROTECT normal mutexes */ #define TMQ_NORM_PP_PRIV 3 /* PRIO_PROTECT normal priv */ -#define TMQ_NITEMS 4 +#define TMQ_ROBUST_PP 4 /* PRIO_PROTECT robust mutexes */ +#define TMQ_ROBUST_PP_PRIV 5 /* PRIO_PROTECT robust priv */ +#define TMQ_NITEMS 6 struct mutex_queue mq[TMQ_NITEMS]; void *ret; @@ -545,6 +552,11 @@ struct pthread { /* Number of threads deferred. */ int nwaiter_defer; + int robust_inited; + uintptr_t robust_list; + uintptr_t priv_robust_list; + uintptr_t inact_mtx; + /* Deferred threads from pthread_cond_signal. */ unsigned int *defer_waiters[MAX_DEFER_WAITERS]; #define _pthread_endzero wake_addr @@ -754,13 +766,17 @@ extern struct pthread *_single_thread __hidden; */ __BEGIN_DECLS int _thr_setthreaded(int) __hidden; -int _mutex_cv_lock(struct pthread_mutex *, int) __hidden; +int _mutex_cv_lock(struct pthread_mutex *, int, bool) __hidden; int _mutex_cv_unlock(struct pthread_mutex *, int *, int *) __hidden; int _mutex_cv_attach(struct pthread_mutex *, int) __hidden; int _mutex_cv_detach(struct pthread_mutex *, int *) __hidden; int _mutex_owned(struct pthread *, const struct pthread_mutex *) __hidden; int _mutex_reinit(pthread_mutex_t *) __hidden; void _mutex_fork(struct pthread *curthread) __hidden; +int _mutex_enter_robust(struct pthread *curthread, struct pthread_mutex *m) + __hidden; +void _mutex_leave_robust(struct pthread *curthread, struct pthread_mutex *m) + __hidden; void _libpthread_init(struct pthread *) __hidden; struct pthread *_thr_alloc(struct pthread *) __hidden; void _thread_exit(const char *, int, const char *) __hidden __dead2; @@ -819,6 +835,11 @@ void _pthread_cleanup_pop(int); void _pthread_exit_mask(void *status, sigset_t *mask) __dead2 __hidden; void _pthread_cancel_enter(int maycancel); void _pthread_cancel_leave(int maycancel); +int _pthread_mutex_consistent(pthread_mutex_t *) __nonnull(1); +int _pthread_mutexattr_getrobust(pthread_mutexattr_t *__restrict, + int *__restrict) __nonnull_all; +int _pthread_mutexattr_setrobust(pthread_mutexattr_t *, int) + __nonnull(1); /* #include <fcntl.h> */ #ifdef _SYS_FCNTL_H_ diff --git a/lib/libthr/thread/thr_umtx.c b/lib/libthr/thread/thr_umtx.c index ebf344b..cd2b101 100644 --- a/lib/libthr/thread/thr_umtx.c +++ b/lib/libthr/thread/thr_umtx.c @@ -33,6 +33,7 @@ __FBSDID("$FreeBSD$"); #ifndef HAS__UMTX_OP_ERR int _umtx_op_err(void *obj, int op, u_long val, void *uaddr, void *uaddr2) { + if (_umtx_op(obj, op, val, uaddr, uaddr2) == -1) return (errno); return (0); @@ -60,19 +61,24 @@ __thr_umutex_lock(struct umutex *mtx, uint32_t id) { uint32_t owner; - if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { - for (;;) { - /* wait in kernel */ - _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); + if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0) + return (_umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0)); - owner = mtx->m_owner; - if ((owner & ~UMUTEX_CONTESTED) == 0 && - atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) - return (0); - } + for (;;) { + owner = mtx->m_owner; + if ((owner & ~UMUTEX_CONTESTED) == 0 && + atomic_cmpset_acq_32(&mtx->m_owner, owner, id | owner)) + return (0); + if (owner == UMUTEX_RB_OWNERDEAD && + atomic_cmpset_acq_32(&mtx->m_owner, owner, + id | UMUTEX_CONTESTED)) + return (EOWNERDEAD); + if (owner == UMUTEX_RB_NOTRECOV) + return (ENOTRECOVERABLE); + + /* wait in kernel */ + _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); } - - return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0); } #define SPINLOOPS 1000 @@ -81,31 +87,33 @@ int __thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) { uint32_t owner; + int count; if (!_thr_is_smp) - return __thr_umutex_lock(mtx, id); - - if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { - for (;;) { - int count = SPINLOOPS; - while (count--) { - owner = mtx->m_owner; - if ((owner & ~UMUTEX_CONTESTED) == 0) { - if (atomic_cmpset_acq_32( - &mtx->m_owner, - owner, id|owner)) { - return (0); - } - } - CPU_SPINWAIT; - } + return (__thr_umutex_lock(mtx, id)); + if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0) + return (_umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0)); - /* wait in kernel */ - _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); + for (;;) { + count = SPINLOOPS; + while (count--) { + owner = mtx->m_owner; + if ((owner & ~UMUTEX_CONTESTED) == 0 && + atomic_cmpset_acq_32(&mtx->m_owner, owner, + id | owner)) + return (0); + if (__predict_false(owner == UMUTEX_RB_OWNERDEAD) && + atomic_cmpset_acq_32(&mtx->m_owner, owner, + id | UMUTEX_CONTESTED)) + return (EOWNERDEAD); + if (__predict_false(owner == UMUTEX_RB_NOTRECOV)) + return (ENOTRECOVERABLE); + CPU_SPINWAIT; } - } - return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0); + /* wait in kernel */ + _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0); + } } int @@ -129,21 +137,28 @@ __thr_umutex_timedlock(struct umutex *mtx, uint32_t id, } for (;;) { - if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { - - /* wait in kernel */ - ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, - (void *)tm_size, __DECONST(void *, tm_p)); - - /* now try to lock it */ + if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | + UMUTEX_PRIO_INHERIT)) == 0) { + /* try to lock it */ owner = mtx->m_owner; if ((owner & ~UMUTEX_CONTESTED) == 0 && - atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner)) + atomic_cmpset_acq_32(&mtx->m_owner, owner, + id | owner)) return (0); + if (__predict_false(owner == UMUTEX_RB_OWNERDEAD) && + atomic_cmpset_acq_32(&mtx->m_owner, owner, + id | UMUTEX_CONTESTED)) + return (EOWNERDEAD); + if (__predict_false(owner == UMUTEX_RB_NOTRECOV)) + return (ENOTRECOVERABLE); + /* wait in kernel */ + ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, + (void *)tm_size, __DECONST(void *, tm_p)); } else { ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, - (void *)tm_size, __DECONST(void *, tm_p)); - if (ret == 0) + (void *)tm_size, __DECONST(void *, tm_p)); + if (ret == 0 || ret == EOWNERDEAD || + ret == ENOTRECOVERABLE) break; } if (ret == ETIMEDOUT) @@ -155,46 +170,52 @@ __thr_umutex_timedlock(struct umutex *mtx, uint32_t id, int __thr_umutex_unlock(struct umutex *mtx, uint32_t id) { - return _umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0); + + return (_umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0)); } int __thr_umutex_trylock(struct umutex *mtx) { - return _umtx_op_err(mtx, UMTX_OP_MUTEX_TRYLOCK, 0, 0, 0); + + return (_umtx_op_err(mtx, UMTX_OP_MUTEX_TRYLOCK, 0, 0, 0)); } int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling, - uint32_t *oldceiling) + uint32_t *oldceiling) { - return _umtx_op_err(mtx, UMTX_OP_SET_CEILING, ceiling, oldceiling, 0); + + return (_umtx_op_err(mtx, UMTX_OP_SET_CEILING, ceiling, oldceiling, 0)); } int _thr_umtx_wait(volatile long *mtx, long id, const struct timespec *timeout) { + if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && - timeout->tv_nsec <= 0))) + timeout->tv_nsec <= 0))) return (ETIMEDOUT); - return _umtx_op_err(__DEVOLATILE(void *, mtx), UMTX_OP_WAIT, id, 0, - __DECONST(void*, timeout)); + return (_umtx_op_err(__DEVOLATILE(void *, mtx), UMTX_OP_WAIT, id, 0, + __DECONST(void*, timeout))); } int -_thr_umtx_wait_uint(volatile u_int *mtx, u_int id, const struct timespec *timeout, int shared) +_thr_umtx_wait_uint(volatile u_int *mtx, u_int id, + const struct timespec *timeout, int shared) { + if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && - timeout->tv_nsec <= 0))) + timeout->tv_nsec <= 0))) return (ETIMEDOUT); - return _umtx_op_err(__DEVOLATILE(void *, mtx), - shared ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, 0, - __DECONST(void*, timeout)); + return (_umtx_op_err(__DEVOLATILE(void *, mtx), shared ? + UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, 0, + __DECONST(void*, timeout))); } int _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int id, int clockid, - const struct timespec *abstime, int shared) + const struct timespec *abstime, int shared) { struct _umtx_time *tm_p, timeout; size_t tm_size; @@ -210,21 +231,23 @@ _thr_umtx_timedwait_uint(volatile u_int *mtx, u_int id, int clockid, tm_size = sizeof(timeout); } - return _umtx_op_err(__DEVOLATILE(void *, mtx), - shared ? UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, - (void *)tm_size, __DECONST(void *, tm_p)); + return (_umtx_op_err(__DEVOLATILE(void *, mtx), shared ? + UMTX_OP_WAIT_UINT : UMTX_OP_WAIT_UINT_PRIVATE, id, + (void *)tm_size, __DECONST(void *, tm_p))); } int _thr_umtx_wake(volatile void *mtx, int nr_wakeup, int shared) { - return _umtx_op_err(__DEVOLATILE(void *, mtx), shared ? UMTX_OP_WAKE : UMTX_OP_WAKE_PRIVATE, - nr_wakeup, 0, 0); + + return (_umtx_op_err(__DEVOLATILE(void *, mtx), shared ? + UMTX_OP_WAKE : UMTX_OP_WAKE_PRIVATE, nr_wakeup, 0, 0)); } void _thr_ucond_init(struct ucond *cv) { + bzero(cv, sizeof(struct ucond)); } @@ -232,30 +255,34 @@ int _thr_ucond_wait(struct ucond *cv, struct umutex *m, const struct timespec *timeout, int flags) { + struct pthread *curthread; + if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 && timeout->tv_nsec <= 0))) { - struct pthread *curthread = _get_curthread(); + curthread = _get_curthread(); _thr_umutex_unlock(m, TID(curthread)); return (ETIMEDOUT); } - return _umtx_op_err(cv, UMTX_OP_CV_WAIT, flags, - m, __DECONST(void*, timeout)); + return (_umtx_op_err(cv, UMTX_OP_CV_WAIT, flags, m, + __DECONST(void*, timeout))); } int _thr_ucond_signal(struct ucond *cv) { + if (!cv->c_has_waiters) return (0); - return _umtx_op_err(cv, UMTX_OP_CV_SIGNAL, 0, NULL, NULL); + return (_umtx_op_err(cv, UMTX_OP_CV_SIGNAL, 0, NULL, NULL)); } int _thr_ucond_broadcast(struct ucond *cv) { + if (!cv->c_has_waiters) return (0); - return _umtx_op_err(cv, UMTX_OP_CV_BROADCAST, 0, NULL, NULL); + return (_umtx_op_err(cv, UMTX_OP_CV_BROADCAST, 0, NULL, NULL)); } int @@ -275,7 +302,8 @@ __thr_rwlock_rdlock(struct urwlock *rwlock, int flags, tm_p = &timeout; tm_size = sizeof(timeout); } - return _umtx_op_err(rwlock, UMTX_OP_RW_RDLOCK, flags, (void *)tm_size, tm_p); + return (_umtx_op_err(rwlock, UMTX_OP_RW_RDLOCK, flags, + (void *)tm_size, tm_p)); } int @@ -294,13 +322,15 @@ __thr_rwlock_wrlock(struct urwlock *rwlock, const struct timespec *tsp) tm_p = &timeout; tm_size = sizeof(timeout); } - return _umtx_op_err(rwlock, UMTX_OP_RW_WRLOCK, 0, (void *)tm_size, tm_p); + return (_umtx_op_err(rwlock, UMTX_OP_RW_WRLOCK, 0, (void *)tm_size, + tm_p)); } int __thr_rwlock_unlock(struct urwlock *rwlock) { - return _umtx_op_err(rwlock, UMTX_OP_RW_UNLOCK, 0, NULL, NULL); + + return (_umtx_op_err(rwlock, UMTX_OP_RW_UNLOCK, 0, NULL, NULL)); } void @@ -338,6 +368,7 @@ _thr_rwl_wrlock(struct urwlock *rwlock) void _thr_rwl_unlock(struct urwlock *rwlock) { + if (_thr_rwlock_unlock(rwlock)) PANIC("unlock error"); } diff --git a/lib/libthr/thread/thr_umtx.h b/lib/libthr/thread/thr_umtx.h index 2c289a7..fff8729 100644 --- a/lib/libthr/thread/thr_umtx.h +++ b/lib/libthr/thread/thr_umtx.h @@ -32,7 +32,11 @@ #include <strings.h> #include <sys/umtx.h> -#define DEFAULT_UMUTEX {0,0,{0,0},{0,0,0,0}} +#ifdef __LP64__ +#define DEFAULT_UMUTEX {0,0,{0,0},0,{0,0}} +#else +#define DEFAULT_UMUTEX {0,0,{0,0},0,0,{0,0}} +#endif #define DEFAULT_URWLOCK {0,0,0,0,{0,0,0,0}} int _umtx_op_err(void *, int op, u_long, void *, void *) __hidden; @@ -75,95 +79,122 @@ void _thr_rwl_unlock(struct urwlock *rwlock) __hidden; static inline int _thr_umutex_trylock(struct umutex *mtx, uint32_t id) { - if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id)) - return (0); - if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0) - return (EBUSY); - return (__thr_umutex_trylock(mtx)); + + if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id)) + return (0); + if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) && + atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD, + id | UMUTEX_CONTESTED)) + return (EOWNERDEAD); + if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV)) + return (ENOTRECOVERABLE); + if ((mtx->m_flags & UMUTEX_PRIO_PROTECT) == 0) + return (EBUSY); + return (__thr_umutex_trylock(mtx)); } static inline int _thr_umutex_trylock2(struct umutex *mtx, uint32_t id) { - if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0) - return (0); - if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED && - __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0)) - if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED)) + + if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0) return (0); - return (EBUSY); + if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED && + __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | + UMUTEX_PRIO_INHERIT)) == 0) && + atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, + id | UMUTEX_CONTESTED)) + return (0); + if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_OWNERDEAD) && + atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_RB_OWNERDEAD, + id | UMUTEX_CONTESTED)) + return (EOWNERDEAD); + if (__predict_false((uint32_t)mtx->m_owner == UMUTEX_RB_NOTRECOV)) + return (ENOTRECOVERABLE); + return (EBUSY); } static inline int _thr_umutex_lock(struct umutex *mtx, uint32_t id) { - if (_thr_umutex_trylock2(mtx, id) == 0) - return (0); - return (__thr_umutex_lock(mtx, id)); + + if (_thr_umutex_trylock2(mtx, id) == 0) + return (0); + return (__thr_umutex_lock(mtx, id)); } static inline int _thr_umutex_lock_spin(struct umutex *mtx, uint32_t id) { - if (_thr_umutex_trylock2(mtx, id) == 0) - return (0); - return (__thr_umutex_lock_spin(mtx, id)); + + if (_thr_umutex_trylock2(mtx, id) == 0) + return (0); + return (__thr_umutex_lock_spin(mtx, id)); } static inline int _thr_umutex_timedlock(struct umutex *mtx, uint32_t id, - const struct timespec *timeout) + const struct timespec *timeout) { - if (_thr_umutex_trylock2(mtx, id) == 0) - return (0); - return (__thr_umutex_timedlock(mtx, id, timeout)); + + if (_thr_umutex_trylock2(mtx, id) == 0) + return (0); + return (__thr_umutex_timedlock(mtx, id, timeout)); } static inline int _thr_umutex_unlock2(struct umutex *mtx, uint32_t id, int *defer) { - uint32_t flags = mtx->m_flags; + uint32_t flags, owner; + bool noncst; - if ((flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) { - uint32_t owner; - do { - owner = mtx->m_owner; - if (__predict_false((owner & ~UMUTEX_CONTESTED) != id)) - return (EPERM); - } while (__predict_false(!atomic_cmpset_rel_32(&mtx->m_owner, - owner, UMUTEX_UNOWNED))); - if ((owner & UMUTEX_CONTESTED)) { - if (defer == NULL) - (void)_umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE2, flags, 0, 0); - else - *defer = 1; - } - return (0); + flags = mtx->m_flags; + noncst = (flags & UMUTEX_NONCONSISTENT) != 0; + + if ((flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) != 0) { + if (atomic_cmpset_rel_32(&mtx->m_owner, id, noncst ? + UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED)) + return (0); + return (__thr_umutex_unlock(mtx, id)); } - if (atomic_cmpset_rel_32(&mtx->m_owner, id, UMUTEX_UNOWNED)) - return (0); - return (__thr_umutex_unlock(mtx, id)); + + do { + owner = mtx->m_owner; + if (__predict_false((owner & ~UMUTEX_CONTESTED) != id)) + return (EPERM); + } while (__predict_false(!atomic_cmpset_rel_32(&mtx->m_owner, owner, + noncst ? UMUTEX_RB_NOTRECOV : UMUTEX_UNOWNED))); + if ((owner & UMUTEX_CONTESTED) != 0) { + if (defer == NULL || noncst) + (void)_umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE2, + flags, 0, 0); + else + *defer = 1; + } + return (0); } static inline int _thr_umutex_unlock(struct umutex *mtx, uint32_t id) { - return _thr_umutex_unlock2(mtx, id, NULL); + + return (_thr_umutex_unlock2(mtx, id, NULL)); } static inline int _thr_rwlock_tryrdlock(struct urwlock *rwlock, int flags) { - int32_t state; - int32_t wrflags; + int32_t state, wrflags; - if (flags & URWLOCK_PREFER_READER || rwlock->rw_flags & URWLOCK_PREFER_READER) + if ((flags & URWLOCK_PREFER_READER) != 0 || + (rwlock->rw_flags & URWLOCK_PREFER_READER) != 0) wrflags = URWLOCK_WRITE_OWNER; else wrflags = URWLOCK_WRITE_OWNER | URWLOCK_WRITE_WAITERS; state = rwlock->rw_state; while (!(state & wrflags)) { - if (__predict_false(URWLOCK_READER_COUNT(state) == URWLOCK_MAX_READERS)) + if (__predict_false(URWLOCK_READER_COUNT(state) == + URWLOCK_MAX_READERS)) return (EAGAIN); if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state + 1)) return (0); @@ -179,8 +210,10 @@ _thr_rwlock_trywrlock(struct urwlock *rwlock) int32_t state; state = rwlock->rw_state; - while (!(state & URWLOCK_WRITE_OWNER) && URWLOCK_READER_COUNT(state) == 0) { - if (atomic_cmpset_acq_32(&rwlock->rw_state, state, state | URWLOCK_WRITE_OWNER)) + while ((state & URWLOCK_WRITE_OWNER) == 0 && + URWLOCK_READER_COUNT(state) == 0) { + if (atomic_cmpset_acq_32(&rwlock->rw_state, state, + state | URWLOCK_WRITE_OWNER)) return (0); state = rwlock->rw_state; } @@ -191,6 +224,7 @@ _thr_rwlock_trywrlock(struct urwlock *rwlock) static inline int _thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp) { + if (_thr_rwlock_tryrdlock(rwlock, flags) == 0) return (0); return (__thr_rwlock_rdlock(rwlock, flags, tsp)); @@ -199,6 +233,7 @@ _thr_rwlock_rdlock(struct urwlock *rwlock, int flags, struct timespec *tsp) static inline int _thr_rwlock_wrlock(struct urwlock *rwlock, struct timespec *tsp) { + if (_thr_rwlock_trywrlock(rwlock) == 0) return (0); return (__thr_rwlock_wrlock(rwlock, tsp)); @@ -210,18 +245,19 @@ _thr_rwlock_unlock(struct urwlock *rwlock) int32_t state; state = rwlock->rw_state; - if (state & URWLOCK_WRITE_OWNER) { - if (atomic_cmpset_rel_32(&rwlock->rw_state, URWLOCK_WRITE_OWNER, 0)) + if ((state & URWLOCK_WRITE_OWNER) != 0) { + if (atomic_cmpset_rel_32(&rwlock->rw_state, + URWLOCK_WRITE_OWNER, 0)) return (0); } else { for (;;) { if (__predict_false(URWLOCK_READER_COUNT(state) == 0)) return (EPERM); if (!((state & (URWLOCK_WRITE_WAITERS | - URWLOCK_READ_WAITERS)) && + URWLOCK_READ_WAITERS)) != 0 && URWLOCK_READER_COUNT(state) == 1)) { if (atomic_cmpset_rel_32(&rwlock->rw_state, - state, state-1)) + state, state - 1)) return (0); state = rwlock->rw_state; } else { diff --git a/share/man/man3/Makefile b/share/man/man3/Makefile index 2a3c12a..72e4c54 100644 --- a/share/man/man3/Makefile +++ b/share/man/man3/Makefile @@ -238,6 +238,7 @@ PTHREAD_MAN= pthread.3 \ pthread_multi_np.3 \ pthread_mutexattr.3 \ pthread_mutexattr_getkind_np.3 \ + pthread_mutex_consistent.3 \ pthread_mutex_destroy.3 \ pthread_mutex_init.3 \ pthread_mutex_lock.3 \ @@ -312,10 +313,12 @@ PTHREAD_MLINKS+=pthread_multi_np.3 pthread_single_np.3 PTHREAD_MLINKS+=pthread_mutexattr.3 pthread_mutexattr_destroy.3 \ pthread_mutexattr.3 pthread_mutexattr_getprioceiling.3 \ pthread_mutexattr.3 pthread_mutexattr_getprotocol.3 \ + pthread_mutexattr.3 pthread_mutexattr_getrobust.3 \ pthread_mutexattr.3 pthread_mutexattr_gettype.3 \ pthread_mutexattr.3 pthread_mutexattr_init.3 \ pthread_mutexattr.3 pthread_mutexattr_setprioceiling.3 \ pthread_mutexattr.3 pthread_mutexattr_setprotocol.3 \ + pthread_mutexattr.3 pthread_mutexattr_setrobust.3 \ pthread_mutexattr.3 pthread_mutexattr_settype.3 PTHREAD_MLINKS+=pthread_mutexattr_getkind_np.3 pthread_mutexattr_setkind_np.3 PTHREAD_MLINKS+=pthread_rwlock_rdlock.3 pthread_rwlock_tryrdlock.3 diff --git a/share/man/man3/pthread_cond_wait.3 b/share/man/man3/pthread_cond_wait.3 index c057098..427c347 100644 --- a/share/man/man3/pthread_cond_wait.3 +++ b/share/man/man3/pthread_cond_wait.3 @@ -27,7 +27,7 @@ .\" .\" $FreeBSD$ .\" -.Dd February 16, 2006 +.Dd April 29, 2016 .Dt PTHREAD_COND_WAIT 3 .Os .Sh NAME @@ -75,13 +75,25 @@ is invalid. The specified .Fa mutex was not locked by the calling thread. +.It Bq Er EOWNERDEAD +The argument +.Fa mutex +points to a robust mutex and the previous owning thread terminated +while holding the mutex lock. +The lock was granted to the caller and it is up to the new owner +to make the state consistent. +.It Bq Er ENOTRECOVERABLE +The state protected by the +.Fa mutex +is not recoverable. .El .Sh SEE ALSO .Xr pthread_cond_broadcast 3 , .Xr pthread_cond_destroy 3 , .Xr pthread_cond_init 3 , .Xr pthread_cond_signal 3 , -.Xr pthread_cond_timedwait 3 +.Xr pthread_cond_timedwait 3 , +.Xr pthread_mutex_consistent 3 .Sh STANDARDS The .Fn pthread_cond_wait diff --git a/share/man/man3/pthread_mutex_consistent.3 b/share/man/man3/pthread_mutex_consistent.3 new file mode 100644 index 0000000..67bfbcb --- /dev/null +++ b/share/man/man3/pthread_mutex_consistent.3 @@ -0,0 +1,94 @@ +.\" Copyright (c) 2016 The FreeBSD Foundation, Inc. +.\" All rights reserved. +.\" +.\" This documentation was written by +.\" Konstantin Belousov <kib@FreeBSD.org> under sponsorship +.\" from the FreeBSD Foundation. +.\" +.\" Redistribution and use in source and binary forms, with or without +.\" modification, are permitted provided that the following conditions +.\" are met: +.\" 1. Redistributions of source code must retain the above copyright +.\" notice, this list of conditions and the following disclaimer. +.\" 2. Redistributions in binary form must reproduce the above copyright +.\" notice, this list of conditions and the following disclaimer in the +.\" documentation and/or other materials provided with the distribution. +.\" +.\" THIS SOFTWARE IS PROVIDED BY THE AUTHORS AND CONTRIBUTORS ``AS IS'' AND +.\" ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +.\" IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE +.\" ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHORS OR CONTRIBUTORS BE LIABLE +.\" FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +.\" DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS +.\" OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) +.\" HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT +.\" LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY +.\" OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF +.\" SUCH DAMAGE. +.\" +.\" $FreeBSD$ +.\" +.Dd May 8, 2016 +.Dt PTHREAD_MUTEX_CONSISTENT 3 +.Os +.Sh NAME +.Nm pthread_mutex_consistent +.Nd mark state protected by robust mutex as consistent +.Sh LIBRARY +.Lb libpthread +.Sh SYNOPSIS +.In pthread.h +.Ft int +.Fn pthread_mutex_consistent "pthread_mutex_t *mutex" +.Sh DESCRIPTION +If the thread owning a robust mutex terminates while holding the +mutex, the mutex becomes inconsistent and the next thread that +acquires the mutex lock is notified of the state by the return value +.Er EOWNERDEAD . +In this case, the mutex does not become normally usable again until +the state is marked consistent. +.Pp +The +.Fn pthread_mutex_consistent , +when called with the +.Fa mutex +argument, which points to the initialized robust mutex in an +inconsistent state, marks the by mutex as consistent again. +The consequent unlock of the mutex, by either +.Fn pthread_mutex_unlock +or other methods, allows other contenders to lock the mutex. +.Pp +If the mutex in the inconsistent state is not marked consistent +by the call to +.Fn pthread_mutex_consistent +before unlock, +further attempts to lock the +.Fa mutex +result in the +.Er ENOTRECOVERABLE +condition reported by the locking functions. +.Sh RETURN VALUES +If successful, +.Fn pthread_mutex_consistent +will return zero, otherwise an error number will be returned to +indicate the error. +.Sh ERRORS +The +.Fn pthread_mutex_lock +function will fail if: +.Bl -tag -width Er +.It Bq Er EINVAL +The mutex pointed to by the +.Fa mutex +argument is not robust, or is not in the inconsistent state. +.El +.Sh SEE ALSO +.Xr pthread_mutexattr_setrobust 3 , +.Xr pthread_mutex_init 3 , +.Xr pthread_mutex_lock 3 , +.Xr pthread_mutex_unlock 3 +.Sh STANDARDS +The +.Fn pthread_mutex_lock +function conforms to +.St -susv4 . diff --git a/share/man/man3/pthread_mutex_lock.3 b/share/man/man3/pthread_mutex_lock.3 index 8479a69..bd94380 100644 --- a/share/man/man3/pthread_mutex_lock.3 +++ b/share/man/man3/pthread_mutex_lock.3 @@ -27,7 +27,7 @@ .\" .\" $FreeBSD$ .\" -.Dd January 31, 2006 +.Dd April 29, 2016 .Dt PTHREAD_MUTEX_LOCK 3 .Os .Sh NAME @@ -55,7 +55,7 @@ indicate the error. The .Fn pthread_mutex_lock function will fail if: -.Bl -tag -width Er +.Bl -tag -width "Er ENOTRECOVERABLE" .It Bq Er EINVAL The value specified by .Fa mutex @@ -63,8 +63,20 @@ is invalid. .It Bq Er EDEADLK A deadlock would occur if the thread blocked waiting for .Fa mutex . +.It Bq Er EOWNERDEAD +The argument +.Fa mutex +points to a robust mutex and the previous owning thread terminated +while holding the mutex lock. +The lock was granted to the caller and it is up to the new owner +to make the state consistent. +.It Bq Er ENOTRECOVERABLE +The state protected by the +.Fa mutex +is not recoverable. .El .Sh SEE ALSO +.Xr pthread_mutex_consistent 3 , .Xr pthread_mutex_destroy 3 , .Xr pthread_mutex_init 3 , .Xr pthread_mutex_trylock 3 , diff --git a/share/man/man3/pthread_mutex_timedlock.3 b/share/man/man3/pthread_mutex_timedlock.3 index abc7e2a..384ee9d 100644 --- a/share/man/man3/pthread_mutex_timedlock.3 +++ b/share/man/man3/pthread_mutex_timedlock.3 @@ -24,7 +24,7 @@ .\" .\" $FreeBSD$ .\" -.Dd December 30, 2003 +.Dd April 29, 2016 .Dt PTHREAD_MUTEX_TIMEDLOCK 3 .Os .Sh NAME @@ -59,7 +59,7 @@ The .Fn pthread_mutex_timedlock function will fail if: .Bl -tag -width Er -.It Bq Er EINVAL +.It Bq "Er ENOTRECOVERABLE" The .Fa mutex was created with the protocol attribute having the @@ -89,8 +89,20 @@ has been exceeded. .It Bq Er EDEADLK The current thread already owns the .Fa mutex . +.It Bq Er EOWNERDEAD +The argument +.Fa mutex +points to a robust mutex and the previous owning thread terminated +while holding the mutex lock. +The lock was granted to the caller and it is up to the new owner +to make the state consistent. +.It Bq Er ENOTRECOVERABLE +The state protected by the +.Fa mutex +is not recoverable. .El .Sh SEE ALSO +.Xr pthread_mutex_consistent 3 , .Xr pthread_mutex_destroy 3 , .Xr pthread_mutex_init 3 , .Xr pthread_mutex_lock 3 , diff --git a/share/man/man3/pthread_mutex_trylock.3 b/share/man/man3/pthread_mutex_trylock.3 index 049006f..2837704 100644 --- a/share/man/man3/pthread_mutex_trylock.3 +++ b/share/man/man3/pthread_mutex_trylock.3 @@ -27,7 +27,7 @@ .\" .\" $FreeBSD$ .\" -.Dd July 30, 1998 +.Dd April 29, 2016 .Dt PTHREAD_MUTEX_TRYLOCK 3 .Os .Sh NAME @@ -56,7 +56,7 @@ indicate the error. The .Fn pthread_mutex_trylock function will fail if: -.Bl -tag -width Er +.Bl -tag -width "Er ENOTRECOVERABLE" .It Bq Er EINVAL The value specified by .Fa mutex @@ -64,8 +64,20 @@ is invalid. .It Bq Er EBUSY .Fa Mutex is already locked. +.It Bq Er EOWNERDEAD +The argument +.Fa mutex +points to a robust mutex and the previous owning thread terminated +while holding the mutex lock. +The lock was granted to the caller and it is up to the new owner +to make the state consistent. +.It Bq Er ENOTRECOVERABLE +The state protected by the +.Fa mutex +is not recoverable. .El .Sh SEE ALSO +.Xr pthread_mutex_consistent 3 , .Xr pthread_mutex_destroy 3 , .Xr pthread_mutex_init 3 , .Xr pthread_mutex_lock 3 , diff --git a/share/man/man3/pthread_mutex_unlock.3 b/share/man/man3/pthread_mutex_unlock.3 index 77784e1..4d3badd 100644 --- a/share/man/man3/pthread_mutex_unlock.3 +++ b/share/man/man3/pthread_mutex_unlock.3 @@ -27,7 +27,7 @@ .\" .\" $FreeBSD$ .\" -.Dd July 30, 1998 +.Dd April 29, 2016 .Dt PTHREAD_MUTEX_UNLOCK 3 .Os .Sh NAME @@ -46,6 +46,17 @@ then the .Fn pthread_mutex_unlock function unlocks .Fa mutex . +.Pp +If the argument pointed by the +.Fa mutex +is a robust mutex in the inconsistent state, and the call to +.Fn pthread_mutex_consistent +function was not done prior to unlocking, further locking attempts on +the mutex +.Fa mutex +are denied and locking functions return +.Er ENOTRECOVERABLE +error. .Sh RETURN VALUES If successful, .Fn pthread_mutex_unlock diff --git a/share/man/man3/pthread_mutexattr.3 b/share/man/man3/pthread_mutexattr.3 index f976026..e7042d5 100644 --- a/share/man/man3/pthread_mutexattr.3 +++ b/share/man/man3/pthread_mutexattr.3 @@ -26,7 +26,7 @@ .\" EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. .\" .\" $FreeBSD$ -.Dd May 1, 2000 +.Dd April 29, 2016 .Dt PTHREAD_MUTEXATTR 3 .Os .Sh NAME @@ -36,6 +36,8 @@ .Nm pthread_mutexattr_getprioceiling , .Nm pthread_mutexattr_setprotocol , .Nm pthread_mutexattr_getprotocol , +.Nm pthread_mutexattr_setrobust , +.Nm pthread_mutexattr_getrobust , .Nm pthread_mutexattr_settype , .Nm pthread_mutexattr_gettype .Nd mutex attribute operations @@ -56,6 +58,10 @@ .Ft int .Fn pthread_mutexattr_getprotocol "pthread_mutexattr_t *attr" "int *protocol" .Ft int +.Fn pthread_mutexattr_setrobust "pthread_mutexattr_t *attr" "int robust" +.Ft int +.Fn pthread_mutexattr_getrobust "pthread_mutexattr_t *attr" "int *robust" +.Ft int .Fn pthread_mutexattr_settype "pthread_mutexattr_t *attr" "int type" .Ft int .Fn pthread_mutexattr_gettype "pthread_mutexattr_t *attr" "int *type" @@ -165,6 +171,26 @@ function will fail if: Invalid value for .Fa attr . .El +.Pp +The +.Fn pthread_mutexattr_setrobust +function will fail if: +.Bl -tag -width Er +.It Bq Er EINVAL +Invalid value for +.Fa attr , +or invalid value for +.Fa robust . +.El +.Pp +The +.Fn pthread_mutexattr_getrobust +function will fail if: +.Bl -tag -width Er +.It Bq Er EINVAL +Invalid value for +.Fa attr . +.El .Sh SEE ALSO .Xr pthread_mutex_init 3 .Sh STANDARDS @@ -184,4 +210,10 @@ The and .Fn pthread_mutexattr_gettype functions conform to -.St -susv2 +.St -susv2 . +The +.Fn pthread_mutexattr_setrobust +and +.Fn pthread_mutexattr_getrobust +functions conform to +.St -susv4 . diff --git a/sys/compat/cloudabi/cloudabi_thread.c b/sys/compat/cloudabi/cloudabi_thread.c index 37dc794..dd54e89 100644 --- a/sys/compat/cloudabi/cloudabi_thread.c +++ b/sys/compat/cloudabi/cloudabi_thread.c @@ -30,6 +30,7 @@ __FBSDID("$FreeBSD$"); #include <sys/proc.h> #include <sys/sched.h> #include <sys/syscallsubr.h> +#include <sys/umtx.h> #include <contrib/cloudabi/cloudabi_types_common.h> @@ -44,6 +45,8 @@ cloudabi_sys_thread_exit(struct thread *td, .scope = uap->scope, }; + umtx_thread_exit(td); + /* Wake up joining thread. */ cloudabi_sys_lock_unlock(td, &cloudabi_sys_lock_unlock_args); diff --git a/sys/compat/linux/linux_fork.c b/sys/compat/linux/linux_fork.c index de1d041..4c30c9a 100644 --- a/sys/compat/linux/linux_fork.c +++ b/sys/compat/linux/linux_fork.c @@ -42,6 +42,7 @@ __FBSDID("$FreeBSD$"); #include <sys/sched.h> #include <sys/syscallsubr.h> #include <sys/sx.h> +#include <sys/umtx.h> #include <sys/unistd.h> #include <sys/wait.h> @@ -410,6 +411,8 @@ linux_exit(struct thread *td, struct linux_exit_args *args) LINUX_CTR2(exit, "thread(%d) (%d)", em->em_tid, args->rval); + umtx_thread_exit(td); + linux_thread_detach(td); /* diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c index 1cd9b62..0a0659d 100644 --- a/sys/kern/kern_exit.c +++ b/sys/kern/kern_exit.c @@ -350,8 +350,11 @@ exit1(struct thread *td, int rval, int signo) KASSERT(!timevalisset(&p->p_realtimer.it_value), ("realtime timer is still armed")); } + PROC_UNLOCK(p); + umtx_thread_exit(td); + /* * Reset any sigio structures pointing to us as a result of * F_SETOWN with our pid. @@ -595,7 +598,6 @@ exit1(struct thread *td, int rval, int signo) wakeup(p->p_pptr); cv_broadcast(&p->p_pwait); sched_exit(p->p_pptr, td); - umtx_thread_exit(td); PROC_SLOCK(p); p->p_state = PRS_ZOMBIE; PROC_UNLOCK(p->p_pptr); diff --git a/sys/kern/kern_thr.c b/sys/kern/kern_thr.c index 1b226dd..75bd83d 100644 --- a/sys/kern/kern_thr.c +++ b/sys/kern/kern_thr.c @@ -308,6 +308,8 @@ sys_thr_exit(struct thread *td, struct thr_exit_args *uap) /* long *state */ { + umtx_thread_exit(td); + /* Signal userland that it can free the stack. */ if ((void *)uap->state != NULL) { suword_lwpid(uap->state, 1); @@ -367,7 +369,6 @@ kern_thr_exit(struct thread *td) KASSERT(p->p_numthreads > 1, ("too few threads")); racct_sub(p, RACCT_NTHR, 1); tdsigcleanup(td); - umtx_thread_exit(td); PROC_SLOCK(p); thread_stopped(p); thread_exit(); diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c index 66b6120..21f3587 100644 --- a/sys/kern/kern_thread.c +++ b/sys/kern/kern_thread.c @@ -950,6 +950,7 @@ thread_suspend_check(int return_instead) */ if (__predict_false(p->p_sysent->sv_thread_detach != NULL)) (p->p_sysent->sv_thread_detach)(td); + umtx_thread_exit(td); kern_thr_exit(td); panic("stopped thread did not exit"); } diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c index 7ceb2cc..8471591 100644 --- a/sys/kern/kern_umtx.c +++ b/sys/kern/kern_umtx.c @@ -1,5 +1,5 @@ /*- - * Copyright (c) 2015 The FreeBSD Foundation + * Copyright (c) 2015, 2016 The FreeBSD Foundation * Copyright (c) 2004, David Xu <davidxu@freebsd.org> * Copyright (c) 2002, Jeffrey Roberson <jeff@freebsd.org> * All rights reserved. @@ -212,6 +212,30 @@ struct abs_timeout { struct timespec end; }; +#ifdef COMPAT_FREEBSD32 +struct umutex32 { + volatile __lwpid_t m_owner; /* Owner of the mutex */ + __uint32_t m_flags; /* Flags of the mutex */ + __uint32_t m_ceilings[2]; /* Priority protect ceiling */ + __uint32_t m_rb_lnk; /* Robust linkage */ + __uint32_t m_pad; + __uint32_t m_spare[2]; +}; + +_Static_assert(sizeof(struct umutex) == sizeof(struct umutex32), "umutex32"); +_Static_assert(__offsetof(struct umutex, m_spare[0]) == + __offsetof(struct umutex32, m_spare[0]), "m_spare32"); +#endif + +int umtx_shm_vnobj_persistent = 0; +SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_vnode_persistent, CTLFLAG_RWTUN, + &umtx_shm_vnobj_persistent, 0, + "False forces destruction of umtx attached to file, on last close"); +static int umtx_max_rb = 1000; +SYSCTL_INT(_kern_ipc, OID_AUTO, umtx_max_robust, CTLFLAG_RWTUN, + &umtx_max_rb, 0, + ""); + static uma_zone_t umtx_pi_zone; static struct umtxq_chain umtxq_chains[2][UMTX_CHAINS]; static MALLOC_DEFINE(M_UMTX, "umtx", "UMTX queue memory"); @@ -220,6 +244,10 @@ static int umtx_pi_allocated; static SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug"); SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD, &umtx_pi_allocated, 0, "Allocated umtx_pi"); +static int umtx_verbose_rb = 1; +SYSCTL_INT(_debug_umtx, OID_AUTO, robust_faults_verbose, CTLFLAG_RWTUN, + &umtx_verbose_rb, 0, + ""); #ifdef UMTX_PROFILING static long max_length; @@ -241,10 +269,11 @@ static int umtxq_sleep(struct umtx_q *uq, const char *wmesg, struct abs_timeout static int umtxq_count(struct umtx_key *key); static struct umtx_pi *umtx_pi_alloc(int); static void umtx_pi_free(struct umtx_pi *pi); -static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags); +static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, + bool rb); static void umtx_thread_cleanup(struct thread *td); static void umtx_exec_hook(void *arg __unused, struct proc *p __unused, - struct image_params *imgp __unused); + struct image_params *imgp __unused); SYSINIT(umtx, SI_SUB_EVENTHANDLER+1, SI_ORDER_MIDDLE, umtxq_sysinit, NULL); #define umtxq_signal(key, nwake) umtxq_signal_queue((key), (nwake), UMTX_SHARED_QUEUE) @@ -423,7 +452,8 @@ umtxq_alloc(void) struct umtx_q *uq; uq = malloc(sizeof(struct umtx_q), M_UMTX, M_WAITOK | M_ZERO); - uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, M_WAITOK | M_ZERO); + uq->uq_spare_queue = malloc(sizeof(struct umtxq_queue), M_UMTX, + M_WAITOK | M_ZERO); TAILQ_INIT(&uq->uq_spare_queue->head); TAILQ_INIT(&uq->uq_pi_contested); uq->uq_inherited_pri = PRI_MAX; @@ -433,6 +463,7 @@ umtxq_alloc(void) void umtxq_free(struct umtx_q *uq) { + MPASS(uq->uq_spare_queue != NULL); free(uq->uq_spare_queue, M_UMTX); free(uq, M_UMTX); @@ -441,13 +472,16 @@ umtxq_free(struct umtx_q *uq) static inline void umtxq_hash(struct umtx_key *key) { - unsigned n = (uintptr_t)key->info.both.a + key->info.both.b; + unsigned n; + + n = (uintptr_t)key->info.both.a + key->info.both.b; key->hash = ((n * GOLDEN_RATIO_PRIME) >> UMTX_SHIFTS) % UMTX_CHAINS; } static inline struct umtxq_chain * umtxq_getchain(struct umtx_key *key) { + if (key->type <= TYPE_SEM) return (&umtxq_chains[1][key->hash]); return (&umtxq_chains[0][key->hash]); @@ -750,13 +784,13 @@ abs_timeout_init2(struct abs_timeout *timo, const struct _umtx_time *umtxtime) { abs_timeout_init(timo, umtxtime->_clockid, - (umtxtime->_flags & UMTX_ABSTIME) != 0, - &umtxtime->_timeout); + (umtxtime->_flags & UMTX_ABSTIME) != 0, &umtxtime->_timeout); } static inline void abs_timeout_update(struct abs_timeout *timo) { + kern_clock_gettime(curthread, timo->clockid, &timo->cur); } @@ -772,6 +806,19 @@ abs_timeout_gethz(struct abs_timeout *timo) return (tstohz(&tts)); } +static uint32_t +umtx_unlock_val(uint32_t flags, bool rb) +{ + + if (rb) + return (UMUTEX_RB_OWNERDEAD); + else if ((flags & UMUTEX_NONCONSISTENT) != 0) + return (UMUTEX_RB_NOTRECOV); + else + return (UMUTEX_UNOWNED); + +} + /* * Put thread into sleep state, before sleeping, check if * thread was removed from umtx queue. @@ -866,7 +913,7 @@ umtx_key_release(struct umtx_key *key) */ static int do_wait(struct thread *td, void *addr, u_long id, - struct _umtx_time *timeout, int compat32, int is_private) + struct _umtx_time *timeout, int compat32, int is_private) { struct abs_timeout timo; struct umtx_q *uq; @@ -925,7 +972,7 @@ kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private) int ret; if ((ret = umtx_key_get(uaddr, TYPE_SIMPLE_WAIT, - is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0) + is_private ? THREAD_SHARE : AUTO_SHARE, &key)) != 0) return (ret); umtxq_lock(&key); umtxq_signal(&key, n_wake); @@ -939,7 +986,7 @@ kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private) */ static int do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, - struct _umtx_time *timeout, int mode) + struct _umtx_time *timeout, int mode) { struct abs_timeout timo; struct umtx_q *uq; @@ -961,11 +1008,38 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, if (rv == -1) return (EFAULT); if (mode == _UMUTEX_WAIT) { - if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED) + if (owner == UMUTEX_UNOWNED || + owner == UMUTEX_CONTESTED || + owner == UMUTEX_RB_OWNERDEAD || + owner == UMUTEX_RB_NOTRECOV) return (0); } else { /* - * Try the uncontested case. This should be done in userland. + * Robust mutex terminated. Kernel duty is to + * return EOWNERDEAD to the userspace. The + * umutex.m_flags UMUTEX_NONCONSISTENT is set + * by the common userspace code. + */ + if (owner == UMUTEX_RB_OWNERDEAD) { + rv = casueword32(&m->m_owner, + UMUTEX_RB_OWNERDEAD, &owner, + id | UMUTEX_CONTESTED); + if (rv == -1) + return (EFAULT); + if (owner == UMUTEX_RB_OWNERDEAD) + return (EOWNERDEAD); /* success */ + rv = umtxq_check_susp(td); + if (rv != 0) + return (rv); + continue; + } + if (owner == UMUTEX_RB_NOTRECOV) + return (ENOTRECOVERABLE); + + + /* + * Try the uncontested case. This should be + * done in userland. */ rv = casueword32(&m->m_owner, UMUTEX_UNOWNED, &owner, id); @@ -977,7 +1051,10 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, if (owner == UMUTEX_UNOWNED) return (0); - /* If no one owns it but it is contested try to acquire it. */ + /* + * If no one owns it but it is contested try + * to acquire it. + */ if (owner == UMUTEX_CONTESTED) { rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, @@ -993,7 +1070,10 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, if (rv != 0) return (rv); - /* If this failed the lock has changed, restart. */ + /* + * If this failed the lock has + * changed, restart. + */ continue; } } @@ -1061,12 +1141,11 @@ do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, * Unlock PTHREAD_PRIO_NONE protocol POSIX mutex. */ static int -do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) +do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags, bool rb) { struct umtx_key key; - uint32_t owner, old, id; - int error; - int count; + uint32_t owner, old, id, newlock; + int error, count; id = td->td_tid; /* @@ -1079,8 +1158,9 @@ do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); + newlock = umtx_unlock_val(flags, rb); if ((owner & UMUTEX_CONTESTED) == 0) { - error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED); + error = casueword32(&m->m_owner, owner, &old, newlock); if (error == -1) return (EFAULT); if (old == owner) @@ -1103,10 +1183,11 @@ do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags) * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ - error = casueword32(&m->m_owner, owner, &old, - count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); + if (count > 1) + newlock |= UMUTEX_CONTESTED; + error = casueword32(&m->m_owner, owner, &old, newlock); umtxq_lock(&key); - umtxq_signal(&key,1); + umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); umtx_key_release(&key); @@ -1134,7 +1215,8 @@ do_wake_umutex(struct thread *td, struct umutex *m) if (error == -1) return (EFAULT); - if ((owner & ~UMUTEX_CONTESTED) != 0) + if ((owner & ~UMUTEX_CONTESTED) != 0 && owner != UMUTEX_RB_OWNERDEAD && + owner != UMUTEX_RB_NOTRECOV) return (0); error = fueword32(&m->m_flags, &flags); @@ -1151,7 +1233,8 @@ do_wake_umutex(struct thread *td, struct umutex *m) count = umtxq_count(&key); umtxq_unlock(&key); - if (count <= 1) { + if (count <= 1 && owner != UMUTEX_RB_OWNERDEAD && + owner != UMUTEX_RB_NOTRECOV) { error = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, UMUTEX_UNOWNED); if (error == -1) @@ -1159,7 +1242,8 @@ do_wake_umutex(struct thread *td, struct umutex *m) } umtxq_lock(&key); - if (error == 0 && count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) + if (error == 0 && count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || + owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); @@ -1179,21 +1263,28 @@ do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags) int error; int count; - switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { + switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT | + UMUTEX_ROBUST)) { case 0: + case UMUTEX_ROBUST: type = TYPE_NORMAL_UMUTEX; break; case UMUTEX_PRIO_INHERIT: type = TYPE_PI_UMUTEX; break; + case (UMUTEX_PRIO_INHERIT | UMUTEX_ROBUST): + type = TYPE_PI_ROBUST_UMUTEX; + break; case UMUTEX_PRIO_PROTECT: type = TYPE_PP_UMUTEX; break; + case (UMUTEX_PRIO_PROTECT | UMUTEX_ROBUST): + type = TYPE_PP_ROBUST_UMUTEX; + break; default: return (EINVAL); } - if ((error = umtx_key_get(m, type, GET_SHARE(flags), - &key)) != 0) + if ((error = umtx_key_get(m, type, GET_SHARE(flags), &key)) != 0) return (error); owner = 0; @@ -1229,7 +1320,7 @@ do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags) if (error == -1) error = EFAULT; while (error == 0 && (owner & ~UMUTEX_CONTESTED) != 0 && - (owner & UMUTEX_CONTESTED) == 0) { + (owner & UMUTEX_CONTESTED) == 0) { error = casueword32(&m->m_owner, owner, &old, owner | UMUTEX_CONTESTED); if (error == -1) { @@ -1247,7 +1338,8 @@ do_wake2_umutex(struct thread *td, struct umutex *m, uint32_t flags) umtxq_lock(&key); if (error == EFAULT) { umtxq_signal(&key, INT_MAX); - } else if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0) + } else if (count != 0 && ((owner & ~UMUTEX_CONTESTED) == 0 || + owner == UMUTEX_RB_OWNERDEAD || owner == UMUTEX_RB_NOTRECOV)) umtxq_signal(&key, 1); umtxq_unbusy(&key); umtxq_unlock(&key); @@ -1481,6 +1573,7 @@ static int umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) { struct umtx_q *uq; + int pri; mtx_lock(&umtx_lock); if (pi->pi_owner == owner) { @@ -1498,8 +1591,6 @@ umtx_pi_claim(struct umtx_pi *pi, struct thread *owner) umtx_pi_setowner(pi, owner); uq = TAILQ_FIRST(&pi->pi_blocked); if (uq != NULL) { - int pri; - pri = UPRI(uq->uq_thread); thread_lock(owner); if (pri < UPRI(owner)) @@ -1537,15 +1628,15 @@ umtx_pi_adjust(struct thread *td, u_char oldpri) * Sleep on a PI mutex. */ static int -umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, - uint32_t owner, const char *wmesg, struct abs_timeout *timo) +umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, uint32_t owner, + const char *wmesg, struct abs_timeout *timo, bool shared) { struct umtxq_chain *uc; struct thread *td, *td1; struct umtx_q *uq1; - int pri; - int error = 0; + int error, pri; + error = 0; td = uq->uq_thread; KASSERT(td == curthread, ("inconsistent uq_thread")); uc = umtxq_getchain(&uq->uq_key); @@ -1555,8 +1646,7 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi, mtx_lock(&umtx_lock); if (pi->pi_owner == NULL) { mtx_unlock(&umtx_lock); - /* XXX Only look up thread in current process. */ - td1 = tdfind(owner, curproc->p_pid); + td1 = tdfind(owner, shared ? -1 : td->td_proc->p_pid); mtx_lock(&umtx_lock); if (td1 != NULL) { if (pi->pi_owner == NULL) @@ -1680,13 +1770,14 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, struct abs_timeout timo; struct umtx_q *uq; struct umtx_pi *pi, *new_pi; - uint32_t id, owner, old; + uint32_t id, old_owner, owner, old; int error, rv; id = td->td_tid; uq = td->td_umtxq; - if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), + if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? + TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); @@ -1737,17 +1828,23 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, break; } + if (owner == UMUTEX_RB_NOTRECOV) { + error = ENOTRECOVERABLE; + break; + } + /* If no one owns it but it is contested try to acquire it. */ - if (owner == UMUTEX_CONTESTED) { - rv = casueword32(&m->m_owner, - UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); + if (owner == UMUTEX_CONTESTED || owner == UMUTEX_RB_OWNERDEAD) { + old_owner = owner; + rv = casueword32(&m->m_owner, owner, &owner, + id | UMUTEX_CONTESTED); /* The address was invalid. */ if (rv == -1) { error = EFAULT; break; } - if (owner == UMUTEX_CONTESTED) { + if (owner == old_owner) { umtxq_lock(&uq->uq_key); umtxq_busy(&uq->uq_key); error = umtx_pi_claim(pi, td); @@ -1762,8 +1859,11 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, */ (void)casuword32(&m->m_owner, id | UMUTEX_CONTESTED, - UMUTEX_CONTESTED); + old_owner); } + if (error == 0 && + old_owner == UMUTEX_RB_OWNERDEAD) + error = EOWNERDEAD; break; } @@ -1802,8 +1902,8 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, * either some one else has acquired the lock or it has been * released. */ - rv = casueword32(&m->m_owner, owner, &old, - owner | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, owner, &old, owner | + UMUTEX_CONTESTED); /* The address was invalid. */ if (rv == -1) { @@ -1816,11 +1916,14 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, /* * We set the contested bit, sleep. Otherwise the lock changed * and we need to retry or we lost a race to the thread - * unlocking the umtx. + * unlocking the umtx. Note that the UMUTEX_RB_OWNERDEAD + * value for owner is impossible there. */ if (old == owner) { - error = umtxq_sleep_pi(uq, pi, owner & ~UMUTEX_CONTESTED, - "umtxpi", timeout == NULL ? NULL : &timo); + error = umtxq_sleep_pi(uq, pi, + owner & ~UMUTEX_CONTESTED, + "umtxpi", timeout == NULL ? NULL : &timo, + (flags & USYNC_PROCESS_SHARED) != 0); if (error != 0) continue; } else { @@ -1845,15 +1948,13 @@ do_lock_pi(struct thread *td, struct umutex *m, uint32_t flags, * Unlock a PI mutex. */ static int -do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) +do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags, bool rb) { struct umtx_key key; struct umtx_q *uq_first, *uq_first2, *uq_me; struct umtx_pi *pi, *pi2; - uint32_t owner, old, id; - int error; - int count; - int pri; + uint32_t id, new_owner, old, owner; + int count, error, pri; id = td->td_tid; /* @@ -1866,9 +1967,11 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) if ((owner & ~UMUTEX_CONTESTED) != id) return (EPERM); + new_owner = umtx_unlock_val(flags, rb); + /* This should be done in userland */ if ((owner & UMUTEX_CONTESTED) == 0) { - error = casueword32(&m->m_owner, owner, &old, UMUTEX_UNOWNED); + error = casueword32(&m->m_owner, owner, &old, new_owner); if (error == -1) return (EFAULT); if (old == owner) @@ -1877,7 +1980,8 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) } /* We should only ever be in here for contested locks */ - if ((error = umtx_key_get(m, TYPE_PI_UMUTEX, GET_SHARE(flags), + if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? + TYPE_PI_ROBUST_UMUTEX : TYPE_PI_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); @@ -1888,7 +1992,7 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) mtx_lock(&umtx_lock); pi = uq_first->uq_pi_blocked; KASSERT(pi != NULL, ("pi == NULL?")); - if (pi->pi_owner != td) { + if (pi->pi_owner != td && !(rb && pi->pi_owner == NULL)) { mtx_unlock(&umtx_lock); umtxq_unbusy(&key); umtxq_unlock(&key); @@ -1897,11 +2001,12 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) return (EPERM); } uq_me = td->td_umtxq; - umtx_pi_disown(pi); + if (pi->pi_owner == td) + umtx_pi_disown(pi); /* get highest priority thread which is still sleeping. */ uq_first = TAILQ_FIRST(&pi->pi_blocked); while (uq_first != NULL && - (uq_first->uq_flags & UQF_UMTXQ) == 0) { + (uq_first->uq_flags & UQF_UMTXQ) == 0) { uq_first = TAILQ_NEXT(uq_first, uq_lockq); } pri = PRI_MAX; @@ -1945,8 +2050,10 @@ do_unlock_pi(struct thread *td, struct umutex *m, uint32_t flags) * there is zero or one thread only waiting for it. * Otherwise, it must be marked as contested. */ - error = casueword32(&m->m_owner, owner, &old, - count <= 1 ? UMUTEX_UNOWNED : UMUTEX_CONTESTED); + + if (count > 1) + new_owner |= UMUTEX_CONTESTED; + error = casueword32(&m->m_owner, owner, &old, new_owner); umtxq_unbusy_unlocked(&key); umtx_key_release(&key); @@ -1973,7 +2080,8 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, id = td->td_tid; uq = td->td_umtxq; - if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), + if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? + TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), &uq->uq_key)) != 0) return (error); @@ -2013,8 +2121,8 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, } mtx_unlock(&umtx_lock); - rv = casueword32(&m->m_owner, - UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, + id | UMUTEX_CONTESTED); /* The address was invalid. */ if (rv == -1) { error = EFAULT; @@ -2024,6 +2132,21 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, if (owner == UMUTEX_CONTESTED) { error = 0; break; + } else if (owner == UMUTEX_RB_OWNERDEAD) { + rv = casueword32(&m->m_owner, UMUTEX_RB_OWNERDEAD, + &owner, id | UMUTEX_CONTESTED); + if (rv == -1) { + error = EFAULT; + break; + } + if (owner == UMUTEX_RB_OWNERDEAD) { + error = EOWNERDEAD; /* success */ + break; + } + error = 0; + } else if (owner == UMUTEX_RB_NOTRECOV) { + error = ENOTRECOVERABLE; + break; } if (try != 0) { @@ -2064,7 +2187,7 @@ do_lock_pp(struct thread *td, struct umutex *m, uint32_t flags, mtx_unlock(&umtx_lock); } - if (error != 0) { + if (error != 0 && error != EOWNERDEAD) { mtx_lock(&umtx_lock); uq->uq_inherited_pri = old_inherited_pri; pri = PRI_MAX; @@ -2093,13 +2216,12 @@ out: * Unlock a PP mutex. */ static int -do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) +do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags, bool rb) { struct umtx_key key; struct umtx_q *uq, *uq2; struct umtx_pi *pi; - uint32_t owner, id; - uint32_t rceiling; + uint32_t id, owner, rceiling; int error, pri, new_inherited_pri, su; id = td->td_tid; @@ -2129,7 +2251,8 @@ do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) new_inherited_pri = PRI_MIN_REALTIME + rceiling; } - if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), + if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? + TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), &key)) != 0) return (error); umtxq_lock(&key); @@ -2141,7 +2264,8 @@ do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) * to lock the mutex, it is necessary because thread priority * has to be adjusted for such mutex. */ - error = suword32(&m->m_owner, UMUTEX_CONTESTED); + error = suword32(&m->m_owner, umtx_unlock_val(flags, rb) | + UMUTEX_CONTESTED); umtxq_lock(&key); if (error == 0) @@ -2176,13 +2300,11 @@ do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags) static int do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, - uint32_t *old_ceiling) + uint32_t *old_ceiling) { struct umtx_q *uq; - uint32_t save_ceiling; - uint32_t owner, id; - uint32_t flags; - int error, rv; + uint32_t flags, id, owner, save_ceiling; + int error, rv, rv1; error = fueword32(&m->m_flags, &flags); if (error == -1) @@ -2193,8 +2315,9 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, return (EINVAL); id = td->td_tid; uq = td->td_umtxq; - if ((error = umtx_key_get(m, TYPE_PP_UMUTEX, GET_SHARE(flags), - &uq->uq_key)) != 0) + if ((error = umtx_key_get(m, (flags & UMUTEX_ROBUST) != 0 ? + TYPE_PP_ROBUST_UMUTEX : TYPE_PP_UMUTEX, GET_SHARE(flags), + &uq->uq_key)) != 0) return (error); for (;;) { umtxq_lock(&uq->uq_key); @@ -2207,23 +2330,31 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, break; } - rv = casueword32(&m->m_owner, - UMUTEX_CONTESTED, &owner, id | UMUTEX_CONTESTED); + rv = casueword32(&m->m_owner, UMUTEX_CONTESTED, &owner, + id | UMUTEX_CONTESTED); if (rv == -1) { error = EFAULT; break; } if (owner == UMUTEX_CONTESTED) { - suword32(&m->m_ceilings[0], ceiling); - suword32(&m->m_owner, UMUTEX_CONTESTED); - error = 0; + rv = suword32(&m->m_ceilings[0], ceiling); + rv1 = suword32(&m->m_owner, UMUTEX_CONTESTED); + error = (rv == 0 && rv1 == 0) ? 0: EFAULT; break; } if ((owner & ~UMUTEX_CONTESTED) == id) { - suword32(&m->m_ceilings[0], ceiling); - error = 0; + rv = suword32(&m->m_ceilings[0], ceiling); + error = rv == 0 ? 0 : EFAULT; + break; + } + + if (owner == UMUTEX_RB_OWNERDEAD) { + error = EOWNERDEAD; + break; + } else if (owner == UMUTEX_RB_NOTRECOV) { + error = ENOTRECOVERABLE; break; } @@ -2252,8 +2383,10 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling, umtxq_unbusy(&uq->uq_key); umtxq_unlock(&uq->uq_key); umtx_key_release(&uq->uq_key); - if (error == 0 && old_ceiling != NULL) - suword32(old_ceiling, save_ceiling); + if (error == 0 && old_ceiling != NULL) { + rv = suword32(old_ceiling, save_ceiling); + error = rv == 0 ? 0 : EFAULT; + } return (error); } @@ -2271,7 +2404,7 @@ do_lock_umutex(struct thread *td, struct umutex *m, if (error == -1) return (EFAULT); - switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { + switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { case 0: error = do_lock_normal(td, m, flags, timeout, mode); break; @@ -2299,7 +2432,7 @@ do_lock_umutex(struct thread *td, struct umutex *m, * Unlock a userland POSIX mutex. */ static int -do_unlock_umutex(struct thread *td, struct umutex *m) +do_unlock_umutex(struct thread *td, struct umutex *m, bool rb) { uint32_t flags; int error; @@ -2308,13 +2441,13 @@ do_unlock_umutex(struct thread *td, struct umutex *m) if (error == -1) return (EFAULT); - switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { + switch (flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) { case 0: - return (do_unlock_normal(td, m, flags)); + return (do_unlock_normal(td, m, flags, rb)); case UMUTEX_PRIO_INHERIT: - return (do_unlock_pi(td, m, flags)); + return (do_unlock_pi(td, m, flags, rb)); case UMUTEX_PRIO_PROTECT: - return (do_unlock_pp(td, m, flags)); + return (do_unlock_pp(td, m, flags, rb)); } return (EINVAL); @@ -2322,7 +2455,7 @@ do_unlock_umutex(struct thread *td, struct umutex *m) static int do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, - struct timespec *timeout, u_long wflags) + struct timespec *timeout, u_long wflags) { struct abs_timeout timo; struct umtx_q *uq; @@ -2368,11 +2501,11 @@ do_cv_wait(struct thread *td, struct ucond *cv, struct umutex *m, umtxq_unbusy_unlocked(&uq->uq_key); - error = do_unlock_umutex(td, m); + error = do_unlock_umutex(td, m, false); if (timeout != NULL) - abs_timeout_init(&timo, clockid, ((wflags & CVWAIT_ABSTIME) != 0), - timeout); + abs_timeout_init(&timo, clockid, (wflags & CVWAIT_ABSTIME) != 0, + timeout); umtxq_lock(&uq->uq_key); if (error == 0) { @@ -3181,7 +3314,7 @@ __umtx_op_wait(struct thread *td, struct _umtx_op_args *uap) return (error); tm_p = &timeout; } - return do_wait(td, uap->obj, uap->val, tm_p, 0, 0); + return (do_wait(td, uap->obj, uap->val, tm_p, 0, 0)); } static int @@ -3199,7 +3332,7 @@ __umtx_op_wait_uint(struct thread *td, struct _umtx_op_args *uap) return (error); tm_p = &timeout; } - return do_wait(td, uap->obj, uap->val, tm_p, 1, 0); + return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0)); } static int @@ -3217,12 +3350,13 @@ __umtx_op_wait_uint_private(struct thread *td, struct _umtx_op_args *uap) return (error); tm_p = &timeout; } - return do_wait(td, uap->obj, uap->val, tm_p, 1, 1); + return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1)); } static int __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) { + return (kern_umtx_wake(td, uap->obj, uap->val, 0)); } @@ -3230,24 +3364,20 @@ __umtx_op_wake(struct thread *td, struct _umtx_op_args *uap) static int __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap) { - int count = uap->val; - void *uaddrs[BATCH_SIZE]; - char **upp = (char **)uap->obj; - int tocopy; - int error = 0; - int i, pos = 0; + char *uaddrs[BATCH_SIZE], **upp; + int count, error, i, pos, tocopy; - while (count > 0) { - tocopy = count; - if (tocopy > BATCH_SIZE) - tocopy = BATCH_SIZE; - error = copyin(upp+pos, uaddrs, tocopy * sizeof(char *)); + upp = (char **)uap->obj; + error = 0; + for (count = uap->val, pos = 0; count > 0; count -= tocopy, + pos += tocopy) { + tocopy = MIN(count, BATCH_SIZE); + error = copyin(upp + pos, uaddrs, tocopy * sizeof(char *)); if (error != 0) break; for (i = 0; i < tocopy; ++i) kern_umtx_wake(td, uaddrs[i], INT_MAX, 1); - count -= tocopy; - pos += tocopy; + maybe_yield(); } return (error); } @@ -3255,6 +3385,7 @@ __umtx_op_nwake_private(struct thread *td, struct _umtx_op_args *uap) static int __umtx_op_wake_private(struct thread *td, struct _umtx_op_args *uap) { + return (kern_umtx_wake(td, uap->obj, uap->val, 1)); } @@ -3274,13 +3405,14 @@ __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap) return (error); tm_p = &timeout; } - return do_lock_umutex(td, uap->obj, tm_p, 0); + return (do_lock_umutex(td, uap->obj, tm_p, 0)); } static int __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap) { - return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY); + + return (do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY)); } static int @@ -3299,25 +3431,28 @@ __umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap) return (error); tm_p = &timeout; } - return do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT); + return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT)); } static int __umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap) { - return do_wake_umutex(td, uap->obj); + + return (do_wake_umutex(td, uap->obj)); } static int __umtx_op_unlock_umutex(struct thread *td, struct _umtx_op_args *uap) { - return do_unlock_umutex(td, uap->obj); + + return (do_unlock_umutex(td, uap->obj, false)); } static int __umtx_op_set_ceiling(struct thread *td, struct _umtx_op_args *uap) { - return do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1); + + return (do_set_ceiling(td, uap->obj, uap->val, uap->uaddr1)); } static int @@ -3341,13 +3476,15 @@ __umtx_op_cv_wait(struct thread *td, struct _umtx_op_args *uap) static int __umtx_op_cv_signal(struct thread *td, struct _umtx_op_args *uap) { - return do_cv_signal(td, uap->obj); + + return (do_cv_signal(td, uap->obj)); } static int __umtx_op_cv_broadcast(struct thread *td, struct _umtx_op_args *uap) { - return do_cv_broadcast(td, uap->obj); + + return (do_cv_broadcast(td, uap->obj)); } static int @@ -3392,7 +3529,8 @@ __umtx_op_rw_wrlock(struct thread *td, struct _umtx_op_args *uap) static int __umtx_op_rw_unlock(struct thread *td, struct _umtx_op_args *uap) { - return do_rw_unlock(td, uap->obj); + + return (do_rw_unlock(td, uap->obj)); } #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) @@ -3760,6 +3898,31 @@ __umtx_op_shm(struct thread *td, struct _umtx_op_args *uap) return (umtx_shm(td, uap->uaddr1, uap->val)); } +static int +umtx_robust_lists(struct thread *td, struct umtx_robust_lists_params *rbp) +{ + + td->td_rb_list = rbp->robust_list_offset; + td->td_rbp_list = rbp->robust_priv_list_offset; + td->td_rb_inact = rbp->robust_inact_offset; + return (0); +} + +static int +__umtx_op_robust_lists(struct thread *td, struct _umtx_op_args *uap) +{ + struct umtx_robust_lists_params rb; + int error; + + if (uap->val > sizeof(rb)) + return (EINVAL); + bzero(&rb, sizeof(rb)); + error = copyin(uap->uaddr1, &rb, uap->val); + if (error != 0) + return (error); + return (umtx_robust_lists(td, &rb)); +} + typedef int (*_umtx_op_func)(struct thread *td, struct _umtx_op_args *uap); static const _umtx_op_func op_table[] = { @@ -3794,6 +3957,7 @@ static const _umtx_op_func op_table[] = { [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait, [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake, [UMTX_OP_SHM] = __umtx_op_shm, + [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists, }; int @@ -3877,7 +4041,7 @@ __umtx_op_wait_compat32(struct thread *td, struct _umtx_op_args *uap) return (error); tm_p = &timeout; } - return do_wait(td, uap->obj, uap->val, tm_p, 1, 0); + return (do_wait(td, uap->obj, uap->val, tm_p, 1, 0)); } static int @@ -3896,7 +4060,7 @@ __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) return (error); tm_p = &timeout; } - return do_lock_umutex(td, uap->obj, tm_p, 0); + return (do_lock_umutex(td, uap->obj, tm_p, 0)); } static int @@ -3915,7 +4079,7 @@ __umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap) return (error); tm_p = &timeout; } - return do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT); + return (do_lock_umutex(td, uap->obj, tm_p, _UMUTEX_WAIT)); } static int @@ -3989,7 +4153,7 @@ __umtx_op_wait_uint_private_compat32(struct thread *td, struct _umtx_op_args *ua return (error); tm_p = &timeout; } - return do_wait(td, uap->obj, uap->val, tm_p, 1, 1); + return (do_wait(td, uap->obj, uap->val, tm_p, 1, 1)); } #if defined(COMPAT_FREEBSD9) || defined(COMPAT_FREEBSD10) @@ -4035,34 +4199,56 @@ __umtx_op_sem2_wait_compat32(struct thread *td, struct _umtx_op_args *uap) static int __umtx_op_nwake_private32(struct thread *td, struct _umtx_op_args *uap) { - int count = uap->val; - uint32_t uaddrs[BATCH_SIZE]; - uint32_t **upp = (uint32_t **)uap->obj; - int tocopy; - int error = 0; - int i, pos = 0; + uint32_t uaddrs[BATCH_SIZE], **upp; + int count, error, i, pos, tocopy; - while (count > 0) { - tocopy = count; - if (tocopy > BATCH_SIZE) - tocopy = BATCH_SIZE; - error = copyin(upp+pos, uaddrs, tocopy * sizeof(uint32_t)); + upp = (uint32_t **)uap->obj; + error = 0; + for (count = uap->val, pos = 0; count > 0; count -= tocopy, + pos += tocopy) { + tocopy = MIN(count, BATCH_SIZE); + error = copyin(upp + pos, uaddrs, tocopy * sizeof(uint32_t)); if (error != 0) break; for (i = 0; i < tocopy; ++i) kern_umtx_wake(td, (void *)(intptr_t)uaddrs[i], - INT_MAX, 1); - count -= tocopy; - pos += tocopy; + INT_MAX, 1); + maybe_yield(); } return (error); } +struct umtx_robust_lists_params_compat32 { + uint32_t robust_list_offset; + uint32_t robust_priv_list_offset; + uint32_t robust_inact_offset; +}; + +static int +__umtx_op_robust_lists_compat32(struct thread *td, struct _umtx_op_args *uap) +{ + struct umtx_robust_lists_params rb; + struct umtx_robust_lists_params_compat32 rb32; + int error; + + if (uap->val > sizeof(rb32)) + return (EINVAL); + bzero(&rb, sizeof(rb)); + bzero(&rb32, sizeof(rb32)); + error = copyin(uap->uaddr1, &rb32, uap->val); + if (error != 0) + return (error); + rb.robust_list_offset = rb32.robust_list_offset; + rb.robust_priv_list_offset = rb32.robust_priv_list_offset; + rb.robust_inact_offset = rb32.robust_inact_offset; + return (umtx_robust_lists(td, &rb)); +} + static const _umtx_op_func op_table_compat32[] = { [UMTX_OP_RESERVED0] = __umtx_op_unimpl, [UMTX_OP_RESERVED1] = __umtx_op_unimpl, - [UMTX_OP_WAIT] = __umtx_op_wait_compat32, - [UMTX_OP_WAKE] = __umtx_op_wake, + [UMTX_OP_WAIT] = __umtx_op_wait_compat32, + [UMTX_OP_WAKE] = __umtx_op_wake, [UMTX_OP_MUTEX_TRYLOCK] = __umtx_op_trylock_umutex, [UMTX_OP_MUTEX_LOCK] = __umtx_op_lock_umutex_compat32, [UMTX_OP_MUTEX_UNLOCK] = __umtx_op_unlock_umutex, @@ -4090,6 +4276,7 @@ static const _umtx_op_func op_table_compat32[] = { [UMTX_OP_SEM2_WAIT] = __umtx_op_sem2_wait_compat32, [UMTX_OP_SEM2_WAKE] = __umtx_op_sem2_wake, [UMTX_OP_SHM] = __umtx_op_shm, + [UMTX_OP_ROBUST_LISTS] = __umtx_op_robust_lists_compat32, }; int @@ -4107,6 +4294,7 @@ freebsd32_umtx_op(struct thread *td, struct freebsd32_umtx_op_args *uap) void umtx_thread_init(struct thread *td) { + td->td_umtxq = umtxq_alloc(); td->td_umtxq->uq_thread = td; } @@ -4114,6 +4302,7 @@ umtx_thread_init(struct thread *td) void umtx_thread_fini(struct thread *td) { + umtxq_free(td->td_umtxq); } @@ -4136,12 +4325,32 @@ umtx_thread_alloc(struct thread *td) /* * exec() hook. + * + * Clear robust lists for all process' threads, not delaying the + * cleanup to thread_exit hook, since the relevant address space is + * destroyed right now. */ static void -umtx_exec_hook(void *arg __unused, struct proc *p __unused, - struct image_params *imgp __unused) +umtx_exec_hook(void *arg __unused, struct proc *p, + struct image_params *imgp __unused) { - umtx_thread_cleanup(curthread); + struct thread *td; + + KASSERT(p == curproc, ("need curproc")); + PROC_LOCK(p); + KASSERT((p->p_flag & P_HADTHREADS) == 0 || + (p->p_flag & P_STOPPED_SINGLE) != 0, + ("curproc must be single-threaded")); + FOREACH_THREAD_IN_PROC(p, td) { + KASSERT(td == curthread || + ((td->td_flags & TDF_BOUNDARY) != 0 && TD_IS_SUSPENDED(td)), + ("running thread %p %p", p, td)); + PROC_UNLOCK(p); + umtx_thread_cleanup(td); + PROC_LOCK(p); + td->td_rb_list = td->td_rbp_list = td->td_rb_inact = 0; + } + PROC_UNLOCK(p); } /* @@ -4150,29 +4359,136 @@ umtx_exec_hook(void *arg __unused, struct proc *p __unused, void umtx_thread_exit(struct thread *td) { + umtx_thread_cleanup(td); } +static int +umtx_read_uptr(struct thread *td, uintptr_t ptr, uintptr_t *res) +{ + u_long res1; +#ifdef COMPAT_FREEBSD32 + uint32_t res32; +#endif + int error; + +#ifdef COMPAT_FREEBSD32 + if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { + error = fueword32((void *)ptr, &res32); + if (error == 0) + res1 = res32; + } else +#endif + { + error = fueword((void *)ptr, &res1); + } + if (error == 0) + *res = res1; + else + error = EFAULT; + return (error); +} + +static void +umtx_read_rb_list(struct thread *td, struct umutex *m, uintptr_t *rb_list) +{ +#ifdef COMPAT_FREEBSD32 + struct umutex32 m32; + + if (SV_PROC_FLAG(td->td_proc, SV_ILP32)) { + memcpy(&m32, m, sizeof(m32)); + *rb_list = m32.m_rb_lnk; + } else +#endif + *rb_list = m->m_rb_lnk; +} + +static int +umtx_handle_rb(struct thread *td, uintptr_t rbp, uintptr_t *rb_list, bool inact) +{ + struct umutex m; + int error; + + KASSERT(td->td_proc == curproc, ("need current vmspace")); + error = copyin((void *)rbp, &m, sizeof(m)); + if (error != 0) + return (error); + if (rb_list != NULL) + umtx_read_rb_list(td, &m, rb_list); + if ((m.m_flags & UMUTEX_ROBUST) == 0) + return (EINVAL); + if ((m.m_owner & ~UMUTEX_CONTESTED) != td->td_tid) + /* inact is cleared after unlock, allow the inconsistency */ + return (inact ? 0 : EINVAL); + return (do_unlock_umutex(td, (struct umutex *)rbp, true)); +} + +static void +umtx_cleanup_rb_list(struct thread *td, uintptr_t rb_list, uintptr_t *rb_inact, + const char *name) +{ + int error, i; + uintptr_t rbp; + bool inact; + + if (rb_list == 0) + return; + error = umtx_read_uptr(td, rb_list, &rbp); + for (i = 0; error == 0 && rbp != 0 && i < umtx_max_rb; i++) { + if (rbp == *rb_inact) { + inact = true; + *rb_inact = 0; + } else + inact = false; + error = umtx_handle_rb(td, rbp, &rbp, inact); + } + if (i == umtx_max_rb && umtx_verbose_rb) { + uprintf("comm %s pid %d: reached umtx %smax rb %d\n", + td->td_proc->p_comm, td->td_proc->p_pid, name, umtx_max_rb); + } + if (error != 0 && umtx_verbose_rb) { + uprintf("comm %s pid %d: handling %srb error %d\n", + td->td_proc->p_comm, td->td_proc->p_pid, name, error); + } +} + /* - * clean up umtx data. + * Clean up umtx data. */ static void umtx_thread_cleanup(struct thread *td) { struct umtx_q *uq; struct umtx_pi *pi; + uintptr_t rb_inact; - if ((uq = td->td_umtxq) == NULL) - return; - - mtx_lock(&umtx_lock); - uq->uq_inherited_pri = PRI_MAX; - while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { - pi->pi_owner = NULL; - TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); + /* + * Disown pi mutexes. + */ + uq = td->td_umtxq; + if (uq != NULL) { + mtx_lock(&umtx_lock); + uq->uq_inherited_pri = PRI_MAX; + while ((pi = TAILQ_FIRST(&uq->uq_pi_contested)) != NULL) { + pi->pi_owner = NULL; + TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link); + } + mtx_unlock(&umtx_lock); + thread_lock(td); + sched_lend_user_prio(td, PRI_MAX); + thread_unlock(td); } - mtx_unlock(&umtx_lock); - thread_lock(td); - sched_lend_user_prio(td, PRI_MAX); - thread_unlock(td); + + /* + * Handle terminated robust mutexes. Must be done after + * robust pi disown, otherwise unlock could see unowned + * entries. + */ + rb_inact = td->td_rb_inact; + if (rb_inact != 0) + (void)umtx_read_uptr(td, rb_inact, &rb_inact); + umtx_cleanup_rb_list(td, td->td_rb_list, &rb_inact, ""); + umtx_cleanup_rb_list(td, td->td_rbp_list, &rb_inact, "priv "); + if (rb_inact != 0) + (void)umtx_handle_rb(td, rb_inact, NULL, true); } diff --git a/sys/sys/_umtx.h b/sys/sys/_umtx.h index e8a400f..d94f86b 100644 --- a/sys/sys/_umtx.h +++ b/sys/sys/_umtx.h @@ -37,7 +37,11 @@ struct umutex { volatile __lwpid_t m_owner; /* Owner of the mutex */ __uint32_t m_flags; /* Flags of the mutex */ __uint32_t m_ceilings[2]; /* Priority protect ceiling */ - __uint32_t m_spare[4]; + __uintptr_t m_rb_lnk; /* Robust linkage */ +#ifndef __LP64__ + __uint32_t m_pad; +#endif + __uint32_t m_spare[2]; }; struct ucond { diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 35303fc..1cb34bf 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -282,6 +282,9 @@ struct thread { int td_no_sleeping; /* (k) Sleeping disabled count. */ int td_dom_rr_idx; /* (k) RR Numa domain selection. */ void *td_su; /* (k) FFS SU private */ + uintptr_t td_rb_list; /* (k) Robust list head. */ + uintptr_t td_rbp_list; /* (k) Robust priv list head. */ + uintptr_t td_rb_inact; /* (k) Current in-action mutex loc. */ #define td_endzero td_sigmask /* Copied during fork1() or create_thread(). */ diff --git a/sys/sys/umtx.h b/sys/sys/umtx.h index 325148c..d2ff5ac 100644 --- a/sys/sys/umtx.h +++ b/sys/sys/umtx.h @@ -32,13 +32,26 @@ #include <sys/_umtx.h> +/* Common lock flags */ #define USYNC_PROCESS_SHARED 0x0001 /* Process shared sync objs */ -#define UMUTEX_UNOWNED 0x0 -#define UMUTEX_CONTESTED 0x80000000U - +/* umutex flags */ #define UMUTEX_PRIO_INHERIT 0x0004 /* Priority inherited mutex */ #define UMUTEX_PRIO_PROTECT 0x0008 /* Priority protect mutex */ +#define UMUTEX_ROBUST 0x0010 /* Robust mutex */ +#define UMUTEX_NONCONSISTENT 0x0020 /* Robust locked but not consistent */ + +/* + * The umutex.m_lock values and bits. The m_owner is the word which + * serves as the lock. Its high bit is the contention indicator and + * rest of bits records the owner TID. TIDs values start with PID_MAX + * + 2 and end by INT32_MAX. The low range [1..PID_MAX] is guaranteed + * to be useable as the special markers. + */ +#define UMUTEX_UNOWNED 0x0 +#define UMUTEX_CONTESTED 0x80000000U +#define UMUTEX_RB_OWNERDEAD (UMUTEX_CONTESTED | 0x10) +#define UMUTEX_RB_NOTRECOV (UMUTEX_CONTESTED | 0x11) /* urwlock flags */ #define URWLOCK_PREFER_READER 0x0002 @@ -84,6 +97,7 @@ #define UMTX_OP_SEM2_WAIT 23 #define UMTX_OP_SEM2_WAKE 24 #define UMTX_OP_SHM 25 +#define UMTX_OP_ROBUST_LISTS 26 /* Flags for UMTX_OP_CV_WAIT */ #define CVWAIT_CHECK_UNPARKING 0x01 @@ -100,6 +114,12 @@ #define UMTX_SHM_DESTROY 0x0004 #define UMTX_SHM_ALIVE 0x0008 +struct umtx_robust_lists_params { + uintptr_t robust_list_offset; + uintptr_t robust_priv_list_offset; + uintptr_t robust_inact_offset; +}; + #ifndef _KERNEL int _umtx_op(void *obj, int op, u_long val, void *uaddr, void *uaddr2); @@ -122,6 +142,8 @@ enum { TYPE_RWLOCK, TYPE_FUTEX, TYPE_SHM, + TYPE_PI_ROBUST_UMUTEX, + TYPE_PP_ROBUST_UMUTEX, }; /* Key to represent a unique userland synchronous object */ diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 4a0479b..cf6eda8 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -476,7 +476,7 @@ vm_object_vndeallocate(vm_object_t object) } #endif - if (object->ref_count == 1) + if (!umtx_shm_vnobj_persistent && object->ref_count == 1) umtx_shm_object_terminated(object); /* diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h index c68fdce..cfc583b 100644 --- a/sys/vm/vm_object.h +++ b/sys/vm/vm_object.h @@ -300,6 +300,7 @@ vm_object_cache_is_empty(vm_object_t object) void umtx_shm_object_init(vm_object_t object); void umtx_shm_object_terminated(vm_object_t object); +extern int umtx_shm_vnobj_persistent; vm_object_t vm_object_allocate (objtype_t, vm_pindex_t); boolean_t vm_object_coalesce(vm_object_t, vm_ooffset_t, vm_size_t, vm_size_t, diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index f39afc2..cc43976 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -164,6 +164,7 @@ vnode_destroy_vobject(struct vnode *vp) return; ASSERT_VOP_ELOCKED(vp, "vnode_destroy_vobject"); VM_OBJECT_WLOCK(obj); + umtx_shm_object_terminated(obj); if (obj->ref_count == 0) { /* * don't double-terminate the object |
