Implement process-shared locks support for libthr.so.3, without

breaking the ABI.  Special value is stored in the lock pointer to
indicate shared lock, and offline page in the shared memory is
allocated to store the actual lock.

Reviewed by:	vangyzen (previous version)
Discussed with:	deischen, emaste, jhb, rwatson,
	Martin Simmons <martin@lispworks.com>
Tested by:	pho
Sponsored by:	The FreeBSD Foundation

13 files changed, 788 insertions, 307 deletions

diff --git a/lib/libthr/thread/Makefile.inc b/lib/libthr/thread/Makefile.inc
index 1a7c63d..795ed39 100644
--- a/lib/libthr/thread/Makefile.inc
+++ b/lib/libthr/thread/Makefile.inc
@@ -36,6 +36,7 @@ SRCS+= \
 	thr_mutexattr.c \
 	thr_once.c \
 	thr_printf.c \
+	thr_pshared.c \
 	thr_pspinlock.c \
 	thr_resume_np.c \
 	thr_rtld.c \
diff --git a/lib/libthr/thread/thr_barrier.c b/lib/libthr/thread/thr_barrier.c
index 10b6346..45ca41a 100644
--- a/lib/libthr/thread/thr_barrier.c
+++ b/lib/libthr/thread/thr_barrier.c
@@ -41,14 +41,25 @@ __weak_reference(_pthread_barrier_destroy,	pthread_barrier_destroy);
 int
 _pthread_barrier_destroy(pthread_barrier_t *barrier)
 {
-	pthread_barrier_t	bar;
-	struct pthread		*curthread;
+	pthread_barrier_t bar;
+	struct pthread *curthread;
+	int pshared;
 
 	if (barrier == NULL || *barrier == NULL)
 		return (EINVAL);
 
+	if (*barrier == THR_PSHARED_PTR) {
+		bar = __thr_pshared_offpage(barrier, 0);
+		if (bar == NULL) {
+			*barrier = NULL;
+			return (0);
+		}
+		pshared = 1;
+	} else {
+		bar = *barrier;
+		pshared = 0;
+	}
 	curthread = _get_curthread();
-	bar = *barrier;
 	THR_UMUTEX_LOCK(curthread, &bar->b_lock);
 	if (bar->b_destroying) {
 		THR_UMUTEX_UNLOCK(curthread, &bar->b_lock);
@@ -71,37 +82,52 @@ _pthread_barrier_destroy(pthread_barrier_t *barrier)
 	THR_UMUTEX_UNLOCK(curthread, &bar->b_lock);
 
 	*barrier = NULL;
-	free(bar);
+	if (pshared)
+		__thr_pshared_destroy(barrier);
+	else
+		free(bar);
 	return (0);
 }
 
 int
 _pthread_barrier_init(pthread_barrier_t *barrier,
-		      const pthread_barrierattr_t *attr, unsigned count)
+    const pthread_barrierattr_t *attr, unsigned count)
 {
-	pthread_barrier_t	bar;
-
-	(void)attr;
+	pthread_barrier_t bar;
+	int pshared;
 
 	if (barrier == NULL || count <= 0)
 		return (EINVAL);
 
-	bar = calloc(1, sizeof(struct pthread_barrier));
-	if (bar == NULL)
-		return (ENOMEM);
+	if (attr == NULL || *attr == NULL ||
+	    (*attr)->pshared == PTHREAD_PROCESS_PRIVATE) {
+		bar = calloc(1, sizeof(struct pthread_barrier));
+		if (bar == NULL)
+			return (ENOMEM);
+		*barrier = bar;
+		pshared = 0;
+	} else {
+		bar = __thr_pshared_offpage(barrier, 1);
+		if (bar == NULL)
+			return (EFAULT);
+		*barrier = THR_PSHARED_PTR;
+		pshared = 1;
+	}
 
 	_thr_umutex_init(&bar->b_lock);
 	_thr_ucond_init(&bar->b_cv);
-	bar->b_count	= count;
-	*barrier	= bar;
-
+	if (pshared) {
+		bar->b_lock.m_flags |= USYNC_PROCESS_SHARED;
+		bar->b_cv.c_flags |= USYNC_PROCESS_SHARED;
+	}
+	bar->b_count = count;
 	return (0);
 }
 
 int
 _pthread_barrier_wait(pthread_barrier_t *barrier)
 {
-	struct pthread *curthread = _get_curthread();
+	struct pthread *curthread;
 	pthread_barrier_t bar;
 	int64_t cycle;
 	int ret;
@@ -109,7 +135,14 @@ _pthread_barrier_wait(pthread_barrier_t *barrier)
 	if (barrier == NULL || *barrier == NULL)
 		return (EINVAL);
 
-	bar = *barrier;
+	if (*barrier == THR_PSHARED_PTR) {
+		bar = __thr_pshared_offpage(barrier, 0);
+		if (bar == NULL)
+			return (EINVAL);
+	} else {
+		bar = *barrier;
+	}
+	curthread = _get_curthread();
 	THR_UMUTEX_LOCK(curthread, &bar->b_lock);
 	if (++bar->b_waiters == bar->b_count) {
 		/* Current thread is lastest thread */
diff --git a/lib/libthr/thread/thr_barrierattr.c b/lib/libthr/thread/thr_barrierattr.c
index 32b9723..e98a66f 100644
--- a/lib/libthr/thread/thr_barrierattr.c
+++ b/lib/libthr/thread/thr_barrierattr.c
@@ -56,7 +56,7 @@ _pthread_barrierattr_destroy(pthread_barrierattr_t *attr)
 
 int
 _pthread_barrierattr_getpshared(const pthread_barrierattr_t *attr,
-	int *pshared)
+    int *pshared)
 {
 
 	if (attr == NULL || *attr == NULL)
@@ -84,11 +84,9 @@ int
 _pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared)
 {
 
-	if (attr == NULL || *attr == NULL)
-		return (EINVAL);
-
-	/* Only PTHREAD_PROCESS_PRIVATE is supported. */
-	if (pshared != PTHREAD_PROCESS_PRIVATE)
+	if (attr == NULL || *attr == NULL ||
+	    (pshared != PTHREAD_PROCESS_PRIVATE &&
+	    pshared != PTHREAD_PROCESS_SHARED))
 		return (EINVAL);
 
 	(*attr)->pshared = pshared;
diff --git a/lib/libthr/thread/thr_cond.c b/lib/libthr/thread/thr_cond.c
index 71b4293..934f9a0 100644
--- a/lib/libthr/thread/thr_cond.c
+++ b/lib/libthr/thread/thr_cond.c
@@ -1,7 +1,11 @@
 /*
  * Copyright (c) 2005 David Xu <davidxu@freebsd.org>
+ * Copyright (c) 2015 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -63,29 +67,45 @@ __weak_reference(_pthread_cond_broadcast, pthread_cond_broadcast);
 
 #define CV_PSHARED(cvp)	(((cvp)->__flags & USYNC_PROCESS_SHARED) != 0)
 
+static void
+cond_init_body(struct pthread_cond *cvp, const struct pthread_cond_attr *cattr)
+{
+
+	if (cattr == NULL) {
+		cvp->__clock_id = CLOCK_REALTIME;
+	} else {
+		if (cattr->c_pshared)
+			cvp->__flags |= USYNC_PROCESS_SHARED;
+		cvp->__clock_id = cattr->c_clockid;
+	}
+}
+
 static int
 cond_init(pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
 {
-	struct pthread_cond	*cvp;
-	int	error = 0;
-
-	if ((cvp = (pthread_cond_t)
-	    calloc(1, sizeof(struct pthread_cond))) == NULL) {
-		error = ENOMEM;
+	struct pthread_cond *cvp;
+	const struct pthread_cond_attr *cattr;
+	int pshared;
+
+	cattr = cond_attr != NULL ? *cond_attr : NULL;
+	if (cattr == NULL || cattr->c_pshared == PTHREAD_PROCESS_PRIVATE) {
+		pshared = 0;
+		cvp = calloc(1, sizeof(struct pthread_cond));
+		if (cvp == NULL)
+			return (ENOMEM);
 	} else {
-		/*
-		 * Initialise the condition variable structure:
-		 */
-		if (cond_attr == NULL || *cond_attr == NULL) {
-			cvp->__clock_id = CLOCK_REALTIME;
-		} else {
-			if ((*cond_attr)->c_pshared)
-				cvp->__flags |= USYNC_PROCESS_SHARED;
-			cvp->__clock_id = (*cond_attr)->c_clockid;
-		}
-		*cond = cvp;
+		pshared = 1;
+		cvp = __thr_pshared_offpage(cond, 1);
+		if (cvp == NULL)
+			return (EFAULT);
 	}
-	return (error);
+
+	/*
+	 * Initialise the condition variable structure:
+	 */
+	cond_init_body(cvp, cattr);
+	*cond = pshared ? THR_PSHARED_PTR : cvp;
+	return (0);
 }
 
 static int
@@ -106,7 +126,11 @@ init_static(struct pthread *thread, pthread_cond_t *cond)
 }
 
 #define CHECK_AND_INIT_COND							\
-	if (__predict_false((cvp = (*cond)) <= THR_COND_DESTROYED)) {		\
+	if (*cond == THR_PSHARED_PTR) {						\
+		cvp = __thr_pshared_offpage(cond, 0);				\
+		if (cvp == NULL)						\
+			return (EINVAL);					\
+	} else if (__predict_false((cvp = (*cond)) <= THR_COND_DESTROYED)) {	\
 		if (cvp == THR_COND_INITIALIZER) {				\
 			int ret;						\
 			ret = init_static(_get_curthread(), cond);		\
@@ -129,21 +153,22 @@ _pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
 int
 _pthread_cond_destroy(pthread_cond_t *cond)
 {
-	struct pthread_cond	*cvp;
-	int			error = 0;
+	struct pthread_cond *cvp;
+	int error;
 
-	if ((cvp = *cond) == THR_COND_INITIALIZER)
-		error = 0;
-	else if (cvp == THR_COND_DESTROYED)
+	error = 0;
+	if (*cond == THR_PSHARED_PTR) {
+		cvp = __thr_pshared_offpage(cond, 0);
+		if (cvp != NULL)
+			__thr_pshared_destroy(cond);
+		*cond = THR_COND_DESTROYED;
+	} else if ((cvp = *cond) == THR_COND_INITIALIZER) {
+		/* nothing */
+	} else if (cvp == THR_COND_DESTROYED) {
 		error = EINVAL;
-	else {
+	} else {
 		cvp = *cond;
 		*cond = THR_COND_DESTROYED;
-
-		/*
-		 * Free the memory allocated for the condition
-		 * variable structure:
-		 */
 		free(cvp);
 	}
 	return (error);
@@ -297,7 +322,13 @@ cond_wait_common(pthread_cond_t *cond, pthread_mutex_t *mutex,
 
 	CHECK_AND_INIT_COND
 
-	mp = *mutex;
+	if (*mutex == THR_PSHARED_PTR) {
+		mp = __thr_pshared_offpage(mutex, 0);
+		if (mp == NULL)
+			return (EINVAL);
+	} else {
+		mp = *mutex;
+	}
 
 	if ((error = _mutex_owned(curthread, mp)) != 0)
 		return (error);
@@ -385,7 +416,7 @@ cond_signal_common(pthread_cond_t *cond)
 	td = _sleepq_first(sq);
 	mp = td->mutex_obj;
 	cvp->__has_user_waiters = _sleepq_remove(sq, td);
-	if (mp->m_owner == curthread) {
+	if (mp->m_owner == TID(curthread)) {
 		if (curthread->nwaiter_defer >= MAX_DEFER_WAITERS) {
 			_thr_wake_all(curthread->defer_waiters,
 					curthread->nwaiter_defer);
@@ -417,7 +448,7 @@ drop_cb(struct pthread *td, void *arg)
 	struct pthread *curthread = ba->curthread;
 
 	mp = td->mutex_obj;
-	if (mp->m_owner == curthread) {
+	if (mp->m_owner == TID(curthread)) {
 		if (curthread->nwaiter_defer >= MAX_DEFER_WAITERS) {
 			_thr_wake_all(curthread->defer_waiters,
 				curthread->nwaiter_defer);
diff --git a/lib/libthr/thread/thr_condattr.c b/lib/libthr/thread/thr_condattr.c
index 52b9f51..7ce827b 100644
--- a/lib/libthr/thread/thr_condattr.c
+++ b/lib/libthr/thread/thr_condattr.c
@@ -105,20 +105,21 @@ _pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock_id)
 int
 _pthread_condattr_getpshared(const pthread_condattr_t *attr, int *pshared)
 {
+
 	if (attr == NULL || *attr == NULL)
 		return (EINVAL);
-
-	*pshared = PTHREAD_PROCESS_PRIVATE;
+	*pshared = (*attr)->c_pshared;
 	return (0);
 }
 
 int
 _pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared)
 {
-	if (attr == NULL || *attr == NULL)
-		return (EINVAL);
 
-	if  (pshared != PTHREAD_PROCESS_PRIVATE)
+	if (attr == NULL || *attr == NULL ||
+	    (pshared != PTHREAD_PROCESS_PRIVATE &&
+	    pshared != PTHREAD_PROCESS_SHARED))
 		return (EINVAL);
+	(*attr)->c_pshared = pshared;
 	return (0);
 }
diff --git a/lib/libthr/thread/thr_create.c b/lib/libthr/thread/thr_create.c
index 701a1bd..81a95da 100644
--- a/lib/libthr/thread/thr_create.c
+++ b/lib/libthr/thread/thr_create.c
@@ -56,12 +56,12 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	struct thr_param param;
 	struct sched_param sched_param;
 	struct rtprio rtp;
-	int ret = 0, locked, create_suspended;
 	sigset_t set, oset;
-	cpuset_t *cpusetp = NULL;
-	int cpusetsize = 0;
-	int old_stack_prot;
+	cpuset_t *cpusetp;
+	int i, cpusetsize, create_suspended, locked, old_stack_prot, ret;
 
+	cpusetp = NULL;
+	ret = cpusetsize = 0;
 	_thr_check_init();
 
 	/*
@@ -118,8 +118,8 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	new_thread->cancel_enable = 1;
 	new_thread->cancel_async = 0;
 	/* Initialize the mutex queue: */
-	TAILQ_INIT(&new_thread->mutexq);
-	TAILQ_INIT(&new_thread->pp_mutexq);
+	for (i = 0; i < TMQ_NITEMS; i++)
+		TAILQ_INIT(&new_thread->mq[i]);
 
 	/* Initialise hooks in the thread structure: */
 	if (new_thread->attr.suspend == THR_CREATE_SUSPENDED) {
diff --git a/lib/libthr/thread/thr_init.c b/lib/libthr/thread/thr_init.c
index e0400e4..3c81299 100644
--- a/lib/libthr/thread/thr_init.c
+++ b/lib/libthr/thread/thr_init.c
@@ -91,13 +91,15 @@ struct pthread_attr _pthread_attr_default = {
 struct pthread_mutex_attr _pthread_mutexattr_default = {
 	.m_type = PTHREAD_MUTEX_DEFAULT,
 	.m_protocol = PTHREAD_PRIO_NONE,
-	.m_ceiling = 0
+	.m_ceiling = 0,
+	.m_pshared = PTHREAD_PROCESS_PRIVATE,
 };
 
 struct pthread_mutex_attr _pthread_mutexattr_adaptive_default = {
 	.m_type = PTHREAD_MUTEX_ADAPTIVE_NP,
 	.m_protocol = PTHREAD_PRIO_NONE,
-	.m_ceiling = 0
+	.m_ceiling = 0,
+	.m_pshared = PTHREAD_PROCESS_PRIVATE,
 };
 
 /* Default condition variable attributes: */
@@ -387,6 +389,7 @@ static void
 init_main_thread(struct pthread *thread)
 {
 	struct sched_param sched_param;
+	int i;
 
 	/* Setup the thread attributes. */
 	thr_self(&thread->tid);
@@ -428,9 +431,9 @@ init_main_thread(struct pthread *thread)
 	thread->cancel_enable = 1;
 	thread->cancel_async = 0;
 
-	/* Initialize the mutex queue: */
-	TAILQ_INIT(&thread->mutexq);
-	TAILQ_INIT(&thread->pp_mutexq);
+	/* Initialize the mutex queues */
+	for (i = 0; i < TMQ_NITEMS; i++)
+		TAILQ_INIT(&thread->mq[i]);
 
 	thread->state = PS_RUNNING;
 
@@ -463,6 +466,7 @@ init_private(void)
 	_thr_once_init();
 	_thr_spinlock_init();
 	_thr_list_init();
+	__thr_pshared_init();
 	_thr_wake_addr_init();
 	_sleepq_init();
 	_single_thread = NULL;
diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c
index 26e8776..30a8be2 100644
--- a/lib/libthr/thread/thr_mutex.c
+++ b/lib/libthr/thread/thr_mutex.c
@@ -1,8 +1,13 @@
 /*
  * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
  * Copyright (c) 2006 David Xu <davidxu@freebsd.org>.
+ * Copyright (c) 2015 The FreeBSD Foundation
+ *
  * All rights reserved.
  *
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -45,26 +50,6 @@
 
 #include "thr_private.h"
 
-#if defined(_PTHREADS_INVARIANTS)
-#define MUTEX_INIT_LINK(m) 		do {		\
-	(m)->m_qe.tqe_prev = NULL;			\
-	(m)->m_qe.tqe_next = NULL;			\
-} while (0)
-#define MUTEX_ASSERT_IS_OWNED(m)	do {		\
-	if (__predict_false((m)->m_qe.tqe_prev == NULL))\
-		PANIC("mutex is not on list");		\
-} while (0)
-#define MUTEX_ASSERT_NOT_OWNED(m)	do {		\
-	if (__predict_false((m)->m_qe.tqe_prev != NULL ||	\
-	    (m)->m_qe.tqe_next != NULL))	\
-		PANIC("mutex is on list");		\
-} while (0)
-#else
-#define MUTEX_INIT_LINK(m)
-#define MUTEX_ASSERT_IS_OWNED(m)
-#define MUTEX_ASSERT_NOT_OWNED(m)
-#endif
-
 /*
  * For adaptive mutexes, how many times to spin doing trylock2
  * before entering the kernel to block
@@ -122,36 +107,71 @@ __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloop
 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np);
 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np);
 
+static void
+mutex_init_link(struct pthread_mutex *m)
+{
+
+#if defined(_PTHREADS_INVARIANTS)
+	m->m_qe.tqe_prev = NULL;
+	m->m_qe.tqe_next = NULL;
+	m->m_pqe.tqe_prev = NULL;
+	m->m_pqe.tqe_next = NULL;
+#endif
+}
+
+static void
+mutex_assert_is_owned(struct pthread_mutex *m)
+{
+
+#if defined(_PTHREADS_INVARIANTS)
+	if (__predict_false(m->m_qe.tqe_prev == NULL))
+		PANIC("mutex is not on list");
+#endif
+}
+
+static void
+mutex_assert_not_owned(struct pthread_mutex *m)
+{
+
+#if defined(_PTHREADS_INVARIANTS)
+	if (__predict_false(m->m_qe.tqe_prev != NULL ||
+	    m->m_qe.tqe_next != NULL))
+		PANIC("mutex is on list");
+#endif
+}
+
 static int
-mutex_init(pthread_mutex_t *mutex,
-    const struct pthread_mutex_attr *mutex_attr,
-    void *(calloc_cb)(size_t, size_t))
+is_pshared_mutex(struct pthread_mutex *m)
 {
-	const struct pthread_mutex_attr *attr;
-	struct pthread_mutex *pmutex;
 
-	if (mutex_attr == NULL) {
-		attr = &_pthread_mutexattr_default;
-	} else {
-		attr = mutex_attr;
-		if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
-		    attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
-			return (EINVAL);
-		if (attr->m_protocol < PTHREAD_PRIO_NONE ||
-		    attr->m_protocol > PTHREAD_PRIO_PROTECT)
-			return (EINVAL);
-	}
-	if ((pmutex = (pthread_mutex_t)
-		calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
-		return (ENOMEM);
+	return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0);
+}
+
+static int
+mutex_check_attr(const struct pthread_mutex_attr *attr)
+{
+
+	if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
+	    attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
+		return (EINVAL);
+	if (attr->m_protocol < PTHREAD_PRIO_NONE ||
+	    attr->m_protocol > PTHREAD_PRIO_PROTECT)
+		return (EINVAL);
+	return (0);
+}
+
+static void
+mutex_init_body(struct pthread_mutex *pmutex,
+    const struct pthread_mutex_attr *attr)
+{
 
 	pmutex->m_flags = attr->m_type;
-	pmutex->m_owner = NULL;
+	pmutex->m_owner = 0;
 	pmutex->m_count = 0;
 	pmutex->m_spinloops = 0;
 	pmutex->m_yieldloops = 0;
-	MUTEX_INIT_LINK(pmutex);
-	switch(attr->m_protocol) {
+	mutex_init_link(pmutex);
+	switch (attr->m_protocol) {
 	case PTHREAD_PRIO_NONE:
 		pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
 		pmutex->m_lock.m_flags = 0;
@@ -166,13 +186,37 @@ mutex_init(pthread_mutex_t *mutex,
 		pmutex->m_lock.m_ceilings[0] = attr->m_ceiling;
 		break;
 	}
+	if (attr->m_pshared == PTHREAD_PROCESS_SHARED)
+		pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED;
 
 	if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) {
 		pmutex->m_spinloops =
 		    _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS;
 		pmutex->m_yieldloops = _thr_yieldloops;
 	}
+}
 
+static int
+mutex_init(pthread_mutex_t *mutex,
+    const struct pthread_mutex_attr *mutex_attr,
+    void *(calloc_cb)(size_t, size_t))
+{
+	const struct pthread_mutex_attr *attr;
+	struct pthread_mutex *pmutex;
+	int error;
+
+	if (mutex_attr == NULL) {
+		attr = &_pthread_mutexattr_default;
+	} else {
+		attr = mutex_attr;
+		error = mutex_check_attr(attr);
+		if (error != 0)
+			return (error);
+	}
+	if ((pmutex = (pthread_mutex_t)
+		calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
+		return (ENOMEM);
+	mutex_init_body(pmutex, attr);
 	*mutex = pmutex;
 	return (0);
 }
@@ -187,7 +231,8 @@ init_static(struct pthread *thread, pthread_mutex_t *mutex)
 	if (*mutex == THR_MUTEX_INITIALIZER)
 		ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc);
 	else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER)
-		ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, calloc);
+		ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default,
+		    calloc);
 	else
 		ret = 0;
 	THR_LOCK_RELEASE(thread, &_mutex_static_lock);
@@ -200,7 +245,7 @@ set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)
 {
 	struct pthread_mutex *m2;
 
-	m2 = TAILQ_LAST(&curthread->pp_mutexq, mutex_queue);
+	m2 = TAILQ_LAST(&curthread->mq[TMQ_NORM_PP], mutex_queue);
 	if (m2 != NULL)
 		m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0];
 	else
@@ -211,7 +256,25 @@ int
 __pthread_mutex_init(pthread_mutex_t *mutex,
     const pthread_mutexattr_t *mutex_attr)
 {
-	return mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, calloc);
+	struct pthread_mutex *pmtx;
+	int ret;
+
+	if (mutex_attr != NULL) {
+		ret = mutex_check_attr(*mutex_attr);
+		if (ret != 0)
+			return (ret);
+	}
+	if (mutex_attr == NULL ||
+	    (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) {
+		return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL,
+		   calloc));
+	}
+	pmtx = __thr_pshared_offpage(mutex, 1);
+	if (pmtx == NULL)
+		return (EFAULT);
+	*mutex = THR_PSHARED_PTR;
+	mutex_init_body(pmtx, *mutex_attr);
+	return (0);
 }
 
 /* This function is used internally by malloc. */
@@ -222,7 +285,8 @@ _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
 	static const struct pthread_mutex_attr attr = {
 		.m_type = PTHREAD_MUTEX_NORMAL,
 		.m_protocol = PTHREAD_PRIO_NONE,
-		.m_ceiling = 0
+		.m_ceiling = 0,
+		.m_pshared = PTHREAD_PROCESS_PRIVATE,
 	};
 	int ret;
 
@@ -232,31 +296,44 @@ _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
 	return (ret);
 }
 
-void
-_mutex_fork(struct pthread *curthread)
+/*
+ * Fix mutex ownership for child process.
+ *
+ * Process private mutex ownership is transmitted from the forking
+ * thread to the child process.
+ *
+ * Process shared mutex should not be inherited because owner is
+ * forking thread which is in parent process, they are removed from
+ * the owned mutex list.
+ */
+static void
+queue_fork(struct pthread *curthread, struct mutex_queue *q,
+    struct mutex_queue *qp, uint bit)
 {
 	struct pthread_mutex *m;
 
-	/*
-	 * Fix mutex ownership for child process.
-	 * note that process shared mutex should not
-	 * be inherited because owner is forking thread
-	 * which is in parent process, they should be
-	 * removed from the owned mutex list, current,
-	 * process shared mutex is not supported, so I
-	 * am not worried.
-	 */
+	TAILQ_INIT(q);
+	TAILQ_FOREACH(m, qp, m_pqe) {
+		TAILQ_INSERT_TAIL(q, m, m_qe);
+		m->m_lock.m_owner = TID(curthread) | bit;
+		m->m_owner = TID(curthread);
+	}
+}
+
+void
+_mutex_fork(struct pthread *curthread)
+{
 
-	TAILQ_FOREACH(m, &curthread->mutexq, m_qe)
-		m->m_lock.m_owner = TID(curthread);
-	TAILQ_FOREACH(m, &curthread->pp_mutexq, m_qe)
-		m->m_lock.m_owner = TID(curthread) | UMUTEX_CONTESTED;
+	queue_fork(curthread, &curthread->mq[TMQ_NORM],
+	    &curthread->mq[TMQ_NORM_PRIV], 0);
+	queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],
+	    &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);
 }
 
 int
 _pthread_mutex_destroy(pthread_mutex_t *mutex)
 {
-	pthread_mutex_t m;
+	pthread_mutex_t m, m1;
 	int ret;
 
 	m = *mutex;
@@ -265,11 +342,20 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex)
 	} else if (m == THR_MUTEX_DESTROYED) {
 		ret = EINVAL;
 	} else {
-		if (m->m_owner != NULL) {
+		if (m == THR_PSHARED_PTR) {
+			m1 = __thr_pshared_offpage(mutex, 0);
+			if (m1 != NULL) {
+				mutex_assert_not_owned(m1);
+				__thr_pshared_destroy(mutex);
+			}
+			*mutex = THR_MUTEX_DESTROYED;
+			return (0);
+		}
+		if (m->m_owner != 0) {
 			ret = EBUSY;
 		} else {
 			*mutex = THR_MUTEX_DESTROYED;
-			MUTEX_ASSERT_NOT_OWNED(m);
+			mutex_assert_not_owned(m);
 			free(m);
 			ret = 0;
 		}
@@ -278,54 +364,87 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex)
 	return (ret);
 }
 
-#define ENQUEUE_MUTEX(curthread, m)  					\
-	do {								\
-		(m)->m_owner = curthread;				\
-		/* Add to the list of owned mutexes: */			\
-		MUTEX_ASSERT_NOT_OWNED((m));				\
-		if (((m)->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)	\
-			TAILQ_INSERT_TAIL(&curthread->mutexq, (m), m_qe);\
-		else							\
-			TAILQ_INSERT_TAIL(&curthread->pp_mutexq, (m), m_qe);\
-	} while (0)
-
-#define DEQUEUE_MUTEX(curthread, m)					\
-		(m)->m_owner = NULL;					\
-		MUTEX_ASSERT_IS_OWNED(m);				\
-		if (__predict_true(((m)->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)) \
-			TAILQ_REMOVE(&curthread->mutexq, (m), m_qe);		\
-		else {							\
-			TAILQ_REMOVE(&curthread->pp_mutexq, (m), m_qe);	\
-			set_inherited_priority(curthread, m);		\
-		}							\
-		MUTEX_INIT_LINK(m);
-
-#define CHECK_AND_INIT_MUTEX						\
-	if (__predict_false((m = *mutex) <= THR_MUTEX_DESTROYED)) {	\
-		if (m == THR_MUTEX_DESTROYED)				\
-			return (EINVAL);				\
-		int ret;						\
-		ret = init_static(_get_curthread(), mutex);		\
-		if (ret)						\
-			return (ret);					\
-		m = *mutex;						\
-	}
+static int
+mutex_qidx(struct pthread_mutex *m)
+{
+
+	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
+		return (TMQ_NORM);
+	return (TMQ_NORM_PP);
+}
+
+static void
+enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m)
+{
+	int qidx;
+
+	m->m_owner = TID(curthread);
+	/* Add to the list of owned mutexes: */
+	mutex_assert_not_owned(m);
+	qidx = mutex_qidx(m);
+	TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);
+	if (!is_pshared_mutex(m))
+		TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);
+}
+
+static void
+dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)
+{
+	int qidx;
+
+	m->m_owner = 0;
+	mutex_assert_is_owned(m);
+	qidx = mutex_qidx(m);
+	TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);
+	if (!is_pshared_mutex(m))
+		TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);
+	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0)
+		set_inherited_priority(curthread, m);
+	mutex_init_link(m);
+}
 
 static int
-mutex_trylock_common(pthread_mutex_t *mutex)
+check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m)
 {
-	struct pthread *curthread = _get_curthread();
-	struct pthread_mutex *m = *mutex;
+	int ret;
+
+	*m = *mutex;
+	ret = 0;
+	if (*m == THR_PSHARED_PTR) {
+		*m = __thr_pshared_offpage(mutex, 0);
+		if (*m == NULL)
+			ret = EINVAL;
+	} else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) {
+		if (*m == THR_MUTEX_DESTROYED) {
+			ret = EINVAL;
+		} else {
+			ret = init_static(_get_curthread(), mutex);
+			if (ret == 0)
+				*m = *mutex;
+		}
+	}
+	return (ret);
+}
+
+int
+__pthread_mutex_trylock(pthread_mutex_t *mutex)
+{
+	struct pthread *curthread;
+	struct pthread_mutex *m;
 	uint32_t id;
 	int ret;
 
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret != 0)
+		return (ret);
+	curthread = _get_curthread();
 	id = TID(curthread);
 	if (m->m_flags & PMUTEX_FLAG_PRIVATE)
 		THR_CRITICAL_ENTER(curthread);
 	ret = _thr_umutex_trylock(&m->m_lock, id);
 	if (__predict_true(ret == 0)) {
-		ENQUEUE_MUTEX(curthread, m);
-	} else if (m->m_owner == curthread) {
+		enqueue_mutex(curthread, m);
+	} else if (m->m_owner == id) {
 		ret = mutex_self_trylock(m);
 	} /* else {} */
 	if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE))
@@ -333,16 +452,6 @@ mutex_trylock_common(pthread_mutex_t *mutex)
 	return (ret);
 }
 
-int
-__pthread_mutex_trylock(pthread_mutex_t *mutex)
-{
-	struct pthread_mutex *m;
-
-	CHECK_AND_INIT_MUTEX
-
-	return (mutex_trylock_common(mutex));
-}
-
 static int
 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
 	const struct timespec *abstime)
@@ -351,10 +460,10 @@ mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
 	int	count;
 	int	ret;
 
-	if (m->m_owner == curthread)
-		return mutex_self_lock(m, abstime);
-
 	id = TID(curthread);
+	if (m->m_owner == id)
+		return (mutex_self_lock(m, abstime));
+
 	/*
 	 * For adaptive mutexes, spin for a bit in the expectation
 	 * that if the application requests this mutex type then
@@ -406,7 +515,7 @@ sleep_in_kernel:
 	}
 done:
 	if (ret == 0)
-		ENQUEUE_MUTEX(curthread, m);
+		enqueue_mutex(curthread, m);
 
 	return (ret);
 }
@@ -421,7 +530,7 @@ mutex_lock_common(struct pthread_mutex *m,
 	if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE)
 		THR_CRITICAL_ENTER(curthread);
 	if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) {
-		ENQUEUE_MUTEX(curthread, m);
+		enqueue_mutex(curthread, m);
 		ret = 0;
 	} else {
 		ret = mutex_lock_sleep(curthread, m, abstime);
@@ -434,25 +543,28 @@ mutex_lock_common(struct pthread_mutex *m,
 int
 __pthread_mutex_lock(pthread_mutex_t *mutex)
 {
-	struct pthread_mutex	*m;
+	struct pthread_mutex *m;
+	int ret;
 
 	_thr_check_init();
-
-	CHECK_AND_INIT_MUTEX
-
-	return (mutex_lock_common(m, NULL, 0));
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		ret = mutex_lock_common(m, NULL, 0);
+	return (ret);
 }
 
 int
-__pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *abstime)
+__pthread_mutex_timedlock(pthread_mutex_t *mutex,
+    const struct timespec *abstime)
 {
-	struct pthread_mutex	*m;
+	struct pthread_mutex *m;
+	int ret;
 
 	_thr_check_init();
-
-	CHECK_AND_INIT_MUTEX
-
-	return (mutex_lock_common(m, abstime, 0));
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		ret = mutex_lock_common(m, abstime, 0);
+	return (ret);
 }
 
 int
@@ -460,7 +572,13 @@ _pthread_mutex_unlock(pthread_mutex_t *mutex)
 {
 	struct pthread_mutex *mp;
 
-	mp = *mutex;
+	if (*mutex == THR_PSHARED_PTR) {
+		mp = __thr_pshared_offpage(mutex, 0);
+		if (mp == NULL)
+			return (EINVAL);
+	} else {
+		mp = *mutex;
+	}
 	return (mutex_unlock_common(mp, 0, NULL));
 }
 
@@ -493,7 +611,7 @@ _mutex_cv_attach(struct pthread_mutex *m, int count)
 {
 	struct pthread *curthread = _get_curthread();
 
-	ENQUEUE_MUTEX(curthread, m);
+	enqueue_mutex(curthread, m);
 	m->m_count = count;
 	return (0);
 }
@@ -513,7 +631,7 @@ _mutex_cv_detach(struct pthread_mutex *mp, int *recurse)
 	 */
 	*recurse = mp->m_count;
 	mp->m_count = 0;
-	DEQUEUE_MUTEX(curthread, mp);
+	dequeue_mutex(curthread, mp);
 
 	/* Will this happen in real-world ? */
         if ((mp->m_flags & PMUTEX_FLAG_DEFERED) != 0) {
@@ -641,14 +759,15 @@ mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer)
 		return (EPERM);
 	}
 
+	id = TID(curthread);
+
 	/*
 	 * Check if the running thread is not the owner of the mutex.
 	 */
-	if (__predict_false(m->m_owner != curthread))
+	if (__predict_false(m->m_owner != id))
 		return (EPERM);
 
 	error = 0;
-	id = TID(curthread);
 	if (__predict_false(
 		PMUTEX_TYPE(m->m_flags) == PTHREAD_MUTEX_RECURSIVE &&
 		m->m_count > 0)) {
@@ -660,7 +779,7 @@ mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer)
         	} else
 			defered = 0;
 
-		DEQUEUE_MUTEX(curthread, m);
+		dequeue_mutex(curthread, m);
 		error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer);
 
 		if (mtx_defer == NULL && defered)  {
@@ -676,54 +795,85 @@ mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer)
 
 int
 _pthread_mutex_getprioceiling(pthread_mutex_t *mutex,
-			      int *prioceiling)
+    int *prioceiling)
 {
 	struct pthread_mutex *m;
-	int ret;
 
-	m = *mutex;
-	if ((m <= THR_MUTEX_DESTROYED) ||
-	    (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
-		ret = EINVAL;
-	else {
-		*prioceiling = m->m_lock.m_ceilings[0];
-		ret = 0;
+	if (*mutex == THR_PSHARED_PTR) {
+		m = __thr_pshared_offpage(mutex, 0);
+		if (m == NULL)
+			return (EINVAL);
+	} else {
+		m = *mutex;
+		if (m <= THR_MUTEX_DESTROYED)
+			return (EINVAL);
 	}
-
-	return (ret);
+	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
+		return (EINVAL);
+	*prioceiling = m->m_lock.m_ceilings[0];
+	return (0);
 }
 
 int
 _pthread_mutex_setprioceiling(pthread_mutex_t *mutex,
-			      int ceiling, int *old_ceiling)
+    int ceiling, int *old_ceiling)
 {
-	struct pthread *curthread = _get_curthread();
+	struct pthread *curthread;
 	struct pthread_mutex *m, *m1, *m2;
+	struct mutex_queue *q, *qp;
 	int ret;
 
-	m = *mutex;
-	if ((m <= THR_MUTEX_DESTROYED) ||
-	    (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
+	if (*mutex == THR_PSHARED_PTR) {
+		m = __thr_pshared_offpage(mutex, 0);
+		if (m == NULL)
+			return (EINVAL);
+	} else {
+		m = *mutex;
+		if (m <= THR_MUTEX_DESTROYED)
+			return (EINVAL);
+	}
+	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
 		return (EINVAL);
 
 	ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling);
 	if (ret != 0)
 		return (ret);
 
-	if (m->m_owner == curthread) {
-		MUTEX_ASSERT_IS_OWNED(m);
+	curthread = _get_curthread();
+	if (m->m_owner == TID(curthread)) {
+		mutex_assert_is_owned(m);
 		m1 = TAILQ_PREV(m, mutex_queue, m_qe);
 		m2 = TAILQ_NEXT(m, m_qe);
 		if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||
 		    (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) {
-			TAILQ_REMOVE(&curthread->pp_mutexq, m, m_qe);
-			TAILQ_FOREACH(m2, &curthread->pp_mutexq, m_qe) {
+			q = &curthread->mq[TMQ_NORM_PP];
+			qp = &curthread->mq[TMQ_NORM_PP_PRIV];
+			TAILQ_REMOVE(q, m, m_qe);
+			if (!is_pshared_mutex(m))
+				TAILQ_REMOVE(qp, m, m_pqe);
+			TAILQ_FOREACH(m2, q, m_qe) {
 				if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) {
 					TAILQ_INSERT_BEFORE(m2, m, m_qe);
+					if (!is_pshared_mutex(m)) {
+						while (m2 != NULL &&
+						    is_pshared_mutex(m2)) {
+							m2 = TAILQ_PREV(m2,
+							    mutex_queue, m_qe);
+						}
+						if (m2 == NULL) {
+							TAILQ_INSERT_HEAD(qp,
+							    m, m_pqe);
+						} else {
+							TAILQ_INSERT_BEFORE(m2,
+							    m, m_pqe);
+						}
+					}
 					return (0);
 				}
 			}
-			TAILQ_INSERT_TAIL(&curthread->pp_mutexq, m, m_qe);
+			TAILQ_INSERT_TAIL(q, m, m_qe);
+			if (!is_pshared_mutex(m))
+				TAILQ_INSERT_TAIL(qp, m, m_pqe);
 		}
 	}
 	return (0);
@@ -732,44 +882,48 @@ _pthread_mutex_setprioceiling(pthread_mutex_t *mutex,
 int
 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count)
 {
-	struct pthread_mutex	*m;
-
-	CHECK_AND_INIT_MUTEX
+	struct pthread_mutex *m;
+	int ret;
 
-	*count = m->m_spinloops;
-	return (0);
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		*count = m->m_spinloops;
+	return (ret);
 }
 
 int
 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count)
 {
-	struct pthread_mutex	*m;
-
-	CHECK_AND_INIT_MUTEX
+	struct pthread_mutex *m;
+	int ret;
 
-	m->m_spinloops = count;
-	return (0);
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		m->m_spinloops = count;
+	return (ret);
 }
 
 int
 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count)
 {
-	struct pthread_mutex	*m;
-
-	CHECK_AND_INIT_MUTEX
+	struct pthread_mutex *m;
+	int ret;
 
-	*count = m->m_yieldloops;
-	return (0);
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		*count = m->m_yieldloops;
+	return (ret);
 }
 
 int
 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count)
 {
-	struct pthread_mutex	*m;
-
-	CHECK_AND_INIT_MUTEX
+	struct pthread_mutex *m;
+	int ret;
 
-	m->m_yieldloops = count;
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		m->m_yieldloops = count;
 	return (0);
 }
 
@@ -778,10 +932,16 @@ _pthread_mutex_isowned_np(pthread_mutex_t *mutex)
 {
 	struct pthread_mutex	*m;
 
-	m = *mutex;
-	if (m <= THR_MUTEX_DESTROYED)
-		return (0);
-	return (m->m_owner == _get_curthread());
+	if (*mutex == THR_PSHARED_PTR) {
+		m = __thr_pshared_offpage(mutex, 0);
+		if (m == NULL)
+			return (0);
+	} else {
+		m = *mutex;
+		if (m <= THR_MUTEX_DESTROYED)
+			return (0);
+	}
+	return (m->m_owner == TID(_get_curthread()));
 }
 
 int
@@ -792,7 +952,7 @@ _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp)
 			return (EINVAL);
 		return (EPERM);
 	}
-      	if (mp->m_owner != curthread)
+	if (mp->m_owner != TID(curthread))
 		return (EPERM);
 	return (0);                  
 }
diff --git a/lib/libthr/thread/thr_mutexattr.c b/lib/libthr/thread/thr_mutexattr.c
index 7c48ed22..fa349cf 100644
--- a/lib/libthr/thread/thr_mutexattr.c
+++ b/lib/libthr/thread/thr_mutexattr.c
@@ -176,8 +176,7 @@ _pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr,
 
 	if (attr == NULL || *attr == NULL)
 		return (EINVAL);
-
-	*pshared = PTHREAD_PROCESS_PRIVATE;
+	*pshared = (*attr)->m_pshared;
 	return (0);
 }
 
@@ -185,13 +184,11 @@ int
 _pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
 {
 
-	if (attr == NULL || *attr == NULL)
+	if (attr == NULL || *attr == NULL ||
+	    (pshared != PTHREAD_PROCESS_PRIVATE &&
+	    pshared != PTHREAD_PROCESS_SHARED))
 		return (EINVAL);
-
-	/* Only PTHREAD_PROCESS_PRIVATE is supported. */
-	if (pshared != PTHREAD_PROCESS_PRIVATE)
-		return (EINVAL);
-
+	(*attr)->m_pshared = pshared;
 	return (0);
 }
 
diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h
index 6020e07..31f8e6c 100644
--- a/lib/libthr/thread/thr_private.h
+++ b/lib/libthr/thread/thr_private.h
@@ -126,6 +126,10 @@ TAILQ_HEAD(mutex_queue, pthread_mutex);
 		}						\
 	} while (0)
 
+/* Magic cookie set for shared pthread locks and cv's pointers */
+#define	THR_PSHARED_PTR						\
+    ((void *)(uintptr_t)((1ULL << (NBBY * sizeof(long) - 1)) | 1))
+
 /* XXX These values should be same as those defined in pthread.h */
 #define	THR_MUTEX_INITIALIZER		((struct pthread_mutex *)NULL)
 #define	THR_ADAPTIVE_MUTEX_INITIALIZER	((struct pthread_mutex *)1)
@@ -148,20 +152,24 @@ struct pthread_mutex {
 	 */
 	struct umutex			m_lock;
 	int				m_flags;
-	struct pthread			*m_owner;
+	uint32_t			m_owner;
 	int				m_count;
 	int				m_spinloops;
 	int				m_yieldloops;
 	/*
-	 * Link for all mutexes a thread currently owns.
+	 * Link for all mutexes a thread currently owns, of the same
+	 * prio type.
 	 */
 	TAILQ_ENTRY(pthread_mutex)	m_qe;
+	/* Link for all private mutexes a thread currently owns. */
+	TAILQ_ENTRY(pthread_mutex)	m_pqe;
 };
 
 struct pthread_mutex_attr {
 	enum pthread_mutextype	m_type;
 	int			m_protocol;
 	int			m_ceiling;
+	int			m_pshared;
 };
 
 #define PTHREAD_MUTEXATTR_STATIC_INITIALIZER \
@@ -313,7 +321,7 @@ struct pthread_rwlockattr {
 
 struct pthread_rwlock {
 	struct urwlock 	lock;
-	struct pthread	*owner;
+	uint32_t	owner;
 };
 
 /*
@@ -467,11 +475,16 @@ struct pthread {
 #define	TLFLAGS_IN_TDLIST	0x0002	/* thread in all thread list */
 #define	TLFLAGS_IN_GCLIST	0x0004	/* thread in gc list */
 
-	/* Queue of currently owned NORMAL or PRIO_INHERIT type mutexes. */
-	struct mutex_queue	mutexq;
-
-	/* Queue of all owned PRIO_PROTECT mutexes. */
-	struct mutex_queue	pp_mutexq;
+	/*
+	 * Queues of the owned mutexes.  Private queue must have index
+	 * + 1 of the corresponding full queue.
+	 */
+#define	TMQ_NORM		0	/* NORMAL or PRIO_INHERIT normal */
+#define	TMQ_NORM_PRIV		1	/* NORMAL or PRIO_INHERIT normal priv */
+#define	TMQ_NORM_PP		2	/* PRIO_PROTECT normal mutexes */
+#define	TMQ_NORM_PP_PRIV	3	/* PRIO_PROTECT normal priv */
+#define	TMQ_NITEMS		4
+	struct mutex_queue	mq[TMQ_NITEMS];
 
 	void				*ret;
 	struct pthread_specific_elem	*specific;
@@ -936,6 +949,10 @@ void __thr_spinlock(struct _spinlock *lck);
 struct tcb *_tcb_ctor(struct pthread *, int);
 void	_tcb_dtor(struct tcb *);
 
+void __thr_pshared_init(void) __hidden;
+void *__thr_pshared_offpage(void *key, int doalloc) __hidden;
+void __thr_pshared_destroy(void *key) __hidden;
+
 __END_DECLS
 
 #endif  /* !_THR_PRIVATE_H */
diff --git a/lib/libthr/thread/thr_pshared.c b/lib/libthr/thread/thr_pshared.c
new file mode 100644
index 0000000..d40346d
--- /dev/null
+++ b/lib/libthr/thread/thr_pshared.c
@@ -0,0 +1,223 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ *
+ * This software was developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/queue.h>
+#include "namespace.h"
+#include <stdlib.h>
+#include "un-namespace.h"
+
+#include "thr_private.h"
+
+struct psh {
+	LIST_ENTRY(psh) link;
+	void *key;
+	void *val;
+};
+
+LIST_HEAD(pshared_hash_head, psh);
+#define	HASH_SIZE	128
+static struct pshared_hash_head pshared_hash[HASH_SIZE];
+#define	PSHARED_KEY_HASH(key)	(((unsigned long)(key) >> 8) % HASH_SIZE)
+/* XXXKIB: lock could be split to per-hash chain, if appears contested */
+static struct urwlock pshared_lock = DEFAULT_URWLOCK;
+
+void
+__thr_pshared_init(void)
+{
+	int i;
+
+	_thr_urwlock_init(&pshared_lock);
+	for (i = 0; i < HASH_SIZE; i++)
+		LIST_INIT(&pshared_hash[i]);
+}
+
+static void
+pshared_rlock(struct pthread *curthread)
+{
+
+	curthread->locklevel++;
+	_thr_rwl_rdlock(&pshared_lock);
+}
+
+static void
+pshared_wlock(struct pthread *curthread)
+{
+
+	curthread->locklevel++;
+	_thr_rwl_wrlock(&pshared_lock);
+}
+
+static void
+pshared_unlock(struct pthread *curthread)
+{
+
+	_thr_rwl_unlock(&pshared_lock);
+	curthread->locklevel--;
+	_thr_ast(curthread);
+}
+
+static void
+pshared_gc(struct pthread *curthread)
+{
+	struct pshared_hash_head *hd;
+	struct psh *h, *h1;
+	int error, i;
+
+	pshared_wlock(curthread);
+	for (i = 0; i < HASH_SIZE; i++) {
+		hd = &pshared_hash[i];
+		LIST_FOREACH_SAFE(h, hd, link, h1) {
+			error = _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_ALIVE,
+			    h->val, NULL);
+			if (error == 0)
+				continue;
+			LIST_REMOVE(h, link);
+			munmap(h->val, PAGE_SIZE);
+			free(h);
+		}
+	}
+	pshared_unlock(curthread);
+}
+
+static void *
+pshared_lookup(void *key)
+{
+	struct pshared_hash_head *hd;
+	struct psh *h;
+
+	hd = &pshared_hash[PSHARED_KEY_HASH(key)];
+	LIST_FOREACH(h, hd, link) {
+		if (h->key == key)
+			return (h->val);
+	}
+	return (NULL);
+}
+
+static int
+pshared_insert(void *key, void **val)
+{
+	struct pshared_hash_head *hd;
+	struct psh *h;
+
+	hd = &pshared_hash[PSHARED_KEY_HASH(key)];
+	LIST_FOREACH(h, hd, link) {
+		if (h->key == key) {
+			if (h->val != *val) {
+				munmap(*val, PAGE_SIZE);
+				*val = h->val;
+			}
+			return (1);
+		}
+	}
+
+	h = malloc(sizeof(*h));
+	if (h == NULL)
+		return (0);
+	h->key = key;
+	h->val = *val;
+	LIST_INSERT_HEAD(hd, h, link);
+	return (1);
+}
+
+static void *
+pshared_remove(void *key)
+{
+	struct pshared_hash_head *hd;
+	struct psh *h;
+	void *val;
+
+	hd = &pshared_hash[PSHARED_KEY_HASH(key)];
+	LIST_FOREACH(h, hd, link) {
+		if (h->key == key) {
+			LIST_REMOVE(h, link);
+			val = h->val;
+			free(h);
+			return (val);
+		}
+	}
+	return (NULL);
+}
+
+static void
+pshared_clean(void *key, void *val)
+{
+
+	if (val != NULL)
+		munmap(val, PAGE_SIZE);
+	_umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_DESTROY, key, NULL);
+}
+
+void *
+__thr_pshared_offpage(void *key, int doalloc)
+{
+	struct pthread *curthread;
+	void *res;
+	int fd, ins_done;
+
+	curthread = _get_curthread();
+	pshared_rlock(curthread);
+	res = pshared_lookup(key);
+	pshared_unlock(curthread);
+	if (res != NULL)
+		return (res);
+	fd = _umtx_op(NULL, UMTX_OP_SHM, doalloc ? UMTX_SHM_CREAT :
+	    UMTX_SHM_LOOKUP, key, NULL);
+	if (fd == -1)
+		return (NULL);
+	res = mmap(NULL, PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+	close(fd);
+	if (res == MAP_FAILED)
+		return (NULL);
+	pshared_wlock(curthread);
+	ins_done = pshared_insert(key, &res);
+	pshared_unlock(curthread);
+	if (!ins_done) {
+		pshared_clean(key, res);
+		res = NULL;
+	}
+	return (res);
+}
+
+void
+__thr_pshared_destroy(void *key)
+{
+	struct pthread *curthread;
+	void *val;
+
+	curthread = _get_curthread();
+	pshared_wlock(curthread);
+	val = pshared_remove(key);
+	pshared_unlock(curthread);
+	pshared_clean(key, val);
+	pshared_gc(curthread);
+}
diff --git a/lib/libthr/thread/thr_rwlock.c b/lib/libthr/thread/thr_rwlock.c
index 397663e..aadd0e9 100644
--- a/lib/libthr/thread/thr_rwlock.c
+++ b/lib/libthr/thread/thr_rwlock.c
@@ -46,7 +46,12 @@ __weak_reference(_pthread_rwlock_wrlock, pthread_rwlock_wrlock);
 __weak_reference(_pthread_rwlock_timedwrlock, pthread_rwlock_timedwrlock);
 
 #define CHECK_AND_INIT_RWLOCK							\
-	if (__predict_false((prwlock = (*rwlock)) <= THR_RWLOCK_DESTROYED)) {	\
+	if (*rwlock == THR_PSHARED_PTR) {					\
+		prwlock = __thr_pshared_offpage(rwlock, 0);			\
+		if (prwlock == NULL)						\
+			return (EINVAL);					\
+	} else if (__predict_false((prwlock = (*rwlock)) <=			\
+	    THR_RWLOCK_DESTROYED)) {						\
 		if (prwlock == THR_RWLOCK_INITIALIZER) {			\
 			int ret;						\
 			ret = init_static(_get_curthread(), rwlock);		\
@@ -63,14 +68,23 @@ __weak_reference(_pthread_rwlock_timedwrlock, pthread_rwlock_timedwrlock);
  */
 
 static int
-rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr __unused)
+rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
 {
 	pthread_rwlock_t prwlock;
 
-	prwlock = (pthread_rwlock_t)calloc(1, sizeof(struct pthread_rwlock));
-	if (prwlock == NULL)
-		return (ENOMEM);
-	*rwlock = prwlock;
+	if (attr == NULL || *attr == NULL ||
+	    (*attr)->pshared == PTHREAD_PROCESS_PRIVATE) {
+		prwlock = calloc(1, sizeof(struct pthread_rwlock));
+		if (prwlock == NULL)
+			return (ENOMEM);
+		*rwlock = prwlock;
+	} else {
+		prwlock = __thr_pshared_offpage(rwlock, 1);
+		if (prwlock == NULL)
+			return (EFAULT);
+		prwlock->lock.rw_flags |= USYNC_PROCESS_SHARED;
+		*rwlock = THR_PSHARED_PTR;
+	}
 	return (0);
 }
 
@@ -85,9 +99,12 @@ _pthread_rwlock_destroy (pthread_rwlock_t *rwlock)
 		ret = 0;
 	else if (prwlock == THR_RWLOCK_DESTROYED)
 		ret = EINVAL;
-	else {
+	else if (prwlock == THR_PSHARED_PTR) {
+		*rwlock = THR_RWLOCK_DESTROYED;
+		__thr_pshared_destroy(rwlock);
+		ret = 0;
+	} else {
 		*rwlock = THR_RWLOCK_DESTROYED;
-
 		free(prwlock);
 		ret = 0;
 	}
@@ -112,8 +129,9 @@ init_static(struct pthread *thread, pthread_rwlock_t *rwlock)
 }
 
 int
-_pthread_rwlock_init (pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
+_pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
 {
+
 	*rwlock = NULL;
 	return (rwlock_init(rwlock, attr));
 }
@@ -235,7 +253,7 @@ _pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
 
 	ret = _thr_rwlock_trywrlock(&prwlock->lock);
 	if (ret == 0)
-		prwlock->owner = curthread;
+		prwlock->owner = TID(curthread);
 	return (ret);
 }
 
@@ -254,19 +272,19 @@ rwlock_wrlock_common (pthread_rwlock_t *rwlock, const struct timespec *abstime)
 	 */
 	ret = _thr_rwlock_trywrlock(&prwlock->lock);
 	if (ret == 0) {
-		prwlock->owner = curthread;
+		prwlock->owner = TID(curthread);
 		return (ret);
 	}
 
 	if (__predict_false(abstime && 
-		(abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0)))
+	    (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0)))
 		return (EINVAL);
 
 	for (;;) {
 		/* goto kernel and lock it */
 		ret = __thr_rwlock_wrlock(&prwlock->lock, abstime);
 		if (ret == 0) {
-			prwlock->owner = curthread;
+			prwlock->owner = TID(curthread);
 			break;
 		}
 
@@ -276,7 +294,7 @@ rwlock_wrlock_common (pthread_rwlock_t *rwlock, const struct timespec *abstime)
 		/* if interrupted, try to lock it in userland again. */
 		if (_thr_rwlock_trywrlock(&prwlock->lock) == 0) {
 			ret = 0;
-			prwlock->owner = curthread;
+			prwlock->owner = TID(curthread);
 			break;
 		}
 	}
@@ -297,23 +315,29 @@ _pthread_rwlock_timedwrlock (pthread_rwlock_t *rwlock,
 }
 
 int
-_pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
+_pthread_rwlock_unlock(pthread_rwlock_t *rwlock)
 {
 	struct pthread *curthread = _get_curthread();
 	pthread_rwlock_t prwlock;
 	int ret;
 	int32_t state;
 
-	prwlock = *rwlock;
+	if (*rwlock == THR_PSHARED_PTR) {
+		prwlock = __thr_pshared_offpage(rwlock, 0);
+		if (prwlock == NULL)
+			return (EINVAL);
+	} else {
+		prwlock = *rwlock;
+	}
 
 	if (__predict_false(prwlock <= THR_RWLOCK_DESTROYED))
 		return (EINVAL);
 
 	state = prwlock->lock.rw_state;
 	if (state & URWLOCK_WRITE_OWNER) {
-		if (__predict_false(prwlock->owner != curthread))
+		if (__predict_false(prwlock->owner != TID(curthread)))
 			return (EPERM);
-		prwlock->owner = NULL;
+		prwlock->owner = 0;
 	}
 
 	ret = _thr_rwlock_unlock(&prwlock->lock);
diff --git a/lib/libthr/thread/thr_rwlockattr.c b/lib/libthr/thread/thr_rwlockattr.c
index 73ccdc9..b9e442e 100644
--- a/lib/libthr/thread/thr_rwlockattr.c
+++ b/lib/libthr/thread/thr_rwlockattr.c
@@ -45,25 +45,21 @@ _pthread_rwlockattr_destroy(pthread_rwlockattr_t *rwlockattr)
 	pthread_rwlockattr_t prwlockattr;
 
 	if (rwlockattr == NULL)
-		return(EINVAL);
-
+		return (EINVAL);
 	prwlockattr = *rwlockattr;
-
 	if (prwlockattr == NULL)
-		return(EINVAL);
-
+		return (EINVAL);
 	free(prwlockattr);
-
-	return(0);
+	return (0);
 }
 
 int
 _pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *rwlockattr,
-	int *pshared)
+    int *pshared)
 {
-	*pshared = (*rwlockattr)->pshared;
 
-	return(0);
+	*pshared = (*rwlockattr)->pshared;
+	return (0);
 }
 
 int
@@ -72,28 +68,24 @@ _pthread_rwlockattr_init(pthread_rwlockattr_t *rwlockattr)
 	pthread_rwlockattr_t prwlockattr;
 
 	if (rwlockattr == NULL)
-		return(EINVAL);
-
-	prwlockattr = (pthread_rwlockattr_t)
-		malloc(sizeof(struct pthread_rwlockattr));
+		return (EINVAL);
 
+	prwlockattr = malloc(sizeof(struct pthread_rwlockattr));
 	if (prwlockattr == NULL)
-		return(ENOMEM);
+		return (ENOMEM);
 
-	prwlockattr->pshared 	= PTHREAD_PROCESS_PRIVATE;
-	*rwlockattr		= prwlockattr;
-
-	return(0);
+	prwlockattr->pshared = PTHREAD_PROCESS_PRIVATE;
+	*rwlockattr = prwlockattr;
+	return (0);
 }
 
 int
 _pthread_rwlockattr_setpshared(pthread_rwlockattr_t *rwlockattr, int pshared)
 {
-	/* Only PTHREAD_PROCESS_PRIVATE is supported. */
-	if (pshared != PTHREAD_PROCESS_PRIVATE)
-		return(EINVAL);
 
+	if (pshared != PTHREAD_PROCESS_PRIVATE &&
+	    pshared != PTHREAD_PROCESS_SHARED)
+		return (EINVAL);
 	(*rwlockattr)->pshared = pshared;
-
-	return(0);
+	return (0);
 }