13 files changed, 788 insertions, 307 deletions

diff --git a/lib/libthr/thread/Makefile.inc b/lib/libthr/thread/Makefile.inc
index 1a7c63d14af9..795ed3989abe 100644
--- a/lib/libthr/thread/Makefile.inc
+++ b/lib/libthr/thread/Makefile.inc
@@ -36,6 +36,7 @@ SRCS+= \
 	thr_mutexattr.c \
 	thr_once.c \
 	thr_printf.c \
+	thr_pshared.c \
 	thr_pspinlock.c \
 	thr_resume_np.c \
 	thr_rtld.c \
diff --git a/lib/libthr/thread/thr_barrier.c b/lib/libthr/thread/thr_barrier.c
index 10b634608241..45ca41a18f9d 100644
--- a/lib/libthr/thread/thr_barrier.c
+++ b/lib/libthr/thread/thr_barrier.c
@@ -41,14 +41,25 @@ __weak_reference(_pthread_barrier_destroy,	pthread_barrier_destroy);
 int
 _pthread_barrier_destroy(pthread_barrier_t *barrier)
 {
-	pthread_barrier_t	bar;
-	struct pthread		*curthread;
+	pthread_barrier_t bar;
+	struct pthread *curthread;
+	int pshared;
 
 	if (barrier == NULL || *barrier == NULL)
 		return (EINVAL);
 
+	if (*barrier == THR_PSHARED_PTR) {
+		bar = __thr_pshared_offpage(barrier, 0);
+		if (bar == NULL) {
+			*barrier = NULL;
+			return (0);
+		}
+		pshared = 1;
+	} else {
+		bar = *barrier;
+		pshared = 0;
+	}
 	curthread = _get_curthread();
-	bar = *barrier;
 	THR_UMUTEX_LOCK(curthread, &bar->b_lock);
 	if (bar->b_destroying) {
 		THR_UMUTEX_UNLOCK(curthread, &bar->b_lock);
@@ -71,37 +82,52 @@ _pthread_barrier_destroy(pthread_barrier_t *barrier)
 	THR_UMUTEX_UNLOCK(curthread, &bar->b_lock);
 
 	*barrier = NULL;
-	free(bar);
+	if (pshared)
+		__thr_pshared_destroy(barrier);
+	else
+		free(bar);
 	return (0);
 }
 
 int
 _pthread_barrier_init(pthread_barrier_t *barrier,
-		      const pthread_barrierattr_t *attr, unsigned count)
+    const pthread_barrierattr_t *attr, unsigned count)
 {
-	pthread_barrier_t	bar;
-
-	(void)attr;
+	pthread_barrier_t bar;
+	int pshared;
 
 	if (barrier == NULL || count <= 0)
 		return (EINVAL);
 
-	bar = calloc(1, sizeof(struct pthread_barrier));
-	if (bar == NULL)
-		return (ENOMEM);
+	if (attr == NULL || *attr == NULL ||
+	    (*attr)->pshared == PTHREAD_PROCESS_PRIVATE) {
+		bar = calloc(1, sizeof(struct pthread_barrier));
+		if (bar == NULL)
+			return (ENOMEM);
+		*barrier = bar;
+		pshared = 0;
+	} else {
+		bar = __thr_pshared_offpage(barrier, 1);
+		if (bar == NULL)
+			return (EFAULT);
+		*barrier = THR_PSHARED_PTR;
+		pshared = 1;
+	}
 
 	_thr_umutex_init(&bar->b_lock);
 	_thr_ucond_init(&bar->b_cv);
-	bar->b_count	= count;
-	*barrier	= bar;
-
+	if (pshared) {
+		bar->b_lock.m_flags |= USYNC_PROCESS_SHARED;
+		bar->b_cv.c_flags |= USYNC_PROCESS_SHARED;
+	}
+	bar->b_count = count;
 	return (0);
 }
 
 int
 _pthread_barrier_wait(pthread_barrier_t *barrier)
 {
-	struct pthread *curthread = _get_curthread();
+	struct pthread *curthread;
 	pthread_barrier_t bar;
 	int64_t cycle;
 	int ret;
@@ -109,7 +135,14 @@ _pthread_barrier_wait(pthread_barrier_t *barrier)
 	if (barrier == NULL || *barrier == NULL)
 		return (EINVAL);
 
-	bar = *barrier;
+	if (*barrier == THR_PSHARED_PTR) {
+		bar = __thr_pshared_offpage(barrier, 0);
+		if (bar == NULL)
+			return (EINVAL);
+	} else {
+		bar = *barrier;
+	}
+	curthread = _get_curthread();
 	THR_UMUTEX_LOCK(curthread, &bar->b_lock);
 	if (++bar->b_waiters == bar->b_count) {
 		/* Current thread is lastest thread */
diff --git a/lib/libthr/thread/thr_barrierattr.c b/lib/libthr/thread/thr_barrierattr.c
index 32b9723445c0..e98a66fcb5fa 100644
--- a/lib/libthr/thread/thr_barrierattr.c
+++ b/lib/libthr/thread/thr_barrierattr.c
@@ -56,7 +56,7 @@ _pthread_barrierattr_destroy(pthread_barrierattr_t *attr)
 
 int
 _pthread_barrierattr_getpshared(const pthread_barrierattr_t *attr,
-	int *pshared)
+    int *pshared)
 {
 
 	if (attr == NULL || *attr == NULL)
@@ -84,11 +84,9 @@ int
 _pthread_barrierattr_setpshared(pthread_barrierattr_t *attr, int pshared)
 {
 
-	if (attr == NULL || *attr == NULL)
-		return (EINVAL);
-
-	/* Only PTHREAD_PROCESS_PRIVATE is supported. */
-	if (pshared != PTHREAD_PROCESS_PRIVATE)
+	if (attr == NULL || *attr == NULL ||
+	    (pshared != PTHREAD_PROCESS_PRIVATE &&
+	    pshared != PTHREAD_PROCESS_SHARED))
 		return (EINVAL);
 
 	(*attr)->pshared = pshared;
diff --git a/lib/libthr/thread/thr_cond.c b/lib/libthr/thread/thr_cond.c
index 71b4293e0b2c..934f9a04a63a 100644
--- a/lib/libthr/thread/thr_cond.c
+++ b/lib/libthr/thread/thr_cond.c
@@ -1,7 +1,11 @@
 /*
  * Copyright (c) 2005 David Xu <davidxu@freebsd.org>
+ * Copyright (c) 2015 The FreeBSD Foundation
  * All rights reserved.
  *
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -63,29 +67,45 @@ __weak_reference(_pthread_cond_broadcast, pthread_cond_broadcast);
 
 #define CV_PSHARED(cvp)	(((cvp)->__flags & USYNC_PROCESS_SHARED) != 0)
 
+static void
+cond_init_body(struct pthread_cond *cvp, const struct pthread_cond_attr *cattr)
+{
+
+	if (cattr == NULL) {
+		cvp->__clock_id = CLOCK_REALTIME;
+	} else {
+		if (cattr->c_pshared)
+			cvp->__flags |= USYNC_PROCESS_SHARED;
+		cvp->__clock_id = cattr->c_clockid;
+	}
+}
+
 static int
 cond_init(pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
 {
-	struct pthread_cond	*cvp;
-	int	error = 0;
-
-	if ((cvp = (pthread_cond_t)
-	    calloc(1, sizeof(struct pthread_cond))) == NULL) {
-		error = ENOMEM;
+	struct pthread_cond *cvp;
+	const struct pthread_cond_attr *cattr;
+	int pshared;
+
+	cattr = cond_attr != NULL ? *cond_attr : NULL;
+	if (cattr == NULL || cattr->c_pshared == PTHREAD_PROCESS_PRIVATE) {
+		pshared = 0;
+		cvp = calloc(1, sizeof(struct pthread_cond));
+		if (cvp == NULL)
+			return (ENOMEM);
 	} else {
-		/*
-		 * Initialise the condition variable structure:
-		 */
-		if (cond_attr == NULL || *cond_attr == NULL) {
-			cvp->__clock_id = CLOCK_REALTIME;
-		} else {
-			if ((*cond_attr)->c_pshared)
-				cvp->__flags |= USYNC_PROCESS_SHARED;
-			cvp->__clock_id = (*cond_attr)->c_clockid;
-		}
-		*cond = cvp;
+		pshared = 1;
+		cvp = __thr_pshared_offpage(cond, 1);
+		if (cvp == NULL)
+			return (EFAULT);
 	}
-	return (error);
+
+	/*
+	 * Initialise the condition variable structure:
+	 */
+	cond_init_body(cvp, cattr);
+	*cond = pshared ? THR_PSHARED_PTR : cvp;
+	return (0);
 }
 
 static int
@@ -106,7 +126,11 @@ init_static(struct pthread *thread, pthread_cond_t *cond)
 }
 
 #define CHECK_AND_INIT_COND							\
-	if (__predict_false((cvp = (*cond)) <= THR_COND_DESTROYED)) {		\
+	if (*cond == THR_PSHARED_PTR) {						\
+		cvp = __thr_pshared_offpage(cond, 0);				\
+		if (cvp == NULL)						\
+			return (EINVAL);					\
+	} else if (__predict_false((cvp = (*cond)) <= THR_COND_DESTROYED)) {	\
 		if (cvp == THR_COND_INITIALIZER) {				\
 			int ret;						\
 			ret = init_static(_get_curthread(), cond);		\
@@ -129,21 +153,22 @@ _pthread_cond_init(pthread_cond_t *cond, const pthread_condattr_t *cond_attr)
 int
 _pthread_cond_destroy(pthread_cond_t *cond)
 {
-	struct pthread_cond	*cvp;
-	int			error = 0;
+	struct pthread_cond *cvp;
+	int error;
 
-	if ((cvp = *cond) == THR_COND_INITIALIZER)
-		error = 0;
-	else if (cvp == THR_COND_DESTROYED)
+	error = 0;
+	if (*cond == THR_PSHARED_PTR) {
+		cvp = __thr_pshared_offpage(cond, 0);
+		if (cvp != NULL)
+			__thr_pshared_destroy(cond);
+		*cond = THR_COND_DESTROYED;
+	} else if ((cvp = *cond) == THR_COND_INITIALIZER) {
+		/* nothing */
+	} else if (cvp == THR_COND_DESTROYED) {
 		error = EINVAL;
-	else {
+	} else {
 		cvp = *cond;
 		*cond = THR_COND_DESTROYED;
-
-		/*
-		 * Free the memory allocated for the condition
-		 * variable structure:
-		 */
 		free(cvp);
 	}
 	return (error);
@@ -297,7 +322,13 @@ cond_wait_common(pthread_cond_t *cond, pthread_mutex_t *mutex,
 
 	CHECK_AND_INIT_COND
 
-	mp = *mutex;
+	if (*mutex == THR_PSHARED_PTR) {
+		mp = __thr_pshared_offpage(mutex, 0);
+		if (mp == NULL)
+			return (EINVAL);
+	} else {
+		mp = *mutex;
+	}
 
 	if ((error = _mutex_owned(curthread, mp)) != 0)
 		return (error);
@@ -385,7 +416,7 @@ cond_signal_common(pthread_cond_t *cond)
 	td = _sleepq_first(sq);
 	mp = td->mutex_obj;
 	cvp->__has_user_waiters = _sleepq_remove(sq, td);
-	if (mp->m_owner == curthread) {
+	if (mp->m_owner == TID(curthread)) {
 		if (curthread->nwaiter_defer >= MAX_DEFER_WAITERS) {
 			_thr_wake_all(curthread->defer_waiters,
 					curthread->nwaiter_defer);
@@ -417,7 +448,7 @@ drop_cb(struct pthread *td, void *arg)
 	struct pthread *curthread = ba->curthread;
 
 	mp = td->mutex_obj;
-	if (mp->m_owner == curthread) {
+	if (mp->m_owner == TID(curthread)) {
 		if (curthread->nwaiter_defer >= MAX_DEFER_WAITERS) {
 			_thr_wake_all(curthread->defer_waiters,
 				curthread->nwaiter_defer);
diff --git a/lib/libthr/thread/thr_condattr.c b/lib/libthr/thread/thr_condattr.c
index 52b9f513eb77..7ce827bd561d 100644
--- a/lib/libthr/thread/thr_condattr.c
+++ b/lib/libthr/thread/thr_condattr.c
@@ -105,20 +105,21 @@ _pthread_condattr_setclock(pthread_condattr_t *attr, clockid_t clock_id)
 int
 _pthread_condattr_getpshared(const pthread_condattr_t *attr, int *pshared)
 {
+
 	if (attr == NULL || *attr == NULL)
 		return (EINVAL);
-
-	*pshared = PTHREAD_PROCESS_PRIVATE;
+	*pshared = (*attr)->c_pshared;
 	return (0);
 }
 
 int
 _pthread_condattr_setpshared(pthread_condattr_t *attr, int pshared)
 {
-	if (attr == NULL || *attr == NULL)
-		return (EINVAL);
 
-	if  (pshared != PTHREAD_PROCESS_PRIVATE)
+	if (attr == NULL || *attr == NULL ||
+	    (pshared != PTHREAD_PROCESS_PRIVATE &&
+	    pshared != PTHREAD_PROCESS_SHARED))
 		return (EINVAL);
+	(*attr)->c_pshared = pshared;
 	return (0);
 }
diff --git a/lib/libthr/thread/thr_create.c b/lib/libthr/thread/thr_create.c
index 701a1bd92c16..81a95da8e956 100644
--- a/lib/libthr/thread/thr_create.c
+++ b/lib/libthr/thread/thr_create.c
@@ -56,12 +56,12 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	struct thr_param param;
 	struct sched_param sched_param;
 	struct rtprio rtp;
-	int ret = 0, locked, create_suspended;
 	sigset_t set, oset;
-	cpuset_t *cpusetp = NULL;
-	int cpusetsize = 0;
-	int old_stack_prot;
+	cpuset_t *cpusetp;
+	int i, cpusetsize, create_suspended, locked, old_stack_prot, ret;
 
+	cpusetp = NULL;
+	ret = cpusetsize = 0;
 	_thr_check_init();
 
 	/*
@@ -118,8 +118,8 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	new_thread->cancel_enable = 1;
 	new_thread->cancel_async = 0;
 	/* Initialize the mutex queue: */
-	TAILQ_INIT(&new_thread->mutexq);
-	TAILQ_INIT(&new_thread->pp_mutexq);
+	for (i = 0; i < TMQ_NITEMS; i++)
+		TAILQ_INIT(&new_thread->mq[i]);
 
 	/* Initialise hooks in the thread structure: */
 	if (new_thread->attr.suspend == THR_CREATE_SUSPENDED) {
diff --git a/lib/libthr/thread/thr_init.c b/lib/libthr/thread/thr_init.c
index e0400e457ce3..3c81299937f2 100644
--- a/lib/libthr/thread/thr_init.c
+++ b/lib/libthr/thread/thr_init.c
@@ -91,13 +91,15 @@ struct pthread_attr _pthread_attr_default = {
 struct pthread_mutex_attr _pthread_mutexattr_default = {
 	.m_type = PTHREAD_MUTEX_DEFAULT,
 	.m_protocol = PTHREAD_PRIO_NONE,
-	.m_ceiling = 0
+	.m_ceiling = 0,
+	.m_pshared = PTHREAD_PROCESS_PRIVATE,
 };
 
 struct pthread_mutex_attr _pthread_mutexattr_adaptive_default = {
 	.m_type = PTHREAD_MUTEX_ADAPTIVE_NP,
 	.m_protocol = PTHREAD_PRIO_NONE,
-	.m_ceiling = 0
+	.m_ceiling = 0,
+	.m_pshared = PTHREAD_PROCESS_PRIVATE,
 };
 
 /* Default condition variable attributes: */
@@ -387,6 +389,7 @@ static void
 init_main_thread(struct pthread *thread)
 {
 	struct sched_param sched_param;
+	int i;
 
 	/* Setup the thread attributes. */
 	thr_self(&thread->tid);
@@ -428,9 +431,9 @@ init_main_thread(struct pthread *thread)
 	thread->cancel_enable = 1;
 	thread->cancel_async = 0;
 
-	/* Initialize the mutex queue: */
-	TAILQ_INIT(&thread->mutexq);
-	TAILQ_INIT(&thread->pp_mutexq);
+	/* Initialize the mutex queues */
+	for (i = 0; i < TMQ_NITEMS; i++)
+		TAILQ_INIT(&thread->mq[i]);
 
 	thread->state = PS_RUNNING;
 
@@ -463,6 +466,7 @@ init_private(void)
 	_thr_once_init();
 	_thr_spinlock_init();
 	_thr_list_init();
+	__thr_pshared_init();
 	_thr_wake_addr_init();
 	_sleepq_init();
 	_single_thread = NULL;
diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c
index 26e87763cb8e..30a8be2767f5 100644
--- a/lib/libthr/thread/thr_mutex.c
+++ b/lib/libthr/thread/thr_mutex.c
@@ -1,8 +1,13 @@
 /*
  * Copyright (c) 1995 John Birrell <jb@cimlogic.com.au>.
  * Copyright (c) 2006 David Xu <davidxu@freebsd.org>.
+ * Copyright (c) 2015 The FreeBSD Foundation
+ *
  * All rights reserved.
  *
+ * Portions of this software were developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
  * Redistribution and use in source and binary forms, with or without
  * modification, are permitted provided that the following conditions
  * are met:
@@ -45,26 +50,6 @@
 
 #include "thr_private.h"
 
-#if defined(_PTHREADS_INVARIANTS)
-#define MUTEX_INIT_LINK(m) 		do {		\
-	(m)->m_qe.tqe_prev = NULL;			\
-	(m)->m_qe.tqe_next = NULL;			\
-} while (0)
-#define MUTEX_ASSERT_IS_OWNED(m)	do {		\
-	if (__predict_false((m)->m_qe.tqe_prev == NULL))\
-		PANIC("mutex is not on list");		\
-} while (0)
-#define MUTEX_ASSERT_NOT_OWNED(m)	do {		\
-	if (__predict_false((m)->m_qe.tqe_prev != NULL ||	\
-	    (m)->m_qe.tqe_next != NULL))	\
-		PANIC("mutex is on list");		\
-} while (0)
-#else
-#define MUTEX_INIT_LINK(m)
-#define MUTEX_ASSERT_IS_OWNED(m)
-#define MUTEX_ASSERT_NOT_OWNED(m)
-#endif
-
 /*
  * For adaptive mutexes, how many times to spin doing trylock2
  * before entering the kernel to block
@@ -122,36 +107,71 @@ __strong_reference(__pthread_mutex_setyieldloops_np, _pthread_mutex_setyieldloop
 __weak_reference(_pthread_mutex_getyieldloops_np, pthread_mutex_getyieldloops_np);
 __weak_reference(_pthread_mutex_isowned_np, pthread_mutex_isowned_np);
 
+static void
+mutex_init_link(struct pthread_mutex *m)
+{
+
+#if defined(_PTHREADS_INVARIANTS)
+	m->m_qe.tqe_prev = NULL;
+	m->m_qe.tqe_next = NULL;
+	m->m_pqe.tqe_prev = NULL;
+	m->m_pqe.tqe_next = NULL;
+#endif
+}
+
+static void
+mutex_assert_is_owned(struct pthread_mutex *m)
+{
+
+#if defined(_PTHREADS_INVARIANTS)
+	if (__predict_false(m->m_qe.tqe_prev == NULL))
+		PANIC("mutex is not on list");
+#endif
+}
+
+static void
+mutex_assert_not_owned(struct pthread_mutex *m)
+{
+
+#if defined(_PTHREADS_INVARIANTS)
+	if (__predict_false(m->m_qe.tqe_prev != NULL ||
+	    m->m_qe.tqe_next != NULL))
+		PANIC("mutex is on list");
+#endif
+}
+
 static int
-mutex_init(pthread_mutex_t *mutex,
-    const struct pthread_mutex_attr *mutex_attr,
-    void *(calloc_cb)(size_t, size_t))
+is_pshared_mutex(struct pthread_mutex *m)
 {
-	const struct pthread_mutex_attr *attr;
-	struct pthread_mutex *pmutex;
 
-	if (mutex_attr == NULL) {
-		attr = &_pthread_mutexattr_default;
-	} else {
-		attr = mutex_attr;
-		if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
-		    attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
-			return (EINVAL);
-		if (attr->m_protocol < PTHREAD_PRIO_NONE ||
-		    attr->m_protocol > PTHREAD_PRIO_PROTECT)
-			return (EINVAL);
-	}
-	if ((pmutex = (pthread_mutex_t)
-		calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
-		return (ENOMEM);
+	return ((m->m_lock.m_flags & USYNC_PROCESS_SHARED) != 0);
+}
+
+static int
+mutex_check_attr(const struct pthread_mutex_attr *attr)
+{
+
+	if (attr->m_type < PTHREAD_MUTEX_ERRORCHECK ||
+	    attr->m_type >= PTHREAD_MUTEX_TYPE_MAX)
+		return (EINVAL);
+	if (attr->m_protocol < PTHREAD_PRIO_NONE ||
+	    attr->m_protocol > PTHREAD_PRIO_PROTECT)
+		return (EINVAL);
+	return (0);
+}
+
+static void
+mutex_init_body(struct pthread_mutex *pmutex,
+    const struct pthread_mutex_attr *attr)
+{
 
 	pmutex->m_flags = attr->m_type;
-	pmutex->m_owner = NULL;
+	pmutex->m_owner = 0;
 	pmutex->m_count = 0;
 	pmutex->m_spinloops = 0;
 	pmutex->m_yieldloops = 0;
-	MUTEX_INIT_LINK(pmutex);
-	switch(attr->m_protocol) {
+	mutex_init_link(pmutex);
+	switch (attr->m_protocol) {
 	case PTHREAD_PRIO_NONE:
 		pmutex->m_lock.m_owner = UMUTEX_UNOWNED;
 		pmutex->m_lock.m_flags = 0;
@@ -166,13 +186,37 @@ mutex_init(pthread_mutex_t *mutex,
 		pmutex->m_lock.m_ceilings[0] = attr->m_ceiling;
 		break;
 	}
+	if (attr->m_pshared == PTHREAD_PROCESS_SHARED)
+		pmutex->m_lock.m_flags |= USYNC_PROCESS_SHARED;
 
 	if (PMUTEX_TYPE(pmutex->m_flags) == PTHREAD_MUTEX_ADAPTIVE_NP) {
 		pmutex->m_spinloops =
 		    _thr_spinloops ? _thr_spinloops: MUTEX_ADAPTIVE_SPINS;
 		pmutex->m_yieldloops = _thr_yieldloops;
 	}
+}
 
+static int
+mutex_init(pthread_mutex_t *mutex,
+    const struct pthread_mutex_attr *mutex_attr,
+    void *(calloc_cb)(size_t, size_t))
+{
+	const struct pthread_mutex_attr *attr;
+	struct pthread_mutex *pmutex;
+	int error;
+
+	if (mutex_attr == NULL) {
+		attr = &_pthread_mutexattr_default;
+	} else {
+		attr = mutex_attr;
+		error = mutex_check_attr(attr);
+		if (error != 0)
+			return (error);
+	}
+	if ((pmutex = (pthread_mutex_t)
+		calloc_cb(1, sizeof(struct pthread_mutex))) == NULL)
+		return (ENOMEM);
+	mutex_init_body(pmutex, attr);
 	*mutex = pmutex;
 	return (0);
 }
@@ -187,7 +231,8 @@ init_static(struct pthread *thread, pthread_mutex_t *mutex)
 	if (*mutex == THR_MUTEX_INITIALIZER)
 		ret = mutex_init(mutex, &_pthread_mutexattr_default, calloc);
 	else if (*mutex == THR_ADAPTIVE_MUTEX_INITIALIZER)
-		ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default, calloc);
+		ret = mutex_init(mutex, &_pthread_mutexattr_adaptive_default,
+		    calloc);
 	else
 		ret = 0;
 	THR_LOCK_RELEASE(thread, &_mutex_static_lock);
@@ -200,7 +245,7 @@ set_inherited_priority(struct pthread *curthread, struct pthread_mutex *m)
 {
 	struct pthread_mutex *m2;
 
-	m2 = TAILQ_LAST(&curthread->pp_mutexq, mutex_queue);
+	m2 = TAILQ_LAST(&curthread->mq[TMQ_NORM_PP], mutex_queue);
 	if (m2 != NULL)
 		m->m_lock.m_ceilings[1] = m2->m_lock.m_ceilings[0];
 	else
@@ -211,7 +256,25 @@ int
 __pthread_mutex_init(pthread_mutex_t *mutex,
     const pthread_mutexattr_t *mutex_attr)
 {
-	return mutex_init(mutex, mutex_attr ? *mutex_attr : NULL, calloc);
+	struct pthread_mutex *pmtx;
+	int ret;
+
+	if (mutex_attr != NULL) {
+		ret = mutex_check_attr(*mutex_attr);
+		if (ret != 0)
+			return (ret);
+	}
+	if (mutex_attr == NULL ||
+	    (*mutex_attr)->m_pshared == PTHREAD_PROCESS_PRIVATE) {
+		return (mutex_init(mutex, mutex_attr ? *mutex_attr : NULL,
+		   calloc));
+	}
+	pmtx = __thr_pshared_offpage(mutex, 1);
+	if (pmtx == NULL)
+		return (EFAULT);
+	*mutex = THR_PSHARED_PTR;
+	mutex_init_body(pmtx, *mutex_attr);
+	return (0);
 }
 
 /* This function is used internally by malloc. */
@@ -222,7 +285,8 @@ _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
 	static const struct pthread_mutex_attr attr = {
 		.m_type = PTHREAD_MUTEX_NORMAL,
 		.m_protocol = PTHREAD_PRIO_NONE,
-		.m_ceiling = 0
+		.m_ceiling = 0,
+		.m_pshared = PTHREAD_PROCESS_PRIVATE,
 	};
 	int ret;
 
@@ -232,31 +296,44 @@ _pthread_mutex_init_calloc_cb(pthread_mutex_t *mutex,
 	return (ret);
 }
 
-void
-_mutex_fork(struct pthread *curthread)
+/*
+ * Fix mutex ownership for child process.
+ *
+ * Process private mutex ownership is transmitted from the forking
+ * thread to the child process.
+ *
+ * Process shared mutex should not be inherited because owner is
+ * forking thread which is in parent process, they are removed from
+ * the owned mutex list.
+ */
+static void
+queue_fork(struct pthread *curthread, struct mutex_queue *q,
+    struct mutex_queue *qp, uint bit)
 {
 	struct pthread_mutex *m;
 
-	/*
-	 * Fix mutex ownership for child process.
-	 * note that process shared mutex should not
-	 * be inherited because owner is forking thread
-	 * which is in parent process, they should be
-	 * removed from the owned mutex list, current,
-	 * process shared mutex is not supported, so I
-	 * am not worried.
-	 */
+	TAILQ_INIT(q);
+	TAILQ_FOREACH(m, qp, m_pqe) {
+		TAILQ_INSERT_TAIL(q, m, m_qe);
+		m->m_lock.m_owner = TID(curthread) | bit;
+		m->m_owner = TID(curthread);
+	}
+}
+
+void
+_mutex_fork(struct pthread *curthread)
+{
 
-	TAILQ_FOREACH(m, &curthread->mutexq, m_qe)
-		m->m_lock.m_owner = TID(curthread);
-	TAILQ_FOREACH(m, &curthread->pp_mutexq, m_qe)
-		m->m_lock.m_owner = TID(curthread) | UMUTEX_CONTESTED;
+	queue_fork(curthread, &curthread->mq[TMQ_NORM],
+	    &curthread->mq[TMQ_NORM_PRIV], 0);
+	queue_fork(curthread, &curthread->mq[TMQ_NORM_PP],
+	    &curthread->mq[TMQ_NORM_PP_PRIV], UMUTEX_CONTESTED);
 }
 
 int
 _pthread_mutex_destroy(pthread_mutex_t *mutex)
 {
-	pthread_mutex_t m;
+	pthread_mutex_t m, m1;
 	int ret;
 
 	m = *mutex;
@@ -265,11 +342,20 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex)
 	} else if (m == THR_MUTEX_DESTROYED) {
 		ret = EINVAL;
 	} else {
-		if (m->m_owner != NULL) {
+		if (m == THR_PSHARED_PTR) {
+			m1 = __thr_pshared_offpage(mutex, 0);
+			if (m1 != NULL) {
+				mutex_assert_not_owned(m1);
+				__thr_pshared_destroy(mutex);
+			}
+			*mutex = THR_MUTEX_DESTROYED;
+			return (0);
+		}
+		if (m->m_owner != 0) {
 			ret = EBUSY;
 		} else {
 			*mutex = THR_MUTEX_DESTROYED;
-			MUTEX_ASSERT_NOT_OWNED(m);
+			mutex_assert_not_owned(m);
 			free(m);
 			ret = 0;
 		}
@@ -278,54 +364,87 @@ _pthread_mutex_destroy(pthread_mutex_t *mutex)
 	return (ret);
 }
 
-#define ENQUEUE_MUTEX(curthread, m)  					\
-	do {								\
-		(m)->m_owner = curthread;				\
-		/* Add to the list of owned mutexes: */			\
-		MUTEX_ASSERT_NOT_OWNED((m));				\
-		if (((m)->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)	\
-			TAILQ_INSERT_TAIL(&curthread->mutexq, (m), m_qe);\
-		else							\
-			TAILQ_INSERT_TAIL(&curthread->pp_mutexq, (m), m_qe);\
-	} while (0)
-
-#define DEQUEUE_MUTEX(curthread, m)					\
-		(m)->m_owner = NULL;					\
-		MUTEX_ASSERT_IS_OWNED(m);				\
-		if (__predict_true(((m)->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)) \
-			TAILQ_REMOVE(&curthread->mutexq, (m), m_qe);		\
-		else {							\
-			TAILQ_REMOVE(&curthread->pp_mutexq, (m), m_qe);	\
-			set_inherited_priority(curthread, m);		\
-		}							\
-		MUTEX_INIT_LINK(m);
-
-#define CHECK_AND_INIT_MUTEX						\
-	if (__predict_false((m = *mutex) <= THR_MUTEX_DESTROYED)) {	\
-		if (m == THR_MUTEX_DESTROYED)				\
-			return (EINVAL);				\
-		int ret;						\
-		ret = init_static(_get_curthread(), mutex);		\
-		if (ret)						\
-			return (ret);					\
-		m = *mutex;						\
-	}
+static int
+mutex_qidx(struct pthread_mutex *m)
+{
+
+	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
+		return (TMQ_NORM);
+	return (TMQ_NORM_PP);
+}
+
+static void
+enqueue_mutex(struct pthread *curthread, struct pthread_mutex *m)
+{
+	int qidx;
+
+	m->m_owner = TID(curthread);
+	/* Add to the list of owned mutexes: */
+	mutex_assert_not_owned(m);
+	qidx = mutex_qidx(m);
+	TAILQ_INSERT_TAIL(&curthread->mq[qidx], m, m_qe);
+	if (!is_pshared_mutex(m))
+		TAILQ_INSERT_TAIL(&curthread->mq[qidx + 1], m, m_pqe);
+}
+
+static void
+dequeue_mutex(struct pthread *curthread, struct pthread_mutex *m)
+{
+	int qidx;
+
+	m->m_owner = 0;
+	mutex_assert_is_owned(m);
+	qidx = mutex_qidx(m);
+	TAILQ_REMOVE(&curthread->mq[qidx], m, m_qe);
+	if (!is_pshared_mutex(m))
+		TAILQ_REMOVE(&curthread->mq[qidx + 1], m, m_pqe);
+	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) != 0)
+		set_inherited_priority(curthread, m);
+	mutex_init_link(m);
+}
 
 static int
-mutex_trylock_common(pthread_mutex_t *mutex)
+check_and_init_mutex(pthread_mutex_t *mutex, struct pthread_mutex **m)
 {
-	struct pthread *curthread = _get_curthread();
-	struct pthread_mutex *m = *mutex;
+	int ret;
+
+	*m = *mutex;
+	ret = 0;
+	if (*m == THR_PSHARED_PTR) {
+		*m = __thr_pshared_offpage(mutex, 0);
+		if (*m == NULL)
+			ret = EINVAL;
+	} else if (__predict_false(*m <= THR_MUTEX_DESTROYED)) {
+		if (*m == THR_MUTEX_DESTROYED) {
+			ret = EINVAL;
+		} else {
+			ret = init_static(_get_curthread(), mutex);
+			if (ret == 0)
+				*m = *mutex;
+		}
+	}
+	return (ret);
+}
+
+int
+__pthread_mutex_trylock(pthread_mutex_t *mutex)
+{
+	struct pthread *curthread;
+	struct pthread_mutex *m;
 	uint32_t id;
 	int ret;
 
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret != 0)
+		return (ret);
+	curthread = _get_curthread();
 	id = TID(curthread);
 	if (m->m_flags & PMUTEX_FLAG_PRIVATE)
 		THR_CRITICAL_ENTER(curthread);
 	ret = _thr_umutex_trylock(&m->m_lock, id);
 	if (__predict_true(ret == 0)) {
-		ENQUEUE_MUTEX(curthread, m);
-	} else if (m->m_owner == curthread) {
+		enqueue_mutex(curthread, m);
+	} else if (m->m_owner == id) {
 		ret = mutex_self_trylock(m);
 	} /* else {} */
 	if (ret && (m->m_flags & PMUTEX_FLAG_PRIVATE))
@@ -333,16 +452,6 @@ mutex_trylock_common(pthread_mutex_t *mutex)
 	return (ret);
 }
 
-int
-__pthread_mutex_trylock(pthread_mutex_t *mutex)
-{
-	struct pthread_mutex *m;
-
-	CHECK_AND_INIT_MUTEX
-
-	return (mutex_trylock_common(mutex));
-}
-
 static int
 mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
 	const struct timespec *abstime)
@@ -351,10 +460,10 @@ mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
 	int	count;
 	int	ret;
 
-	if (m->m_owner == curthread)
-		return mutex_self_lock(m, abstime);
-
 	id = TID(curthread);
+	if (m->m_owner == id)
+		return (mutex_self_lock(m, abstime));
+
 	/*
 	 * For adaptive mutexes, spin for a bit in the expectation
 	 * that if the application requests this mutex type then
@@ -406,7 +515,7 @@ sleep_in_kernel:
 	}
 done:
 	if (ret == 0)
-		ENQUEUE_MUTEX(curthread, m);
+		enqueue_mutex(curthread, m);
 
 	return (ret);
 }
@@ -421,7 +530,7 @@ mutex_lock_common(struct pthread_mutex *m,
 	if (!cvattach && m->m_flags & PMUTEX_FLAG_PRIVATE)
 		THR_CRITICAL_ENTER(curthread);
 	if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) {
-		ENQUEUE_MUTEX(curthread, m);
+		enqueue_mutex(curthread, m);
 		ret = 0;
 	} else {
 		ret = mutex_lock_sleep(curthread, m, abstime);
@@ -434,25 +543,28 @@ mutex_lock_common(struct pthread_mutex *m,
 int
 __pthread_mutex_lock(pthread_mutex_t *mutex)
 {
-	struct pthread_mutex	*m;
+	struct pthread_mutex *m;
+	int ret;
 
 	_thr_check_init();
-
-	CHECK_AND_INIT_MUTEX
-
-	return (mutex_lock_common(m, NULL, 0));
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		ret = mutex_lock_common(m, NULL, 0);
+	return (ret);
 }
 
 int
-__pthread_mutex_timedlock(pthread_mutex_t *mutex, const struct timespec *abstime)
+__pthread_mutex_timedlock(pthread_mutex_t *mutex,
+    const struct timespec *abstime)
 {
-	struct pthread_mutex	*m;
+	struct pthread_mutex *m;
+	int ret;
 
 	_thr_check_init();
-
-	CHECK_AND_INIT_MUTEX
-
-	return (mutex_lock_common(m, abstime, 0));
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		ret = mutex_lock_common(m, abstime, 0);
+	return (ret);
 }
 
 int
@@ -460,7 +572,13 @@ _pthread_mutex_unlock(pthread_mutex_t *mutex)
 {
 	struct pthread_mutex *mp;
 
-	mp = *mutex;
+	if (*mutex == THR_PSHARED_PTR) {
+		mp = __thr_pshared_offpage(mutex, 0);
+		if (mp == NULL)
+			return (EINVAL);
+	} else {
+		mp = *mutex;
+	}
 	return (mutex_unlock_common(mp, 0, NULL));
 }
 
@@ -493,7 +611,7 @@ _mutex_cv_attach(struct pthread_mutex *m, int count)
 {
 	struct pthread *curthread = _get_curthread();
 
-	ENQUEUE_MUTEX(curthread, m);
+	enqueue_mutex(curthread, m);
 	m->m_count = count;
 	return (0);
 }
@@ -513,7 +631,7 @@ _mutex_cv_detach(struct pthread_mutex *mp, int *recurse)
 	 */
 	*recurse = mp->m_count;
 	mp->m_count = 0;
-	DEQUEUE_MUTEX(curthread, mp);
+	dequeue_mutex(curthread, mp);
 
 	/* Will this happen in real-world ? */
         if ((mp->m_flags & PMUTEX_FLAG_DEFERED) != 0) {
@@ -641,14 +759,15 @@ mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer)
 		return (EPERM);
 	}
 
+	id = TID(curthread);
+
 	/*
 	 * Check if the running thread is not the owner of the mutex.
 	 */
-	if (__predict_false(m->m_owner != curthread))
+	if (__predict_false(m->m_owner != id))
 		return (EPERM);
 
 	error = 0;
-	id = TID(curthread);
 	if (__predict_false(
 		PMUTEX_TYPE(m->m_flags) == PTHREAD_MUTEX_RECURSIVE &&
 		m->m_count > 0)) {
@@ -660,7 +779,7 @@ mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer)
         	} else
 			defered = 0;
 
-		DEQUEUE_MUTEX(curthread, m);
+		dequeue_mutex(curthread, m);
 		error = _thr_umutex_unlock2(&m->m_lock, id, mtx_defer);
 
 		if (mtx_defer == NULL && defered)  {
@@ -676,54 +795,85 @@ mutex_unlock_common(struct pthread_mutex *m, int cv, int *mtx_defer)
 
 int
 _pthread_mutex_getprioceiling(pthread_mutex_t *mutex,
-			      int *prioceiling)
+    int *prioceiling)
 {
 	struct pthread_mutex *m;
-	int ret;
 
-	m = *mutex;
-	if ((m <= THR_MUTEX_DESTROYED) ||
-	    (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
-		ret = EINVAL;
-	else {
-		*prioceiling = m->m_lock.m_ceilings[0];
-		ret = 0;
+	if (*mutex == THR_PSHARED_PTR) {
+		m = __thr_pshared_offpage(mutex, 0);
+		if (m == NULL)
+			return (EINVAL);
+	} else {
+		m = *mutex;
+		if (m <= THR_MUTEX_DESTROYED)
+			return (EINVAL);
 	}
-
-	return (ret);
+	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
+		return (EINVAL);
+	*prioceiling = m->m_lock.m_ceilings[0];
+	return (0);
 }
 
 int
 _pthread_mutex_setprioceiling(pthread_mutex_t *mutex,
-			      int ceiling, int *old_ceiling)
+    int ceiling, int *old_ceiling)
 {
-	struct pthread *curthread = _get_curthread();
+	struct pthread *curthread;
 	struct pthread_mutex *m, *m1, *m2;
+	struct mutex_queue *q, *qp;
 	int ret;
 
-	m = *mutex;
-	if ((m <= THR_MUTEX_DESTROYED) ||
-	    (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
+	if (*mutex == THR_PSHARED_PTR) {
+		m = __thr_pshared_offpage(mutex, 0);
+		if (m == NULL)
+			return (EINVAL);
+	} else {
+		m = *mutex;
+		if (m <= THR_MUTEX_DESTROYED)
+			return (EINVAL);
+	}
+	if ((m->m_lock.m_flags & UMUTEX_PRIO_PROTECT) == 0)
 		return (EINVAL);
 
 	ret = __thr_umutex_set_ceiling(&m->m_lock, ceiling, old_ceiling);
 	if (ret != 0)
 		return (ret);
 
-	if (m->m_owner == curthread) {
-		MUTEX_ASSERT_IS_OWNED(m);
+	curthread = _get_curthread();
+	if (m->m_owner == TID(curthread)) {
+		mutex_assert_is_owned(m);
 		m1 = TAILQ_PREV(m, mutex_queue, m_qe);
 		m2 = TAILQ_NEXT(m, m_qe);
 		if ((m1 != NULL && m1->m_lock.m_ceilings[0] > (u_int)ceiling) ||
 		    (m2 != NULL && m2->m_lock.m_ceilings[0] < (u_int)ceiling)) {
-			TAILQ_REMOVE(&curthread->pp_mutexq, m, m_qe);
-			TAILQ_FOREACH(m2, &curthread->pp_mutexq, m_qe) {
+			q = &curthread->mq[TMQ_NORM_PP];
+			qp = &curthread->mq[TMQ_NORM_PP_PRIV];
+			TAILQ_REMOVE(q, m, m_qe);
+			if (!is_pshared_mutex(m))
+				TAILQ_REMOVE(qp, m, m_pqe);
+			TAILQ_FOREACH(m2, q, m_qe) {
 				if (m2->m_lock.m_ceilings[0] > (u_int)ceiling) {
 					TAILQ_INSERT_BEFORE(m2, m, m_qe);
+					if (!is_pshared_mutex(m)) {
+						while (m2 != NULL &&
+						    is_pshared_mutex(m2)) {
+							m2 = TAILQ_PREV(m2,
+							    mutex_queue, m_qe);
+						}
+						if (m2 == NULL) {
+							TAILQ_INSERT_HEAD(qp,
+							    m, m_pqe);
+						} else {
+							TAILQ_INSERT_BEFORE(m2,
+							    m, m_pqe);
+						}
+					}
 					return (0);
 				}
 			}
-			TAILQ_INSERT_TAIL(&curthread->pp_mutexq, m, m_qe);
+			TAILQ_INSERT_TAIL(q, m, m_qe);
+			if (!is_pshared_mutex(m))
+				TAILQ_INSERT_TAIL(qp, m, m_pqe);
 		}
 	}
 	return (0);
@@ -732,44 +882,48 @@ _pthread_mutex_setprioceiling(pthread_mutex_t *mutex,
 int
 _pthread_mutex_getspinloops_np(pthread_mutex_t *mutex, int *count)
 {
-	struct pthread_mutex	*m;
-
-	CHECK_AND_INIT_MUTEX
+	struct pthread_mutex *m;
+	int ret;
 
-	*count = m->m_spinloops;
-	return (0);
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		*count = m->m_spinloops;
+	return (ret);
 }
 
 int
 __pthread_mutex_setspinloops_np(pthread_mutex_t *mutex, int count)
 {
-	struct pthread_mutex	*m;
-
-	CHECK_AND_INIT_MUTEX
+	struct pthread_mutex *m;
+	int ret;
 
-	m->m_spinloops = count;
-	return (0);
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		m->m_spinloops = count;
+	return (ret);
 }
 
 int
 _pthread_mutex_getyieldloops_np(pthread_mutex_t *mutex, int *count)
 {
-	struct pthread_mutex	*m;
-
-	CHECK_AND_INIT_MUTEX
+	struct pthread_mutex *m;
+	int ret;
 
-	*count = m->m_yieldloops;
-	return (0);
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		*count = m->m_yieldloops;
+	return (ret);
 }
 
 int
 __pthread_mutex_setyieldloops_np(pthread_mutex_t *mutex, int count)
 {
-	struct pthread_mutex	*m;
-
-	CHECK_AND_INIT_MUTEX
+	struct pthread_mutex *m;
+	int ret;
 
-	m->m_yieldloops = count;
+	ret = check_and_init_mutex(mutex, &m);
+	if (ret == 0)
+		m->m_yieldloops = count;
 	return (0);
 }
 
@@ -778,10 +932,16 @@ _pthread_mutex_isowned_np(pthread_mutex_t *mutex)
 {
 	struct pthread_mutex	*m;
 
-	m = *mutex;
-	if (m <= THR_MUTEX_DESTROYED)
-		return (0);
-	return (m->m_owner == _get_curthread());
+	if (*mutex == THR_PSHARED_PTR) {
+		m = __thr_pshared_offpage(mutex, 0);
+		if (m == NULL)
+			return (0);
+	} else {
+		m = *mutex;
+		if (m <= THR_MUTEX_DESTROYED)
+			return (0);
+	}
+	return (m->m_owner == TID(_get_curthread()));
 }
 
 int
@@ -792,7 +952,7 @@ _mutex_owned(struct pthread *curthread, const struct pthread_mutex *mp)
 			return (EINVAL);
 		return (EPERM);
 	}
-      	if (mp->m_owner != curthread)
+	if (mp->m_owner != TID(curthread))
 		return (EPERM);
 	return (0);                  
 }
diff --git a/lib/libthr/thread/thr_mutexattr.c b/lib/libthr/thread/thr_mutexattr.c
index 7c48ed2263dc..fa349cfc756b 100644
--- a/lib/libthr/thread/thr_mutexattr.c
+++ b/lib/libthr/thread/thr_mutexattr.c
@@ -176,8 +176,7 @@ _pthread_mutexattr_getpshared(const pthread_mutexattr_t *attr,
 
 	if (attr == NULL || *attr == NULL)
 		return (EINVAL);
-
-	*pshared = PTHREAD_PROCESS_PRIVATE;
+	*pshared = (*attr)->m_pshared;
 	return (0);
 }
 
@@ -185,13 +184,11 @@ int
 _pthread_mutexattr_setpshared(pthread_mutexattr_t *attr, int pshared)
 {
 
-	if (attr == NULL || *attr == NULL)
+	if (attr == NULL || *attr == NULL ||
+	    (pshared != PTHREAD_PROCESS_PRIVATE &&
+	    pshared != PTHREAD_PROCESS_SHARED))
 		return (EINVAL);
-
-	/* Only PTHREAD_PROCESS_PRIVATE is supported. */
-	if (pshared != PTHREAD_PROCESS_PRIVATE)
-		return (EINVAL);
-
+	(*attr)->m_pshared = pshared;
 	return (0);
 }
 
diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h
index 6020e074c4d4..31f8e6c863da 100644
--- a/lib/libthr/thread/thr_private.h
+++ b/lib/libthr/thread/thr_private.h
@@ -126,6 +126,10 @@ TAILQ_HEAD(mutex_queue, pthread_mutex);
 		}						\
 	} while (0)
 
+/* Magic cookie set for shared pthread locks and cv's pointers */
+#define	THR_PSHARED_PTR						\
+    ((void *)(uintptr_t)((1ULL << (NBBY * sizeof(long) - 1)) | 1))
+
 /* XXX These values should be same as those defined in pthread.h */
 #define	THR_MUTEX_INITIALIZER		((struct pthread_mutex *)NULL)
 #define	THR_ADAPTIVE_MUTEX_INITIALIZER	((struct pthread_mutex *)1)
@@ -148,20 +152,24 @@ struct pthread_mutex {
 	 */
 	struct umutex			m_lock;
 	int				m_flags;
-	struct pthread			*m_owner;
+	uint32_t			m_owner;
 	int				m_count;
 	int				m_spinloops;
 	int				m_yieldloops;
 	/*
-	 * Link for all mutexes a thread currently owns.
+	 * Link for all mutexes a thread currently owns, of the same
+	 * prio type.
 	 */
 	TAILQ_ENTRY(pthread_mutex)	m_qe;
+	/* Link for all private mutexes a thread currently owns. */
+	TAILQ_ENTRY(pthread_mutex)	m_pqe;
 };
 
 struct pthread_mutex_attr {
 	enum pthread_mutextype	m_type;
 	int			m_protocol;
 	int			m_ceiling;
+	int			m_pshared;
 };
 
 #define PTHREAD_MUTEXATTR_STATIC_INITIALIZER \
@@ -313,7 +321,7 @@ struct pthread_rwlockattr {
 
 struct pthread_rwlock {
 	struct urwlock 	lock;
-	struct pthread	*owner;
+	uint32_t	owner;
 };
 
 /*
@@ -467,11 +475,16 @@ struct pthread {
 #define	TLFLAGS_IN_TDLIST	0x0002	/* thread in all thread list */
 #define	TLFLAGS_IN_GCLIST	0x0004	/* thread in gc list */
 
-	/* Queue of currently owned NORMAL or PRIO_INHERIT type mutexes. */
-	struct mutex_queue	mutexq;
-
-	/* Queue of all owned PRIO_PROTECT mutexes. */
-	struct mutex_queue	pp_mutexq;
+	/*
+	 * Queues of the owned mutexes.  Private queue must have index
+	 * + 1 of the corresponding full queue.
+	 */
+#define	TMQ_NORM		0	/* NORMAL or PRIO_INHERIT normal */
+#define	TMQ_NORM_PRIV		1	/* NORMAL or PRIO_INHERIT normal priv */
+#define	TMQ_NORM_PP		2	/* PRIO_PROTECT normal mutexes */
+#define	TMQ_NORM_PP_PRIV	3	/* PRIO_PROTECT normal priv */
+#define	TMQ_NITEMS		4
+	struct mutex_queue	mq[TMQ_NITEMS];
 
 	void				*ret;
 	struct pthread_specific_elem	*specific;
@@ -936,6 +949,10 @@ void __thr_spinlock(struct _spinlock *lck);
 struct tcb *_tcb_ctor(struct pthread *, int);
 void	_tcb_dtor(struct tcb *);
 
+void __thr_pshared_init(void) __hidden;
+void *__thr_pshared_offpage(void *key, int doalloc) __hidden;
+void __thr_pshared_destroy(void *key) __hidden;
+
 __END_DECLS
 
 #endif  /* !_THR_PRIVATE_H */
diff --git a/lib/libthr/thread/thr_pshared.c b/lib/libthr/thread/thr_pshared.c
new file mode 100644
index 000000000000..d40346d5d5eb
--- /dev/null
+++ b/lib/libthr/thread/thr_pshared.c
@@ -0,0 +1,223 @@
+/*-
+ * Copyright (c) 2015 The FreeBSD Foundation
+ *
+ * This software was developed by Konstantin Belousov
+ * under sponsorship from the FreeBSD Foundation.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ *    notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ *    notice, this list of conditions and the following disclaimer in the
+ *    documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
+ * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+ * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
+ * ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
+ * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+ * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
+ * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
+ * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
+ * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
+ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
+ * SUCH DAMAGE.
+ */
+
+#include <sys/cdefs.h>
+__FBSDID("$FreeBSD$");
+
+#include <sys/types.h>
+#include <sys/mman.h>
+#include <sys/queue.h>
+#include "namespace.h"
+#include <stdlib.h>
+#include "un-namespace.h"
+
+#include "thr_private.h"
+
+struct psh {
+	LIST_ENTRY(psh) link;
+	void *key;
+	void *val;
+};
+
+LIST_HEAD(pshared_hash_head, psh);
+#define	HASH_SIZE	128
+static struct pshared_hash_head pshared_hash[HASH_SIZE];
+#define	PSHARED_KEY_HASH(key)	(((unsigned long)(key) >> 8) % HASH_SIZE)
+/* XXXKIB: lock could be split to per-hash chain, if appears contested */
+static struct urwlock pshared_lock = DEFAULT_URWLOCK;
+
+void
+__thr_pshared_init(void)
+{
+	int i;
+
+	_thr_urwlock_init(&pshared_lock);
+	for (i = 0; i < HASH_SIZE; i++)
+		LIST_INIT(&pshared_hash[i]);
+}
+
+static void
+pshared_rlock(struct pthread *curthread)
+{
+
+	curthread->locklevel++;
+	_thr_rwl_rdlock(&pshared_lock);
+}
+
+static void
+pshared_wlock(struct pthread *curthread)
+{
+
+	curthread->locklevel++;
+	_thr_rwl_wrlock(&pshared_lock);
+}
+
+static void
+pshared_unlock(struct pthread *curthread)
+{
+
+	_thr_rwl_unlock(&pshared_lock);
+	curthread->locklevel--;
+	_thr_ast(curthread);
+}
+
+static void
+pshared_gc(struct pthread *curthread)
+{
+	struct pshared_hash_head *hd;
+	struct psh *h, *h1;
+	int error, i;
+
+	pshared_wlock(curthread);
+	for (i = 0; i < HASH_SIZE; i++) {
+		hd = &pshared_hash[i];
+		LIST_FOREACH_SAFE(h, hd, link, h1) {
+			error = _umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_ALIVE,
+			    h->val, NULL);
+			if (error == 0)
+				continue;
+			LIST_REMOVE(h, link);
+			munmap(h->val, PAGE_SIZE);
+			free(h);
+		}
+	}
+	pshared_unlock(curthread);
+}
+
+static void *
+pshared_lookup(void *key)
+{
+	struct pshared_hash_head *hd;
+	struct psh *h;
+
+	hd = &pshared_hash[PSHARED_KEY_HASH(key)];
+	LIST_FOREACH(h, hd, link) {
+		if (h->key == key)
+			return (h->val);
+	}
+	return (NULL);
+}
+
+static int
+pshared_insert(void *key, void **val)
+{
+	struct pshared_hash_head *hd;
+	struct psh *h;
+
+	hd = &pshared_hash[PSHARED_KEY_HASH(key)];
+	LIST_FOREACH(h, hd, link) {
+		if (h->key == key) {
+			if (h->val != *val) {
+				munmap(*val, PAGE_SIZE);
+				*val = h->val;
+			}
+			return (1);
+		}
+	}
+
+	h = malloc(sizeof(*h));
+	if (h == NULL)
+		return (0);
+	h->key = key;
+	h->val = *val;
+	LIST_INSERT_HEAD(hd, h, link);
+	return (1);
+}
+
+static void *
+pshared_remove(void *key)
+{
+	struct pshared_hash_head *hd;
+	struct psh *h;
+	void *val;
+
+	hd = &pshared_hash[PSHARED_KEY_HASH(key)];
+	LIST_FOREACH(h, hd, link) {
+		if (h->key == key) {
+			LIST_REMOVE(h, link);
+			val = h->val;
+			free(h);
+			return (val);
+		}
+	}
+	return (NULL);
+}
+
+static void
+pshared_clean(void *key, void *val)
+{
+
+	if (val != NULL)
+		munmap(val, PAGE_SIZE);
+	_umtx_op(NULL, UMTX_OP_SHM, UMTX_SHM_DESTROY, key, NULL);
+}
+
+void *
+__thr_pshared_offpage(void *key, int doalloc)
+{
+	struct pthread *curthread;
+	void *res;
+	int fd, ins_done;
+
+	curthread = _get_curthread();
+	pshared_rlock(curthread);
+	res = pshared_lookup(key);
+	pshared_unlock(curthread);
+	if (res != NULL)
+		return (res);
+	fd = _umtx_op(NULL, UMTX_OP_SHM, doalloc ? UMTX_SHM_CREAT :
+	    UMTX_SHM_LOOKUP, key, NULL);
+	if (fd == -1)
+		return (NULL);
+	res = mmap(NULL, PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0);
+	close(fd);
+	if (res == MAP_FAILED)
+		return (NULL);
+	pshared_wlock(curthread);
+	ins_done = pshared_insert(key, &res);
+	pshared_unlock(curthread);
+	if (!ins_done) {
+		pshared_clean(key, res);
+		res = NULL;
+	}
+	return (res);
+}
+
+void
+__thr_pshared_destroy(void *key)
+{
+	struct pthread *curthread;
+	void *val;
+
+	curthread = _get_curthread();
+	pshared_wlock(curthread);
+	val = pshared_remove(key);
+	pshared_unlock(curthread);
+	pshared_clean(key, val);
+	pshared_gc(curthread);
+}
diff --git a/lib/libthr/thread/thr_rwlock.c b/lib/libthr/thread/thr_rwlock.c
index 397663e9c710..aadd0e9ab183 100644
--- a/lib/libthr/thread/thr_rwlock.c
+++ b/lib/libthr/thread/thr_rwlock.c
@@ -46,7 +46,12 @@ __weak_reference(_pthread_rwlock_wrlock, pthread_rwlock_wrlock);
 __weak_reference(_pthread_rwlock_timedwrlock, pthread_rwlock_timedwrlock);
 
 #define CHECK_AND_INIT_RWLOCK							\
-	if (__predict_false((prwlock = (*rwlock)) <= THR_RWLOCK_DESTROYED)) {	\
+	if (*rwlock == THR_PSHARED_PTR) {					\
+		prwlock = __thr_pshared_offpage(rwlock, 0);			\
+		if (prwlock == NULL)						\
+			return (EINVAL);					\
+	} else if (__predict_false((prwlock = (*rwlock)) <=			\
+	    THR_RWLOCK_DESTROYED)) {						\
 		if (prwlock == THR_RWLOCK_INITIALIZER) {			\
 			int ret;						\
 			ret = init_static(_get_curthread(), rwlock);		\
@@ -63,14 +68,23 @@ __weak_reference(_pthread_rwlock_timedwrlock, pthread_rwlock_timedwrlock);
  */
 
 static int
-rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr __unused)
+rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
 {
 	pthread_rwlock_t prwlock;
 
-	prwlock = (pthread_rwlock_t)calloc(1, sizeof(struct pthread_rwlock));
-	if (prwlock == NULL)
-		return (ENOMEM);
-	*rwlock = prwlock;
+	if (attr == NULL || *attr == NULL ||
+	    (*attr)->pshared == PTHREAD_PROCESS_PRIVATE) {
+		prwlock = calloc(1, sizeof(struct pthread_rwlock));
+		if (prwlock == NULL)
+			return (ENOMEM);
+		*rwlock = prwlock;
+	} else {
+		prwlock = __thr_pshared_offpage(rwlock, 1);
+		if (prwlock == NULL)
+			return (EFAULT);
+		prwlock->lock.rw_flags |= USYNC_PROCESS_SHARED;
+		*rwlock = THR_PSHARED_PTR;
+	}
 	return (0);
 }
 
@@ -85,9 +99,12 @@ _pthread_rwlock_destroy (pthread_rwlock_t *rwlock)
 		ret = 0;
 	else if (prwlock == THR_RWLOCK_DESTROYED)
 		ret = EINVAL;
-	else {
+	else if (prwlock == THR_PSHARED_PTR) {
+		*rwlock = THR_RWLOCK_DESTROYED;
+		__thr_pshared_destroy(rwlock);
+		ret = 0;
+	} else {
 		*rwlock = THR_RWLOCK_DESTROYED;
-
 		free(prwlock);
 		ret = 0;
 	}
@@ -112,8 +129,9 @@ init_static(struct pthread *thread, pthread_rwlock_t *rwlock)
 }
 
 int
-_pthread_rwlock_init (pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
+_pthread_rwlock_init(pthread_rwlock_t *rwlock, const pthread_rwlockattr_t *attr)
 {
+
 	*rwlock = NULL;
 	return (rwlock_init(rwlock, attr));
 }
@@ -235,7 +253,7 @@ _pthread_rwlock_trywrlock (pthread_rwlock_t *rwlock)
 
 	ret = _thr_rwlock_trywrlock(&prwlock->lock);
 	if (ret == 0)
-		prwlock->owner = curthread;
+		prwlock->owner = TID(curthread);
 	return (ret);
 }
 
@@ -254,19 +272,19 @@ rwlock_wrlock_common (pthread_rwlock_t *rwlock, const struct timespec *abstime)
 	 */
 	ret = _thr_rwlock_trywrlock(&prwlock->lock);
 	if (ret == 0) {
-		prwlock->owner = curthread;
+		prwlock->owner = TID(curthread);
 		return (ret);
 	}
 
 	if (__predict_false(abstime && 
-		(abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0)))
+	    (abstime->tv_nsec >= 1000000000 || abstime->tv_nsec < 0)))
 		return (EINVAL);
 
 	for (;;) {
 		/* goto kernel and lock it */
 		ret = __thr_rwlock_wrlock(&prwlock->lock, abstime);
 		if (ret == 0) {
-			prwlock->owner = curthread;
+			prwlock->owner = TID(curthread);
 			break;
 		}
 
@@ -276,7 +294,7 @@ rwlock_wrlock_common (pthread_rwlock_t *rwlock, const struct timespec *abstime)
 		/* if interrupted, try to lock it in userland again. */
 		if (_thr_rwlock_trywrlock(&prwlock->lock) == 0) {
 			ret = 0;
-			prwlock->owner = curthread;
+			prwlock->owner = TID(curthread);
 			break;
 		}
 	}
@@ -297,23 +315,29 @@ _pthread_rwlock_timedwrlock (pthread_rwlock_t *rwlock,
 }
 
 int
-_pthread_rwlock_unlock (pthread_rwlock_t *rwlock)
+_pthread_rwlock_unlock(pthread_rwlock_t *rwlock)
 {
 	struct pthread *curthread = _get_curthread();
 	pthread_rwlock_t prwlock;
 	int ret;
 	int32_t state;
 
-	prwlock = *rwlock;
+	if (*rwlock == THR_PSHARED_PTR) {
+		prwlock = __thr_pshared_offpage(rwlock, 0);
+		if (prwlock == NULL)
+			return (EINVAL);
+	} else {
+		prwlock = *rwlock;
+	}
 
 	if (__predict_false(prwlock <= THR_RWLOCK_DESTROYED))
 		return (EINVAL);
 
 	state = prwlock->lock.rw_state;
 	if (state & URWLOCK_WRITE_OWNER) {
-		if (__predict_false(prwlock->owner != curthread))
+		if (__predict_false(prwlock->owner != TID(curthread)))
 			return (EPERM);
-		prwlock->owner = NULL;
+		prwlock->owner = 0;
 	}
 
 	ret = _thr_rwlock_unlock(&prwlock->lock);
diff --git a/lib/libthr/thread/thr_rwlockattr.c b/lib/libthr/thread/thr_rwlockattr.c
index 73ccdc90597f..b9e442e78bfc 100644
--- a/lib/libthr/thread/thr_rwlockattr.c
+++ b/lib/libthr/thread/thr_rwlockattr.c
@@ -45,25 +45,21 @@ _pthread_rwlockattr_destroy(pthread_rwlockattr_t *rwlockattr)
 	pthread_rwlockattr_t prwlockattr;
 
 	if (rwlockattr == NULL)
-		return(EINVAL);
-
+		return (EINVAL);
 	prwlockattr = *rwlockattr;
-
 	if (prwlockattr == NULL)
-		return(EINVAL);
-
+		return (EINVAL);
 	free(prwlockattr);
-
-	return(0);
+	return (0);
 }
 
 int
 _pthread_rwlockattr_getpshared(const pthread_rwlockattr_t *rwlockattr,
-	int *pshared)
+    int *pshared)
 {
-	*pshared = (*rwlockattr)->pshared;
 
-	return(0);
+	*pshared = (*rwlockattr)->pshared;
+	return (0);
 }
 
 int
@@ -72,28 +68,24 @@ _pthread_rwlockattr_init(pthread_rwlockattr_t *rwlockattr)
 	pthread_rwlockattr_t prwlockattr;
 
 	if (rwlockattr == NULL)
-		return(EINVAL);
-
-	prwlockattr = (pthread_rwlockattr_t)
-		malloc(sizeof(struct pthread_rwlockattr));
+		return (EINVAL);
 
+	prwlockattr = malloc(sizeof(struct pthread_rwlockattr));
 	if (prwlockattr == NULL)
-		return(ENOMEM);
+		return (ENOMEM);
 
-	prwlockattr->pshared 	= PTHREAD_PROCESS_PRIVATE;
-	*rwlockattr		= prwlockattr;
-
-	return(0);
+	prwlockattr->pshared = PTHREAD_PROCESS_PRIVATE;
+	*rwlockattr = prwlockattr;
+	return (0);
 }
 
 int
 _pthread_rwlockattr_setpshared(pthread_rwlockattr_t *rwlockattr, int pshared)
 {
-	/* Only PTHREAD_PROCESS_PRIVATE is supported. */
-	if (pshared != PTHREAD_PROCESS_PRIVATE)
-		return(EINVAL);
 
+	if (pshared != PTHREAD_PROCESS_PRIVATE &&
+	    pshared != PTHREAD_PROCESS_SHARED)
+		return (EINVAL);
 	(*rwlockattr)->pshared = pshared;
-
-	return(0);
+	return (0);
 }