5 files changed, 251 insertions, 90 deletions

diff --git a/lib/libthr/thread/thr_mutex.c b/lib/libthr/thread/thr_mutex.c
index 62d7ac68a56a..5b69952fa86e 100644
--- a/lib/libthr/thread/thr_mutex.c
+++ b/lib/libthr/thread/thr_mutex.c
@@ -93,6 +93,8 @@ static int	mutex_self_trylock(pthread_mutex_t);
 static int	mutex_self_lock(pthread_mutex_t,
 				const struct timespec *abstime);
 static int	mutex_unlock_common(pthread_mutex_t *);
+static int	mutex_lock_sleep(struct pthread *, pthread_mutex_t,
+				const struct timespec *);
 
 __weak_reference(__pthread_mutex_init, pthread_mutex_init);
 __strong_reference(__pthread_mutex_init, _pthread_mutex_init);
@@ -346,25 +348,24 @@ __pthread_mutex_trylock(pthread_mutex_t *mutex)
 }
 
 static int
-mutex_lock_sleep(struct pthread *curthread, pthread_mutex_t m,
-	const struct timespec * abstime)
+mutex_lock_sleep(struct pthread *curthread, struct pthread_mutex *m,
+	const struct timespec *abstime)
 {
-	struct  timespec ts, ts2;
-	uint32_t	id;
-	int	ret;
+	uint32_t	id, owner;
 	int	count;
+	int	ret;
 
-	id = TID(curthread);
-	if (__predict_false(m->m_owner == curthread))
-		return  mutex_self_lock(m, abstime);
+	if (m->m_owner == curthread)
+		return mutex_self_lock(m, abstime);
 
+	id = TID(curthread);
 	/*
 	 * For adaptive mutexes, spin for a bit in the expectation
 	 * that if the application requests this mutex type then
 	 * the lock is likely to be released quickly and it is
 	 * faster than entering the kernel
 	 */
-	if (m->m_lock.m_flags & UMUTEX_PRIO_PROTECT)
+	if (m->m_lock.m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT))
 		goto sleep_in_kernel;
 
 	if (!_thr_is_smp)
@@ -372,10 +373,12 @@ mutex_lock_sleep(struct pthread *curthread, pthread_mutex_t m,
 
 	count = m->m_spinloops;
 	while (count--) {
-		if (m->m_lock.m_owner == UMUTEX_UNOWNED) {
-			ret = _thr_umutex_trylock2(&m->m_lock, id);
-			if (ret == 0)
+		owner = m->m_lock.m_owner;
+		if ((owner & ~UMUTEX_CONTESTED) == 0) {
+			if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) {
+				ret = 0;
 				goto done;
+			}
 		}
 		CPU_SPINWAIT;
 	}
@@ -384,49 +387,43 @@ yield_loop:
 	count = m->m_yieldloops;
 	while (count--) {
 		_sched_yield();
-		ret = _thr_umutex_trylock2(&m->m_lock, id);
-		if (ret == 0)
-			goto done;
+		owner = m->m_lock.m_owner;
+		if ((owner & ~UMUTEX_CONTESTED) == 0) {
+			if (atomic_cmpset_acq_32(&m->m_lock.m_owner, owner, id|owner)) {
+				ret = 0;
+				goto done;
+			}
+		}
 	}
 
 sleep_in_kernel:
 	if (abstime == NULL) {
-		ret = __thr_umutex_lock(&m->m_lock);
+		ret = __thr_umutex_lock(&m->m_lock, id);
 	} else if (__predict_false(
-		   abstime->tv_sec < 0 || abstime->tv_nsec < 0 ||
+		   abstime->tv_nsec < 0 ||
 		   abstime->tv_nsec >= 1000000000)) {
 		ret = EINVAL;
 	} else {
-		clock_gettime(CLOCK_REALTIME, &ts);
-		TIMESPEC_SUB(&ts2, abstime, &ts);
-		ret = __thr_umutex_timedlock(&m->m_lock, &ts2);
-		/*
-		 * Timed out wait is not restarted if
-		 * it was interrupted, not worth to do it.
-		 */
-		if (ret == EINTR)
-			ret = ETIMEDOUT;
+		ret = __thr_umutex_timedlock(&m->m_lock, id, abstime);
 	}
 done:
 	if (ret == 0)
 		ENQUEUE_MUTEX(curthread, m);
+
 	return (ret);
 }
 
 static inline int
 mutex_lock_common(struct pthread *curthread, struct pthread_mutex *m,
-	const struct timespec * abstime)
+	const struct timespec *abstime)
 {
-	uint32_t	id;
-	int	ret;
 
-	id = TID(curthread);
-	ret = _thr_umutex_trylock2(&m->m_lock, id);
-	if (ret == 0)
+	if (_thr_umutex_trylock2(&m->m_lock, TID(curthread)) == 0) {
 		ENQUEUE_MUTEX(curthread, m);
-	else
-		ret = mutex_lock_sleep(curthread, m, abstime);
-	return (ret);
+		return (0);
+	}
+	
+	return (mutex_lock_sleep(curthread, m, abstime));
 }
 
 int
@@ -450,6 +447,7 @@ __pthread_mutex_lock(pthread_mutex_t *mutex)
 			return (ret);
 		m = *mutex;
 	}
+
 	return (mutex_lock_common(curthread, m, NULL));
 }
 
diff --git a/lib/libthr/thread/thr_umtx.c b/lib/libthr/thread/thr_umtx.c
index 3efe8847a1a8..7c59bb78ccb8 100644
--- a/lib/libthr/thread/thr_umtx.c
+++ b/lib/libthr/thread/thr_umtx.c
@@ -48,25 +48,74 @@ _thr_umutex_init(struct umutex *mtx)
 }
 
 int
-__thr_umutex_lock(struct umutex *mtx)
+__thr_umutex_lock(struct umutex *mtx, uint32_t id)
 {
-	return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0);
+	uint32_t owner;
+
+	if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) {
+		for (;;) {
+			/* wait in kernel */
+			_umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, 0);
+
+			owner = mtx->m_owner;
+			if ((owner & ~UMUTEX_CONTESTED) == 0 &&
+			     atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner))
+				return (0);
+		}
+	}
+
+	return	_umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, 0);
 }
 
 int
-__thr_umutex_timedlock(struct umutex *mtx,
-	const struct timespec *timeout)
+__thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
+	const struct timespec *ets)
 {
-	if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 &&
-		timeout->tv_nsec <= 0)))
+	struct timespec timo, cts;
+	uint32_t owner;
+	int ret;
+
+	clock_gettime(CLOCK_REALTIME, &cts);
+	TIMESPEC_SUB(&timo, ets, &cts);
+
+	if (timo.tv_sec < 0)
 		return (ETIMEDOUT);
-	return _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0,
-		__DECONST(void *, timeout));
+
+	for (;;) {
+		if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) {
+
+			/* wait in kernel */
+			ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_WAIT, 0, 0, &timo);
+
+			/* now try to lock it */
+			owner = mtx->m_owner;
+			if ((owner & ~UMUTEX_CONTESTED) == 0 &&
+			     atomic_cmpset_acq_32(&mtx->m_owner, owner, id|owner))
+				return (0);
+		} else {
+			ret = _umtx_op_err(mtx, UMTX_OP_MUTEX_LOCK, 0, 0, &timo);
+			if (ret == 0)
+				break;
+		}
+		if (ret == ETIMEDOUT)
+			break;
+		clock_gettime(CLOCK_REALTIME, &cts);
+		TIMESPEC_SUB(&timo, ets, &cts);
+		if (timo.tv_sec < 0 || (timo.tv_sec == 0 && timo.tv_nsec == 0)) {
+			ret = ETIMEDOUT;
+			break;
+		}
+	}
+	return (ret);
 }
 
 int
-__thr_umutex_unlock(struct umutex *mtx)
+__thr_umutex_unlock(struct umutex *mtx, uint32_t id)
 {
+	if ((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0) {
+		atomic_cmpset_rel_32(&mtx->m_owner, id | UMUTEX_CONTESTED, UMUTEX_CONTESTED);
+		return _umtx_op_err(mtx, UMTX_OP_MUTEX_WAKE, 0, 0, 0);
+	}
 	return _umtx_op_err(mtx, UMTX_OP_MUTEX_UNLOCK, 0, 0, 0);
 }
 
@@ -123,7 +172,8 @@ _thr_ucond_wait(struct ucond *cv, struct umutex *m,
 {
 	if (timeout && (timeout->tv_sec < 0 || (timeout->tv_sec == 0 &&
 	    timeout->tv_nsec <= 0))) {
-		__thr_umutex_unlock(m);
+		struct pthread *curthread = _get_curthread();
+		_thr_umutex_unlock(m, TID(curthread));
                 return (ETIMEDOUT);
 	}
 	return _umtx_op_err(cv, UMTX_OP_CV_WAIT,
diff --git a/lib/libthr/thread/thr_umtx.h b/lib/libthr/thread/thr_umtx.h
index 0ef75a7e49e2..41b5f9686783 100644
--- a/lib/libthr/thread/thr_umtx.h
+++ b/lib/libthr/thread/thr_umtx.h
@@ -34,10 +34,10 @@
 
 #define DEFAULT_UMUTEX	{0,0, {0,0},{0,0,0,0}}
 
-int __thr_umutex_lock(struct umutex *mtx) __hidden;
-int __thr_umutex_timedlock(struct umutex *mtx,
+int __thr_umutex_lock(struct umutex *mtx, uint32_t id) __hidden;
+int __thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
 	const struct timespec *timeout) __hidden;
-int __thr_umutex_unlock(struct umutex *mtx) __hidden;
+int __thr_umutex_unlock(struct umutex *mtx, uint32_t id) __hidden;
 int __thr_umutex_trylock(struct umutex *mtx) __hidden;
 int __thr_umutex_set_ceiling(struct umutex *mtx, uint32_t ceiling,
 	uint32_t *oldceiling) __hidden;
@@ -71,26 +71,30 @@ _thr_umutex_trylock(struct umutex *mtx, uint32_t id)
 static inline int
 _thr_umutex_trylock2(struct umutex *mtx, uint32_t id)
 {
-    if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id))
+    if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id) != 0)
 	return (0);
+    if ((uint32_t)mtx->m_owner == UMUTEX_CONTESTED &&
+        __predict_true((mtx->m_flags & (UMUTEX_PRIO_PROTECT | UMUTEX_PRIO_INHERIT)) == 0))
+    	if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_CONTESTED, id | UMUTEX_CONTESTED))
+		return (0);
     return (EBUSY);
 }
 
 static inline int
 _thr_umutex_lock(struct umutex *mtx, uint32_t id)
 {
-    if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id))
+    if (_thr_umutex_trylock2(mtx, id) == 0)
 	return (0);
-    return (__thr_umutex_lock(mtx));
+    return (__thr_umutex_lock(mtx, id));
 }
 
 static inline int
 _thr_umutex_timedlock(struct umutex *mtx, uint32_t id,
 	const struct timespec *timeout)
 {
-    if (atomic_cmpset_acq_32(&mtx->m_owner, UMUTEX_UNOWNED, id))
+    if (_thr_umutex_trylock2(mtx, id) == 0)
 	return (0);
-    return (__thr_umutex_timedlock(mtx, timeout));
+    return (__thr_umutex_timedlock(mtx, id, timeout));
 }
 
 static inline int
@@ -98,7 +102,7 @@ _thr_umutex_unlock(struct umutex *mtx, uint32_t id)
 {
     if (atomic_cmpset_rel_32(&mtx->m_owner, id, UMUTEX_UNOWNED))
 	return (0);
-    return (__thr_umutex_unlock(mtx));
+    return (__thr_umutex_unlock(mtx, id));
 }
 
 static inline int
diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index 8e5fcab3b867..09573ab8c368 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -66,6 +66,9 @@ __FBSDID("$FreeBSD$");
 #define TYPE_PP_UMUTEX		5
 #define TYPE_RWLOCK		6
 
+#define _UMUTEX_TRY		1
+#define _UMUTEX_WAIT		2
+
 /* Key to represent a unique userland synchronous object */
 struct umtx_key {
 	int	hash;
@@ -1037,7 +1040,7 @@ kern_umtx_wake(struct thread *td, void *uaddr, int n_wake, int is_private)
  */
 static int
 _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
-	int try)
+	int mode)
 {
 	struct umtx_q *uq;
 	uint32_t owner, old, id;
@@ -1051,40 +1054,46 @@ _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 	 * can fault on any access.
 	 */
 	for (;;) {
-		/*
-		 * Try the uncontested case.  This should be done in userland.
-		 */
-		owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
-
-		/* The acquire succeeded. */
-		if (owner == UMUTEX_UNOWNED)
-			return (0);
-
-		/* The address was invalid. */
-		if (owner == -1)
-			return (EFAULT);
-
-		/* If no one owns it but it is contested try to acquire it. */
-		if (owner == UMUTEX_CONTESTED) {
-			owner = casuword32(&m->m_owner,
-			    UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
+		owner = fuword32(__DEVOLATILE(void *, &m->m_owner));
+		if (mode == _UMUTEX_WAIT) {
+			if (owner == UMUTEX_UNOWNED || owner == UMUTEX_CONTESTED)
+				return (0);
+		} else {
+			/*
+			 * Try the uncontested case.  This should be done in userland.
+			 */
+			owner = casuword32(&m->m_owner, UMUTEX_UNOWNED, id);
 
-			if (owner == UMUTEX_CONTESTED)
+			/* The acquire succeeded. */
+			if (owner == UMUTEX_UNOWNED)
 				return (0);
 
 			/* The address was invalid. */
 			if (owner == -1)
 				return (EFAULT);
 
-			/* If this failed the lock has changed, restart. */
-			continue;
+			/* If no one owns it but it is contested try to acquire it. */
+			if (owner == UMUTEX_CONTESTED) {
+				owner = casuword32(&m->m_owner,
+				    UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
+
+				if (owner == UMUTEX_CONTESTED)
+					return (0);
+
+				/* The address was invalid. */
+				if (owner == -1)
+					return (EFAULT);
+
+				/* If this failed the lock has changed, restart. */
+				continue;
+			}
 		}
 
 		if ((flags & UMUTEX_ERROR_CHECK) != 0 &&
 		    (owner & ~UMUTEX_CONTESTED) == id)
 			return (EDEADLK);
 
-		if (try != 0)
+		if (mode == _UMUTEX_TRY)
 			return (EBUSY);
 
 		/*
@@ -1101,7 +1110,6 @@ _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 		umtxq_lock(&uq->uq_key);
 		umtxq_busy(&uq->uq_key);
 		umtxq_insert(uq);
-		umtxq_unbusy(&uq->uq_key);
 		umtxq_unlock(&uq->uq_key);
 
 		/*
@@ -1116,6 +1124,7 @@ _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 		if (old == -1) {
 			umtxq_lock(&uq->uq_key);
 			umtxq_remove(uq);
+			umtxq_unbusy(&uq->uq_key);
 			umtxq_unlock(&uq->uq_key);
 			umtx_key_release(&uq->uq_key);
 			return (EFAULT);
@@ -1127,6 +1136,7 @@ _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 		 * unlocking the umtx.
 		 */
 		umtxq_lock(&uq->uq_key);
+		umtxq_unbusy(&uq->uq_key);
 		if (old == owner)
 			error = umtxq_sleep(uq, "umtxn", timo);
 		umtxq_remove(uq);
@@ -1162,7 +1172,6 @@ do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags)
 	if ((owner & ~UMUTEX_CONTESTED) != id)
 		return (EPERM);
 
-	/* This should be done in userland */
 	if ((owner & UMUTEX_CONTESTED) == 0) {
 		old = casuword32(&m->m_owner, owner, UMUTEX_UNOWNED);
 		if (old == -1)
@@ -1201,6 +1210,50 @@ do_unlock_normal(struct thread *td, struct umutex *m, uint32_t flags)
 	return (0);
 }
 
+/*
+ * Check if the mutex is available and wake up a waiter,
+ * only for simple mutex.
+ */
+static int
+do_wake_umutex(struct thread *td, struct umutex *m)
+{
+	struct umtx_key key;
+	uint32_t owner;
+	uint32_t flags;
+	int error;
+	int count;
+
+	owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
+	if (owner == -1)
+		return (EFAULT);
+
+	if ((owner & ~UMUTEX_CONTESTED) != 0)
+		return (0);
+
+	flags = fuword32(&m->m_flags);
+
+	/* We should only ever be in here for contested locks */
+	if ((error = umtx_key_get(m, TYPE_NORMAL_UMUTEX, GET_SHARE(flags),
+	    &key)) != 0)
+		return (error);
+
+	umtxq_lock(&key);
+	umtxq_busy(&key);
+	count = umtxq_count(&key);
+	umtxq_unlock(&key);
+
+	if (count <= 1)
+		owner = casuword32(&m->m_owner, UMUTEX_CONTESTED, UMUTEX_UNOWNED);
+
+	umtxq_lock(&key);
+	if (count != 0 && (owner & ~UMUTEX_CONTESTED) == 0)
+		umtxq_signal(&key, 1);
+	umtxq_unbusy(&key);
+	umtxq_unlock(&key);
+	umtx_key_release(&key);
+	return (0);
+}
+
 static inline struct umtx_pi *
 umtx_pi_alloc(int flags)
 {
@@ -2144,15 +2197,15 @@ do_set_ceiling(struct thread *td, struct umutex *m, uint32_t ceiling,
 
 static int
 _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo,
-	int try)
+	int mode)
 {
 	switch(flags & (UMUTEX_PRIO_INHERIT | UMUTEX_PRIO_PROTECT)) {
 	case 0:
-		return (_do_lock_normal(td, m, flags, timo, try));
+		return (_do_lock_normal(td, m, flags, timo, mode));
 	case UMUTEX_PRIO_INHERIT:
-		return (_do_lock_pi(td, m, flags, timo, try));
+		return (_do_lock_pi(td, m, flags, timo, mode));
 	case UMUTEX_PRIO_PROTECT:
-		return (_do_lock_pp(td, m, flags, timo, try));
+		return (_do_lock_pp(td, m, flags, timo, mode));
 	}
 	return (EINVAL);
 }
@@ -2162,7 +2215,7 @@ _do_lock_umutex(struct thread *td, struct umutex *m, int flags, int timo,
  */
 static int
 do_lock_umutex(struct thread *td, struct umutex *m,
-	struct timespec *timeout, int try)
+	struct timespec *timeout, int mode)
 {
 	struct timespec ts, ts2, ts3;
 	struct timeval tv;
@@ -2174,16 +2227,16 @@ do_lock_umutex(struct thread *td, struct umutex *m,
 		return (EFAULT);
 
 	if (timeout == NULL) {
-		error = _do_lock_umutex(td, m, flags, 0, try);
+		error = _do_lock_umutex(td, m, flags, 0, mode);
 		/* Mutex locking is restarted if it is interrupted. */
-		if (error == EINTR)
+		if (error == EINTR && mode != _UMUTEX_WAIT)
 			error = ERESTART;
 	} else {
 		getnanouptime(&ts);
 		timespecadd(&ts, timeout);
 		TIMESPEC_TO_TIMEVAL(&tv, timeout);
 		for (;;) {
-			error = _do_lock_umutex(td, m, flags, tvtohz(&tv), try);
+			error = _do_lock_umutex(td, m, flags, tvtohz(&tv), mode);
 			if (error != ETIMEDOUT)
 				break;
 			getnanouptime(&ts2);
@@ -2830,7 +2883,36 @@ __umtx_op_lock_umutex(struct thread *td, struct _umtx_op_args *uap)
 static int
 __umtx_op_trylock_umutex(struct thread *td, struct _umtx_op_args *uap)
 {
-	return do_lock_umutex(td, uap->obj, NULL, 1);
+	return do_lock_umutex(td, uap->obj, NULL, _UMUTEX_TRY);
+}
+
+static int
+__umtx_op_wait_umutex(struct thread *td, struct _umtx_op_args *uap)
+{
+	struct timespec *ts, timeout;
+	int error;
+
+	/* Allow a null timespec (wait forever). */
+	if (uap->uaddr2 == NULL)
+		ts = NULL;
+	else {
+		error = copyin(uap->uaddr2, &timeout,
+		    sizeof(timeout));
+		if (error != 0)
+			return (error);
+		if (timeout.tv_nsec >= 1000000000 ||
+		    timeout.tv_nsec < 0) {
+			return (EINVAL);
+		}
+		ts = &timeout;
+	}
+	return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
+}
+
+static int
+__umtx_op_wake_umutex(struct thread *td, struct _umtx_op_args *uap)
+{
+	return do_wake_umutex(td, uap->obj);
 }
 
 static int
@@ -2952,7 +3034,9 @@ static _umtx_op_func op_table[] = {
 	__umtx_op_rw_wrlock,		/* UMTX_OP_RW_WRLOCK */
 	__umtx_op_rw_unlock,		/* UMTX_OP_RW_UNLOCK */
 	__umtx_op_wait_uint_private,	/* UMTX_OP_WAIT_UINT_PRIVATE */
-	__umtx_op_wake_private		/* UMTX_OP_WAKE_PRIVATE */
+	__umtx_op_wake_private,		/* UMTX_OP_WAKE_PRIVATE */
+	__umtx_op_wait_umutex,		/* UMTX_OP_UMUTEX_WAIT */
+	__umtx_op_wake_umutex		/* UMTX_OP_UMUTEX_WAKE */
 };
 
 int
@@ -3067,6 +3151,27 @@ __umtx_op_lock_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
 }
 
 static int
+__umtx_op_wait_umutex_compat32(struct thread *td, struct _umtx_op_args *uap)
+{
+	struct timespec *ts, timeout;
+	int error;
+
+	/* Allow a null timespec (wait forever). */
+	if (uap->uaddr2 == NULL)
+		ts = NULL;
+	else {
+		error = copyin_timeout32(uap->uaddr2, &timeout);
+		if (error != 0)
+			return (error);
+		if (timeout.tv_nsec >= 1000000000 ||
+		    timeout.tv_nsec < 0)
+			return (EINVAL);
+		ts = &timeout;
+	}
+	return do_lock_umutex(td, uap->obj, ts, _UMUTEX_WAIT);
+}
+
+static int
 __umtx_op_cv_wait_compat32(struct thread *td, struct _umtx_op_args *uap)
 {
 	struct timespec *ts, timeout;
@@ -3170,7 +3275,9 @@ static _umtx_op_func op_table_compat32[] = {
 	__umtx_op_rw_wrlock_compat32,	/* UMTX_OP_RW_WRLOCK */
 	__umtx_op_rw_unlock,		/* UMTX_OP_RW_UNLOCK */
 	__umtx_op_wait_uint_private_compat32,	/* UMTX_OP_WAIT_UINT_PRIVATE */
-	__umtx_op_wake_private		/* UMTX_OP_WAKE_PRIVATE */
+	__umtx_op_wake_private,		/* UMTX_OP_WAKE_PRIVATE */
+	__umtx_op_wait_umutex_compat32, /* UMTX_OP_UMUTEX_WAIT */
+	__umtx_op_wake_umutex		/* UMTX_OP_UMUTEX_WAKE */
 };
 
 int
diff --git a/sys/sys/umtx.h b/sys/sys/umtx.h
index 2857a1a8093d..2d45677ee1ae 100644
--- a/sys/sys/umtx.h
+++ b/sys/sys/umtx.h
@@ -101,7 +101,9 @@ struct urwlock {
 #define	UMTX_OP_RW_UNLOCK	14
 #define	UMTX_OP_WAIT_UINT_PRIVATE	15
 #define	UMTX_OP_WAKE_PRIVATE		16
-#define	UMTX_OP_MAX		17
+#define	UMTX_OP_MUTEX_WAIT		17
+#define	UMTX_OP_MUTEX_WAKE		18
+#define	UMTX_OP_MAX		19
 
 /* flags for UMTX_OP_CV_WAIT */
 #define UMTX_CHECK_UNPARKING	0x01