4 files changed, 48 insertions, 23 deletions

diff --git a/sys/kern/kern_resource.c b/sys/kern/kern_resource.c
index 5086a5921a9c..4e22054c34bc 100644
--- a/sys/kern/kern_resource.c
+++ b/sys/kern/kern_resource.c
@@ -56,6 +56,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/syscallsubr.h>
 #include <sys/sysent.h>
 #include <sys/time.h>
+#include <sys/umtx.h>
 
 #include <vm/vm.h>
 #include <vm/vm_param.h>
@@ -482,6 +483,7 @@ int
 rtp_to_pri(struct rtprio *rtp, struct thread *td)
 {
 	u_char	newpri;
+	u_char	oldpri;
 
 	if (rtp->prio > RTP_PRIO_MAX)
 		return (EINVAL);
@@ -501,10 +503,15 @@ rtp_to_pri(struct rtprio *rtp, struct thread *td)
 		return (EINVAL);
 	}
 	sched_class(td, rtp->type);	/* XXX fix */
+	oldpri = td->td_user_pri;
 	sched_user_prio(td, newpri);
 	if (curthread == td)
 		sched_prio(curthread, td->td_user_pri); /* XXX dubious */
-	thread_unlock(td);
+	if (TD_ON_UPILOCK(td) && oldpri != newpri) {
+		thread_unlock(td);
+		umtx_pi_adjust(td, oldpri);
+	} else
+		thread_unlock(td);
 	return (0);
 }
 
diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index 6df3ce74ece4..5b09fd32714d 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -212,6 +212,7 @@ static int umtx_key_get(void *addr, int type, int share,
 static void umtx_key_release(struct umtx_key *key);
 static struct umtx_pi *umtx_pi_alloc(int);
 static void umtx_pi_free(struct umtx_pi *pi);
+static void umtx_pi_adjust_locked(struct thread *td, u_char oldpri);
 static int do_unlock_pp(struct thread *td, struct umutex *m, uint32_t flags);
 static void umtx_thread_cleanup(struct thread *td);
 static void umtx_exec_hook(void *arg __unused, struct proc *p __unused,
@@ -1294,7 +1295,7 @@ umtx_unpropagate_priority(struct umtx_pi *pi)
 {
 	struct umtx_q *uq, *uq_owner;
 	struct umtx_pi *pi2;
-	int pri;
+	int pri, oldpri;
 
 	mtx_assert(&umtx_lock, MA_OWNED);
 
@@ -1313,8 +1314,10 @@ umtx_unpropagate_priority(struct umtx_pi *pi)
 		if (pri > uq_owner->uq_inherited_pri)
 			pri = uq_owner->uq_inherited_pri;
 		thread_lock(pi->pi_owner);
+		oldpri = pi->pi_owner->td_user_pri;
 		sched_unlend_user_prio(pi->pi_owner, pri);
 		thread_unlock(pi->pi_owner);
+		umtx_pi_adjust_locked(pi->pi_owner, oldpri);
 		pi = uq_owner->uq_pi_blocked;
 	}
 }
@@ -1372,12 +1375,8 @@ umtx_pi_claim(struct umtx_pi *pi, struct thread *owner)
 	return (0);
 }
 
-/*
- * Adjust a thread's order position in its blocked PI mutex,
- * this may result new priority propagating process.
- */
-void
-umtx_pi_adjust(struct thread *td, u_char oldpri)
+static void
+umtx_pi_adjust_locked(struct thread *td, u_char oldpri)
 {
 	struct umtx_q *uq;
 	struct umtx_pi *pi;
@@ -1406,6 +1405,27 @@ umtx_pi_adjust(struct thread *td, u_char oldpri)
 }
 
 /*
+ * Adjust a thread's order position in its blocked PI mutex,
+ * this may result new priority propagating process.
+ */
+void
+umtx_pi_adjust(struct thread *td, u_char oldpri)
+{
+	struct umtx_q *uq;
+	struct umtx_pi *pi;
+
+	uq = td->td_umtxq;
+	mtx_lock_spin(&umtx_lock);
+	/*
+	 * Pick up the lock that td is blocked on.
+	 */
+	pi = uq->uq_pi_blocked;
+	if (pi != NULL)
+		umtx_pi_adjust_locked(td, oldpri);
+	mtx_unlock_spin(&umtx_lock);
+}
+
+/*
  * Sleep on a PI mutex.
  */
 static int
@@ -1480,7 +1500,9 @@ umtxq_sleep_pi(struct umtx_q *uq, struct umtx_pi *pi,
 
 	mtx_lock_spin(&umtx_lock);
 	uq->uq_pi_blocked = NULL;
+	thread_lock(td);
 	td->td_flags &= ~TDF_UPIBLOCKED;
+	thread_unlock(td);
 	TAILQ_REMOVE(&pi->pi_blocked, uq, uq_lockq);
 	umtx_unpropagate_priority(pi);
 	mtx_unlock_spin(&umtx_lock);
@@ -2706,6 +2728,8 @@ umtx_thread_cleanup(struct thread *td)
 		pi->pi_owner = NULL;
 		TAILQ_REMOVE(&uq->uq_pi_contested, pi, pi_link);
 	}
+	thread_lock(td);
 	td->td_flags &= ~TDF_UBORROWING;
+	thread_unlock(td);
 	mtx_unlock_spin(&umtx_lock);
 }
diff --git a/sys/kern/sched_4bsd.c b/sys/kern/sched_4bsd.c
index 50f9aafcab13..480d5c431936 100644
--- a/sys/kern/sched_4bsd.c
+++ b/sys/kern/sched_4bsd.c
@@ -783,14 +783,12 @@ sched_user_prio(struct thread *td, u_char prio)
 {
 	u_char oldprio;
 
+	THREAD_LOCK_ASSERT(td, MA_OWNED);
 	td->td_base_user_pri = prio;
 	if (td->td_flags & TDF_UBORROWING && td->td_user_pri <= prio)
 		return;
 	oldprio = td->td_user_pri;
 	td->td_user_pri = prio;
-
-	if (TD_ON_UPILOCK(td) && oldprio != prio)
-		umtx_pi_adjust(td, oldprio);
 }
 
 void
@@ -798,13 +796,11 @@ sched_lend_user_prio(struct thread *td, u_char prio)
 {
 	u_char oldprio;
 
+	THREAD_LOCK_ASSERT(td, MA_OWNED);
 	td->td_flags |= TDF_UBORROWING;
 
 	oldprio = td->td_user_pri;
 	td->td_user_pri = prio;
-
-	if (TD_ON_UPILOCK(td) && oldprio != prio)
-		umtx_pi_adjust(td, oldprio);
 }
 
 void
@@ -812,12 +808,14 @@ sched_unlend_user_prio(struct thread *td, u_char prio)
 {
 	u_char base_pri;
 
+	THREAD_LOCK_ASSERT(td, MA_OWNED);
 	base_pri = td->td_base_user_pri;
 	if (prio >= base_pri) {
 		td->td_flags &= ~TDF_UBORROWING;
 		sched_user_prio(td, base_pri);
-	} else
+	} else {
 		sched_lend_user_prio(td, prio);
+	}
 }
 
 void
diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index 0acacfe12df1..1ef177f7247d 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -1709,9 +1709,6 @@ sched_user_prio(struct thread *td, u_char prio)
                 return;
 	oldprio = td->td_user_pri;
 	td->td_user_pri = prio;
-
-	if (TD_ON_UPILOCK(td) && oldprio != prio)
-		umtx_pi_adjust(td, oldprio);
 }
 
 void
@@ -1719,13 +1716,10 @@ sched_lend_user_prio(struct thread *td, u_char prio)
 {
 	u_char oldprio;
 
+	THREAD_LOCK_ASSERT(td, MA_OWNED);
 	td->td_flags |= TDF_UBORROWING;
-
 	oldprio = td->td_user_pri;
 	td->td_user_pri = prio;
-
-	if (TD_ON_UPILOCK(td) && oldprio != prio)
-		umtx_pi_adjust(td, oldprio);
 }
 
 void
@@ -1733,12 +1727,14 @@ sched_unlend_user_prio(struct thread *td, u_char prio)
 {
 	u_char base_pri;
 
+	THREAD_LOCK_ASSERT(td, MA_OWNED);
 	base_pri = td->td_base_user_pri;
 	if (prio >= base_pri) {
 		td->td_flags &= ~TDF_UBORROWING;
 		sched_user_prio(td, base_pri);
-	} else
+	} else {
 		sched_lend_user_prio(td, prio);
+	}
 }
 
 /*