From 61a74c5ccd65d1a00a96779f16eda8c41ff3a426 Mon Sep 17 00:00:00 2001 From: Jeff Roberson Date: Sun, 15 Dec 2019 21:11:15 +0000 Subject: schedlock 1/4 Eliminate recursion from most thread_lock consumers. Return from sched_add() without the thread_lock held. This eliminates unnecessary atomics and lock word loads as well as reducing the hold time for scheduler locks. This will eventually allow for lockless remote adds. Discussed with: kib Reviewed by: jhb Tested by: pho Differential Revision: https://reviews.freebsd.org/D22626 --- sys/kern/subr_turnstile.c | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) (limited to 'sys/kern/subr_turnstile.c') diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index 12272e39d45df..6d303cef210c5 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -314,7 +314,7 @@ turnstile_adjust_thread(struct turnstile *ts, struct thread *td) * It needs to be moved if either its priority is lower than * the previous thread or higher than the next thread. */ - THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock); + THREAD_LOCKPTR_BLOCKED_ASSERT(td, &ts->ts_lock); td1 = TAILQ_PREV(td, threadqueue, td_lockq); td2 = TAILQ_NEXT(td, td_lockq); if ((td1 != NULL && td->td_priority < td1->td_priority) || @@ -429,7 +429,7 @@ turnstile_adjust(struct thread *td, u_char oldpri) */ ts = td->td_blocked; MPASS(ts != NULL); - THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock); + THREAD_LOCKPTR_BLOCKED_ASSERT(td, &ts->ts_lock); mtx_assert(&ts->ts_lock, MA_OWNED); /* Resort the turnstile on the list. */ @@ -693,7 +693,7 @@ turnstile_claim(struct turnstile *ts) td = turnstile_first_waiter(ts); MPASS(td != NULL); MPASS(td->td_proc->p_magic == P_MAGIC); - THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock); + THREAD_LOCKPTR_BLOCKED_ASSERT(td, &ts->ts_lock); /* * Update the priority of the new owner if needed. @@ -979,7 +979,7 @@ turnstile_unpend(struct turnstile *ts) td = TAILQ_FIRST(&pending_threads); TAILQ_REMOVE(&pending_threads, td, td_lockq); SDT_PROBE2(sched, , , wakeup, td, td->td_proc); - thread_lock(td); + thread_lock_block_wait(td); THREAD_LOCKPTR_ASSERT(td, &ts->ts_lock); MPASS(td->td_proc->p_magic == P_MAGIC); MPASS(TD_ON_LOCK(td)); @@ -991,8 +991,7 @@ turnstile_unpend(struct turnstile *ts) #ifdef INVARIANTS td->td_tsqueue = 0xff; #endif - sched_add(td, SRQ_BORING); - thread_unlock(td); + sched_add(td, SRQ_HOLD | SRQ_BORING); } mtx_unlock_spin(&ts->ts_lock); } -- cgit v1.2.3