From c84f3479856571c4f6505b23768324258cdf8a1c Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Sun, 5 Feb 2017 04:53:13 +0000 Subject: rwlock: switch to fcmpset Discussed with: jhb Tested by: pho --- sys/kern/kern_rwlock.c | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) (limited to 'sys/kern/kern_rwlock.c') diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c index 99df700c8836..435caa2fa17d 100644 --- a/sys/kern/kern_rwlock.c +++ b/sys/kern/kern_rwlock.c @@ -440,7 +440,7 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line) * if the lock has been unlocked and write waiters * were present. */ - if (atomic_cmpset_acq_ptr(&rw->rw_lock, v, + if (atomic_fcmpset_acq_ptr(&rw->rw_lock, &v, v + RW_ONE_READER)) { if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR4(KTR_LOCK, @@ -449,7 +449,6 @@ __rw_rlock(volatile uintptr_t *c, const char *file, int line) (void *)(v + RW_ONE_READER)); break; } - v = RW_READ_VALUE(rw); continue; } #ifdef KDTRACE_HOOKS @@ -675,7 +674,7 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line) * just drop one and return. */ if (RW_READERS(x) > 1) { - if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, + if (atomic_fcmpset_rel_ptr(&rw->rw_lock, &x, x - RW_ONE_READER)) { if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR4(KTR_LOCK, @@ -684,7 +683,6 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line) (void *)(x - RW_ONE_READER)); break; } - x = RW_READ_VALUE(rw); continue; } /* @@ -694,14 +692,13 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line) if (!(x & RW_LOCK_WAITERS)) { MPASS((x & ~RW_LOCK_WRITE_SPINNER) == RW_READERS_LOCK(1)); - if (atomic_cmpset_rel_ptr(&rw->rw_lock, x, + if (atomic_fcmpset_rel_ptr(&rw->rw_lock, &x, RW_UNLOCKED)) { if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p last succeeded", __func__, rw); break; } - x = RW_READ_VALUE(rw); continue; } /* @@ -769,8 +766,8 @@ _rw_runlock_cookie(volatile uintptr_t *c, const char *file, int line) * read or write lock. */ void -__rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, - int line) +__rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, + const char *file, int line) { struct rwlock *rw; struct turnstile *ts; @@ -779,7 +776,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, int spintries = 0; int i; #endif - uintptr_t v, x; + uintptr_t x; #ifdef LOCK_PROFILING uint64_t waittime = 0; int contested = 0; @@ -803,7 +800,6 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, lock_delay_arg_init(&lda, NULL); #endif rw = rwlock2rw(c); - v = RW_READ_VALUE(rw); if (__predict_false(lv_rw_wowner(v) == (struct thread *)tid)) { KASSERT(rw->lock_object.lo_flags & LO_RECURSABLE, @@ -825,9 +821,8 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, #endif for (;;) { if (v == RW_UNLOCKED) { - if (_rw_write_lock(rw, tid)) + if (_rw_write_lock_fetch(rw, &v, tid)) break; - v = RW_READ_VALUE(rw); continue; } #ifdef KDTRACE_HOOKS -- cgit v1.2.3 From 993ddec44dd4459d14cbd963264aa380ed0074a6 Mon Sep 17 00:00:00 2001 From: Mateusz Guzik Date: Sun, 5 Feb 2017 13:37:23 +0000 Subject: rwlock: move lockstat handling out of inline primitives See r313275 for details. One difference here is that recursion handling was removed from the fallback routine. As it is it was never supposed to see a recursed lock in the first place. Future changes will move it out of inline variants, but right now there is no easy to way to test if the lock is recursed without reading additional words. --- sys/kern/kern_rwlock.c | 25 ++++++++++++++++++------- 1 file changed, 18 insertions(+), 7 deletions(-) (limited to 'sys/kern/kern_rwlock.c') diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c index 435caa2fa17d..c024de5cc8cb 100644 --- a/sys/kern/kern_rwlock.c +++ b/sys/kern/kern_rwlock.c @@ -283,6 +283,7 @@ void _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line) { struct rwlock *rw; + uintptr_t tid, v; if (SCHEDULER_STOPPED()) return; @@ -296,7 +297,14 @@ _rw_wlock_cookie(volatile uintptr_t *c, const char *file, int line) ("rw_wlock() of destroyed rwlock @ %s:%d", file, line)); WITNESS_CHECKORDER(&rw->lock_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, line, NULL); - __rw_wlock(rw, curthread, file, line); + tid = (uintptr_t)curthread; + v = RW_UNLOCKED; + if (!_rw_write_lock_fetch(rw, &v, tid)) + _rw_wlock_hard(rw, v, tid, file, line); + else + LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw, + 0, 0, file, line, LOCKSTAT_WRITER); + LOCK_LOG_LOCK("WLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); WITNESS_LOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); TD_LOCKS_INC(curthread); @@ -355,7 +363,11 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); - __rw_wunlock(rw, curthread, file, line); + if (rw->rw_recurse) + rw->rw_recurse--; + else + _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line); + TD_LOCKS_DEC(curthread); } @@ -998,13 +1010,12 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, return; rw = rwlock2rw(c); + MPASS(!rw_recursed(rw)); - if (rw_wlocked(rw) && rw_recursed(rw)) { - rw->rw_recurse--; - if (LOCK_LOG_TEST(&rw->lock_object, 0)) - CTR2(KTR_LOCK, "%s: %p unrecursing", __func__, rw); + LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, + LOCKSTAT_WRITER); + if (_rw_write_unlock(rw, tid)) return; - } KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), ("%s: neither of the waiter flags are set", __func__)); -- cgit v1.2.3