diff options
| -rw-r--r-- | sys/kern/kern_rwlock.c | 21 | ||||
| -rw-r--r-- | sys/sys/rwlock.h | 15 |
2 files changed, 19 insertions, 17 deletions
diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c index e2fe739a73b5..3b16bb227af7 100644 --- a/sys/kern/kern_rwlock.c +++ b/sys/kern/kern_rwlock.c @@ -312,6 +312,7 @@ __rw_try_wlock(volatile uintptr_t *c, const char *file, int line) if (rw_wlocked(rw) && (rw->lock_object.lo_flags & LO_RECURSABLE) != 0) { rw->rw_recurse++; + atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); rval = 1; } else rval = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_UNLOCKED, @@ -345,10 +346,8 @@ _rw_wunlock_cookie(volatile uintptr_t *c, const char *file, int line) WITNESS_UNLOCK(&rw->lock_object, LOP_EXCLUSIVE, file, line); LOCK_LOG_LOCK("WUNLOCK", &rw->lock_object, 0, rw->rw_recurse, file, line); - if (rw->rw_recurse) - rw->rw_recurse--; - else - _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line); + + _rw_wunlock_hard(rw, (uintptr_t)curthread, file, line); TD_LOCKS_DEC(curthread); } @@ -802,6 +801,7 @@ __rw_wlock_hard(volatile uintptr_t *c, uintptr_t v, uintptr_t tid, ("%s: recursing but non-recursive rw %s @ %s:%d\n", __func__, rw->lock_object.lo_name, file, line)); rw->rw_recurse++; + atomic_set_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); if (LOCK_LOG_TEST(&rw->lock_object, 0)) CTR2(KTR_LOCK, "%s: %p recursing", __func__, rw); return; @@ -994,12 +994,17 @@ __rw_wunlock_hard(volatile uintptr_t *c, uintptr_t tid, const char *file, return; rw = rwlock2rw(c); - MPASS(!rw_recursed(rw)); - LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, - LOCKSTAT_WRITER); - if (_rw_write_unlock(rw, tid)) + if (!rw_recursed(rw)) { + LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw, + LOCKSTAT_WRITER); + if (_rw_write_unlock(rw, tid)) + return; + } else { + if (--(rw->rw_recurse) == 0) + atomic_clear_ptr(&rw->rw_lock, RW_LOCK_WRITER_RECURSED); return; + } KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), ("%s: neither of the waiter flags are set", __func__)); diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h index d6dd4f078753..aa4b0cc2ab76 100644 --- a/sys/sys/rwlock.h +++ b/sys/sys/rwlock.h @@ -58,13 +58,14 @@ #define RW_LOCK_READ_WAITERS 0x02 #define RW_LOCK_WRITE_WAITERS 0x04 #define RW_LOCK_WRITE_SPINNER 0x08 +#define RW_LOCK_WRITER_RECURSED 0x10 #define RW_LOCK_FLAGMASK \ (RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS | \ - RW_LOCK_WRITE_SPINNER) + RW_LOCK_WRITE_SPINNER | RW_LOCK_WRITER_RECURSED) #define RW_LOCK_WAITERS (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS) #define RW_OWNER(x) ((x) & ~RW_LOCK_FLAGMASK) -#define RW_READERS_SHIFT 4 +#define RW_READERS_SHIFT 5 #define RW_READERS(x) (RW_OWNER((x)) >> RW_READERS_SHIFT) #define RW_READERS_LOCK(x) ((x) << RW_READERS_SHIFT | RW_LOCK_READ) #define RW_ONE_READER (1 << RW_READERS_SHIFT) @@ -111,13 +112,9 @@ #define __rw_wunlock(rw, tid, file, line) do { \ uintptr_t _tid = (uintptr_t)(tid); \ \ - if ((rw)->rw_recurse) \ - (rw)->rw_recurse--; \ - else { \ - if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||\ - !_rw_write_unlock((rw), _tid))) \ - _rw_wunlock_hard((rw), _tid, (file), (line)); \ - } \ + if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) || \ + !_rw_write_unlock((rw), _tid))) \ + _rw_wunlock_hard((rw), _tid, (file), (line)); \ } while (0) /* |