diff options
| -rw-r--r-- | sys/conf/files | 1 | ||||
| -rw-r--r-- | sys/conf/options | 1 | ||||
| -rw-r--r-- | sys/kern/kern_rwlock.c | 587 | ||||
| -rw-r--r-- | sys/kern/subr_lock.c | 1 | ||||
| -rw-r--r-- | sys/sys/lock.h | 1 | ||||
| -rw-r--r-- | sys/sys/rwlock.h | 192 |
6 files changed, 783 insertions, 0 deletions
diff --git a/sys/conf/files b/sys/conf/files index 80cc86fd12fe..b8832bfac782 100644 --- a/sys/conf/files +++ b/sys/conf/files @@ -1268,6 +1268,7 @@ kern/kern_poll.c optional device_polling kern/kern_proc.c standard kern/kern_prot.c standard kern/kern_resource.c standard +kern/kern_rwlock.c standard kern/kern_sema.c standard kern/kern_shutdown.c standard kern/kern_sig.c standard diff --git a/sys/conf/options b/sys/conf/options index 41b67d1c2146..494256fb369f 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -513,6 +513,7 @@ MUTEX_PROFILING opt_global.h MSIZE opt_global.h REGRESSION opt_global.h RESTARTABLE_PANICS opt_global.h +RWLOCK_NOINLINE opt_global.h VFS_BIO_DEBUG opt_global.h # These are VM related options diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c new file mode 100644 index 000000000000..448cacfd68b0 --- /dev/null +++ b/sys/kern/kern_rwlock.c @@ -0,0 +1,587 @@ +/*- + * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of any co-contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + */ + +/* + * Machine independent bits of reader/writer lock implementation. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include "opt_ddb.h" + +#include <sys/param.h> +#include <sys/ktr.h> +#include <sys/lock.h> +#include <sys/mutex.h> +#include <sys/proc.h> +#include <sys/rwlock.h> +#include <sys/systm.h> +#include <sys/turnstile.h> + +#include <machine/cpu.h> + +#ifdef DDB +#include <ddb/ddb.h> + +static void db_show_rwlock(struct lock_object *lock); +#endif + +struct lock_class lock_class_rw = { + "rw", + LC_SLEEPLOCK | LC_RECURSABLE /* | LC_UPGRADABLE */, +#ifdef DDB + db_show_rwlock +#endif +}; + +#define rw_owner(rw) \ + ((rw)->rw_lock & RW_LOCK_READ ? NULL : \ + (struct thread *)RW_OWNER((rw)->rw_lock)) + +#ifndef INVARIANTS +#define _rw_assert(rw, what, file, line) +#endif + +void +rw_init(struct rwlock *rw, const char *name) +{ + + rw->rw_lock = RW_UNLOCKED; + + lock_init(&rw->rw_object, &lock_class_rw, name, NULL, LO_WITNESS | + LO_RECURSABLE /* | LO_UPGRADABLE */); +} + +void +rw_destroy(struct rwlock *rw) +{ + + KASSERT(rw->rw_lock == RW_UNLOCKED, ("rw lock not unlocked")); + lock_destroy(&rw->rw_object); +} + +void +rw_sysinit(void *arg) +{ + struct rw_args *args = arg; + + rw_init(args->ra_rw, args->ra_desc); +} + +void +_rw_wlock(struct rwlock *rw, const char *file, int line) +{ + + MPASS(curthread != NULL); + KASSERT(rw_owner(rw) != curthread, + ("%s (%s): wlock already held @ %s:%d", __func__, + rw->rw_object.lo_name, file, line)); + WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER | LOP_EXCLUSIVE, file, + line); + __rw_wlock(rw, curthread, file, line); + LOCK_LOG_LOCK("WLOCK", &rw->rw_object, 0, 0, file, line); + WITNESS_LOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); +} + +void +_rw_wunlock(struct rwlock *rw, const char *file, int line) +{ + + MPASS(curthread != NULL); + _rw_assert(rw, RA_WLOCKED, file, line); + WITNESS_UNLOCK(&rw->rw_object, LOP_EXCLUSIVE, file, line); + LOCK_LOG_LOCK("WUNLOCK", &rw->rw_object, 0, 0, file, line); + __rw_wunlock(rw, curthread, file, line); +} + +void +_rw_rlock(struct rwlock *rw, const char *file, int line) +{ + uintptr_t x; + + KASSERT(rw_owner(rw) != curthread, + ("%s (%s): wlock already held @ %s:%d", __func__, + rw->rw_object.lo_name, file, line)); + WITNESS_CHECKORDER(&rw->rw_object, LOP_NEWORDER, file, line); + + /* + * Note that we don't make any attempt to try to block read + * locks once a writer has blocked on the lock. The reason is + * that we currently allow for read locks to recurse and we + * don't keep track of all the holders of read locks. Thus, if + * we were to block readers once a writer blocked and a reader + * tried to recurse on their reader lock after a writer had + * blocked we would end up in a deadlock since the reader would + * be blocked on the writer, and the writer would be blocked + * waiting for the reader to release its original read lock. + */ + for (;;) { + /* + * Handle the easy case. If no other thread has a write + * lock, then try to bump up the count of read locks. Note + * that we have to preserve the current state of the + * RW_LOCK_WRITE_WAITERS flag. If we fail to acquire a + * read lock, then rw_lock must have changed, so restart + * the loop. Note that this handles the case of a + * completely unlocked rwlock since such a lock is encoded + * as a read lock with no waiters. + */ + x = rw->rw_lock; + if (x & RW_LOCK_READ) { + + /* + * The RW_LOCK_READ_WAITERS flag should only be set + * if another thread currently holds a write lock, + * and in that case RW_LOCK_READ should be clear. + */ + MPASS((x & RW_LOCK_READ_WAITERS) == 0); + if (atomic_cmpset_acq_ptr(&rw->rw_lock, x, + x + RW_ONE_READER)) { + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR4(KTR_LOCK, + "%s: %p succeed %p -> %p", __func__, + rw, (void *)x, + (void *)(x + RW_ONE_READER)); + break; + } + continue; + } + + /* + * Okay, now it's the hard case. Some other thread already + * has a write lock, so acquire the turnstile lock so we can + * begin the process of blocking. + */ + turnstile_lock(&rw->rw_object); + + /* + * The lock might have been released while we spun, so + * recheck its state and restart the loop if there is no + * longer a write lock. + */ + x = rw->rw_lock; + if (x & RW_LOCK_READ) { + turnstile_release(&rw->rw_object); + continue; + } + + /* + * Ok, it's still a write lock. If the RW_LOCK_READ_WAITERS + * flag is already set, then we can go ahead and block. If + * it is not set then try to set it. If we fail to set it + * drop the turnstile lock and restart the loop. + */ + if (!(x & RW_LOCK_READ_WAITERS) && + !atomic_cmpset_ptr(&rw->rw_lock, x, + x | RW_LOCK_READ_WAITERS)) { + turnstile_release(&rw->rw_object); + continue; + } + if (!(x & RW_LOCK_READ_WAITERS) && + LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p set read waiters flag", __func__, + rw); + + /* + * We were unable to acquire the lock and the read waiters + * flag is set, so we must block on the turnstile. + */ + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, + rw); + turnstile_wait(&rw->rw_object, rw_owner(rw), TS_SHARED_QUEUE); + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p resuming from turnstile", + __func__, rw); + } + + /* + * TODO: acquire "owner of record" here. Here be turnstile dragons + * however. turnstiles don't like owners changing between calls to + * turnstile_wait() currently. + */ + + LOCK_LOG_LOCK("RLOCK", &rw->rw_object, 0, 0, file, line); + WITNESS_LOCK(&rw->rw_object, 0, file, line); +} + +void +_rw_runlock(struct rwlock *rw, const char *file, int line) +{ + struct turnstile *ts; + uintptr_t x; + + _rw_assert(rw, RA_RLOCKED, file, line); + WITNESS_UNLOCK(&rw->rw_object, 0, file, line); + LOCK_LOG_LOCK("RUNLOCK", &rw->rw_object, 0, 0, file, line); + + /* TODO: drop "owner of record" here. */ + + for (;;) { + /* + * See if there is more than one read lock held. If so, + * just drop one and return. + */ + x = rw->rw_lock; + if (RW_READERS(x) > 1) { + if (atomic_cmpset_ptr(&rw->rw_lock, x, + x - RW_ONE_READER)) { + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR4(KTR_LOCK, + "%s: %p succeeded %p -> %p", + __func__, rw, (void *)x, + (void *)(x - RW_ONE_READER)); + break; + } + continue; + } + + /* + * We should never have read waiters while at least one + * thread holds a read lock. (See note above) + */ + KASSERT(!(x & RW_LOCK_READ_WAITERS), + ("%s: waiting readers", __func__)); + + /* + * If there aren't any waiters for a write lock, then try + * to drop it quickly. + */ + if (!(x & RW_LOCK_WRITE_WAITERS)) { + + /* + * There shouldn't be any flags set and we should + * be the only read lock. If we fail to release + * the single read lock, then another thread might + * have just acquired a read lock, so go back up + * to the multiple read locks case. + */ + MPASS(x == RW_READERS_LOCK(1)); + if (atomic_cmpset_ptr(&rw->rw_lock, RW_READERS_LOCK(1), + RW_UNLOCKED)) { + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p last succeeded", + __func__, rw); + break; + } + continue; + } + + /* + * There should just be one reader with one or more + * writers waiting. + */ + MPASS(x == (RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS)); + + /* + * Ok, we know we have a waiting writer and we think we + * are the last reader, so grab the turnstile lock. + */ + turnstile_lock(&rw->rw_object); + + /* + * Try to drop our lock leaving the lock in a unlocked + * state. + * + * If you wanted to do explicit lock handoff you'd have to + * do it here. You'd also want to use turnstile_signal() + * and you'd have to handle the race where a higher + * priority thread blocks on the write lock before the + * thread you wakeup actually runs and have the new thread + * "steal" the lock. For now it's a lot simpler to just + * wakeup all of the waiters. + * + * As above, if we fail, then another thread might have + * acquired a read lock, so drop the turnstile lock and + * restart. + */ + if (!atomic_cmpset_ptr(&rw->rw_lock, + RW_READERS_LOCK(1) | RW_LOCK_WRITE_WAITERS, RW_UNLOCKED)) { + turnstile_release(&rw->rw_object); + continue; + } + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p last succeeded with waiters", + __func__, rw); + + /* + * Ok. The lock is released and all that's left is to + * wake up the waiters. Note that the lock might not be + * free anymore, but in that case the writers will just + * block again if they run before the new lock holder(s) + * release the lock. + */ + ts = turnstile_lookup(&rw->rw_object); + turnstile_broadcast(ts, TS_EXCLUSIVE_QUEUE); + turnstile_unpend(ts, TS_SHARED_LOCK); + break; + } +} + +/* + * This function is called when we are unable to obtain a write lock on the + * first try. This means that at least one other thread holds either a + * read or write lock. + */ +void +_rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) +{ + uintptr_t v; + + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR5(KTR_LOCK, "%s: %s contested (lock=%p) at %s:%d", __func__, + rw->rw_object.lo_name, (void *)rw->rw_lock, file, line); + + while (!_rw_write_lock(rw, tid)) { + turnstile_lock(&rw->rw_object); + v = rw->rw_lock; + + /* + * If the lock was released while spinning on the + * turnstile chain lock, try again. + */ + if (v == RW_UNLOCKED) { + turnstile_release(&rw->rw_object); + cpu_spinwait(); + continue; + } + + /* + * If the lock was released by a writer with both readers + * and writers waiting and a reader hasn't woken up and + * acquired the lock yet, rw_lock will be set to the + * value RW_UNLOCKED | RW_LOCK_WRITE_WAITERS. If we see + * that value, try to acquire it once. Note that we have + * to preserve the RW_LOCK_WRITE_WAITERS flag as there are + * other writers waiting still. If we fail, restart the + * loop. + */ + if (v == (RW_UNLOCKED | RW_LOCK_WRITE_WAITERS)) { + if (atomic_cmpset_acq_ptr(&rw->rw_lock, + RW_UNLOCKED | RW_LOCK_WRITE_WAITERS, + tid | RW_LOCK_WRITE_WAITERS)) { + turnstile_claim(&rw->rw_object); + CTR2(KTR_LOCK, "%s: %p claimed by new writer", + __func__, rw); + break; + } + turnstile_release(&rw->rw_object); + cpu_spinwait(); + continue; + } + + /* + * If the RW_LOCK_WRITE_WAITERS flag isn't set, then try to + * set it. If we fail to set it, then loop back and try + * again. + */ + if (!(v & RW_LOCK_WRITE_WAITERS) && + !atomic_cmpset_ptr(&rw->rw_lock, v, + v | RW_LOCK_WRITE_WAITERS)) { + turnstile_release(&rw->rw_object); + cpu_spinwait(); + continue; + } + if (!(v & RW_LOCK_WRITE_WAITERS) && + LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p set write waiters flag", + __func__, rw); + + /* XXX: Adaptively spin if current wlock owner on another CPU? */ + + /* + * We were unable to acquire the lock and the write waiters + * flag is set, so we must block on the turnstile. + */ + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p blocking on turnstile", __func__, + rw); + turnstile_wait(&rw->rw_object, rw_owner(rw), + TS_EXCLUSIVE_QUEUE); + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p resuming from turnstile", + __func__, rw); + } +} + +/* + * This function is called if the first try at releasing a write lock failed. + * This means that one of the 2 waiter bits must be set indicating that at + * least one thread is waiting on this lock. + */ +void +_rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) +{ + struct turnstile *ts; + uintptr_t v; + int queue; + + KASSERT(rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS), + ("%s: neither of the waiter flags are set", __func__)); + + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR2(KTR_LOCK, "%s: %p contested", __func__, rw); + + turnstile_lock(&rw->rw_object); + ts = turnstile_lookup(&rw->rw_object); + + /* XXX: Adaptive fixup would be required here. */ + MPASS(ts != NULL); + + /* + * Use the same algo as sx locks for now. Prefer waking up shared + * waiters if we have any over writers. This is probably not ideal. + * + * 'v' is the value we are going to write back to rw_lock. If we + * have waiters on both queues, we need to preserve the state of + * the waiter flag for the queue we don't wake up. For now this is + * hardcoded for the algorithm mentioned above. + * + * In the case of both readers and writers waiting we wakeup the + * readers but leave the RW_LOCK_WRITE_WAITERS flag set. If a + * new writer comes in before a reader it will claim the lock up + * above. There is probably a potential priority inversion in + * there that could be worked around either by waking both queues + * of waiters or doing some complicated lock handoff gymnastics. + */ + if (rw->rw_lock & RW_LOCK_READ_WAITERS) { + queue = TS_SHARED_QUEUE; + v = RW_UNLOCKED | (rw->rw_lock & RW_LOCK_WRITE_WAITERS); + } else { + queue = TS_EXCLUSIVE_QUEUE; + v = RW_UNLOCKED; + } + if (LOCK_LOG_TEST(&rw->rw_object, 0)) + CTR3(KTR_LOCK, "%s: %p waking up %s waiters", __func__, rw, + queue == TS_SHARED_QUEUE ? "read" : "write"); + + /* Wake up all waiters for the specific queue. */ + turnstile_broadcast(ts, queue); + atomic_store_rel_ptr(&rw->rw_lock, v); + turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); +} + +#ifdef INVARIANT_SUPPORT +#ifndef INVARIANT_SUPPORT +#undef _rw_assert +#endif + +/* + * In the non-WITNESS case, rw_assert() can only detect that at least + * *some* thread owns an rlock, but it cannot guarantee that *this* + * thread owns an rlock. + */ +void +_rw_assert(struct rwlock *rw, int what, const char *file, int line) +{ + + if (panicstr != NULL) + return; + switch (what) { + case RA_LOCKED: + case RA_RLOCKED: +#ifdef WITNESS + witness_assert(&rw->rw_object, what, file, line); +#else + /* + * If some other thread has a write lock or we have one + * and are asserting a read lock, fail. Also, if no one + * has a lock at all, fail. + */ + if (rw->rw_lock == RW_UNLOCKED || + !(rw->rw_lock & RW_LOCK_READ) && (what == RW_RLOCKED || + RW_OWNER(rw) != (uintptr_t)curthread)) + panic("Lock %s not %slocked @ %s:%d\n", + rw->rw_object.lo_name, (what == RW_RLOCKED) ? + "read " : "", file, line); +#endif + break; + case RA_WLOCKED: + if (rw_owner(rw) != curthread) + panic("Lock %s not exclusively locked @ %s:%d\n", + rw->rw_object.lo_name, file, line); + break; + case RA_UNLOCKED: +#ifdef WITNESS + witness_assert(&rw->rw_object, what, file, line); +#else + /* + * If we hold a write lock fail. We can't reliably check + * to see if we hold a read lock or not. + */ + if (rw_owner(rw) == curthread) + panic("Lock %s exclusively locked @ %s:%d\n", + rw->rw_object.lo_name, file, line); +#endif + break; + default: + panic("Unknown rw lock assertion: %d @ %s:%d", what, file, + line); + } +} +#endif /* INVARIANT_SUPPORT */ + +#ifdef DDB +void +db_show_rwlock(struct lock_object *lock) +{ + struct rwlock *rw; + struct thread *td; + + rw = (struct rwlock *)lock; + + db_printf(" state: "); + if (rw->rw_lock == RW_UNLOCKED) + db_printf("UNLOCKED\n"); + else if (rw->rw_lock & RW_LOCK_READ) + db_printf("RLOCK: %d locks\n", RW_READERS(rw->rw_lock)); + else { + td = rw_owner(rw); + db_printf("WLOCK: %p (tid %d, pid %d, \"%s\")\n", td, + td->td_tid, td->td_proc->p_pid, td->td_proc->p_comm); + } + db_printf(" waiters: "); + switch (rw->rw_lock & (RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS)) { + case RW_LOCK_READ_WAITERS: + db_printf("readers\n"); + break; + case RW_LOCK_WRITE_WAITERS: + db_printf("writers\n"); + break; + case RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS: + db_printf("readers and waiters\n"); + break; + default: + db_printf("none\n"); + break; + } +} + +#endif diff --git a/sys/kern/subr_lock.c b/sys/kern/subr_lock.c index cb7bc768380c..ecf300ac3bef 100644 --- a/sys/kern/subr_lock.c +++ b/sys/kern/subr_lock.c @@ -53,6 +53,7 @@ struct lock_class *lock_classes[LOCK_CLASS_MAX + 1] = { &lock_class_mtx_spin, &lock_class_mtx_sleep, &lock_class_sx, + &lock_class_rw, }; void diff --git a/sys/sys/lock.h b/sys/sys/lock.h index 6fcb03312063..99a7e5d3f001 100644 --- a/sys/sys/lock.h +++ b/sys/sys/lock.h @@ -211,6 +211,7 @@ struct lock_list_entry { extern struct lock_class lock_class_mtx_sleep; extern struct lock_class lock_class_mtx_spin; extern struct lock_class lock_class_sx; +extern struct lock_class lock_class_rw; extern struct lock_class *lock_classes[]; diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h new file mode 100644 index 000000000000..38d1b5abeebb --- /dev/null +++ b/sys/sys/rwlock.h @@ -0,0 +1,192 @@ +/*- + * Copyright (c) 2006 John Baldwin <jhb@FreeBSD.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Neither the name of the author nor the names of any co-contributors + * may be used to endorse or promote products derived from this software + * without specific prior written permission. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _SYS_RWLOCK_H_ +#define _SYS_RWLOCK_H_ + +#include <sys/_lock.h> +#include <sys/_rwlock.h> + +#ifdef _KERNEL +#include <machine/atomic.h> +#endif + +/* + * The rw_lock field consists of several fields. The low bit indicates + * if the lock is locked with a read (shared) or write (exclusive) lock. + * A value of 0 indicates a write lock, and a value of 1 indicates a read + * lock. Bit 1 is a boolean indicating if there are any threads waiting + * for a read lock. Bit 2 is a boolean indicating if there are any threads + * waiting for a write lock. The rest of the variable's definition is + * dependent on the value of the first bit. For a write lock, it is a + * pointer to the thread holding the lock, similar to the mtx_lock field of + * mutexes. For read locks, it is a count of read locks that are held. + * + * When the lock is not locked by any thread, it is encoded as a read lock + * with zero waiters. + * + * A note about memory barriers. Write locks need to use the same memory + * barriers as mutexes: _acq when acquiring a write lock and _rel when + * releasing a write lock. Read locks also need to use an _acq barrier when + * acquiring a read lock. However, since read locks do not update any + * locked data (modulo bugs of course), no memory barrier is needed to when + * releasing a read lock. + */ + +#define RW_LOCK_READ 0x01 +#define RW_LOCK_READ_WAITERS 0x02 +#define RW_LOCK_WRITE_WAITERS 0x04 +#define RW_LOCK_FLAGMASK \ + (RW_LOCK_READ | RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS) + +#define RW_OWNER(x) ((x) & ~RW_LOCK_FLAGMASK) +#define RW_READERS_SHIFT 3 +#define RW_READERS(x) (RW_OWNER((x)) >> RW_READERS_SHIFT) +#define RW_READERS_LOCK(x) ((x) << RW_READERS_SHIFT | RW_LOCK_READ) +#define RW_ONE_READER (1 << RW_READERS_SHIFT) + +#define RW_UNLOCKED RW_READERS_LOCK(0) + +#ifdef _KERNEL + +/* Very simple operations on rw_lock. */ + +/* Try to obtain a write lock once. */ +#define _rw_write_lock(rw, tid) \ + atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid)) + +/* Release a write lock quickly if there are no waiters. */ +#define _rw_write_unlock(rw, tid) \ + atomic_cmpset_rel_ptr(&(rw)->rw_lock, (tid), RW_UNLOCKED) + +/* + * Full lock operations that are suitable to be inlined in non-debug + * kernels. If the lock cannot be acquired or released trivially then + * the work is deferred to another function. + */ + +/* Acquire a write lock. */ +#define __rw_wlock(rw, tid, file, line) do { \ + uintptr_t _tid = (uintptr_t)(tid); \ + \ + if (!_rw_write_lock((rw), _tid)) \ + _rw_wlock_hard((rw), _tid, (file), (line)); \ +} while (0) + +/* Release a write lock. */ +#define __rw_wunlock(rw, tid, file, line) do { \ + uintptr_t _tid = (uintptr_t)(tid); \ + \ + if (!_rw_write_unlock((rw), _tid)) \ + _rw_wunlock_hard((rw), _tid, (file), (line)); \ +} while (0) + +/* + * Function prototypes. Routines that start with _ are not part of the + * external API and should not be called directly. Wrapper macros should + * be used instead. + */ + +void rw_init(struct rwlock *rw, const char *name); +void rw_destroy(struct rwlock *rw); +void rw_sysinit(void *arg); +void _rw_wlock(struct rwlock *rw, const char *file, int line); +void _rw_wunlock(struct rwlock *rw, const char *file, int line); +void _rw_rlock(struct rwlock *rw, const char *file, int line); +void _rw_runlock(struct rwlock *rw, const char *file, int line); +void _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, + int line); +void _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, + int line); +#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) +void _rw_assert(struct rwlock *rw, int what, const char *file, int line); +#endif + +/* + * Public interface for lock operations. + * + * XXX: Missing try and upgrade/downgrade. + */ + +#ifndef LOCK_DEBUG +#error LOCK_DEBUG not defined, include <sys/lock.h> before <sys/rwlock.h> +#endif +#if LOCK_DEBUG > 0 || defined(RWLOCK_NOINLINE) +#define rw_wlock(rw) _rw_wlock((rw), LOCK_FILE, LOCK_LINE) +#define rw_wunlock(rw) _rw_wunlock((rw), LOCK_FILE, LOCK_LINE) +#else +#define rw_wlock(rw) \ + __rw_wlock((rw), curthread, LOCK_FILE, LOCK_LINE) +#define rw_wunlock(rw) \ + __rw_wunlock((rw), curthread, LOCK_FILE, LOCK_LINE) +#endif +#define rw_rlock(rw) _rw_rlock((rw), LOCK_FILE, LOCK_LINE) +#define rw_runlock(rw) _rw_runlock((rw), LOCK_FILE, LOCK_LINE) + +#define rw_initialized(rw) lock_initalized(&(rw)->rw_object) + +struct rw_args { + struct rwlock *ra_rw; + const char *ra_desc; +}; + +#define RW_SYSINIT(name, rw, desc) \ + static struct rw_args name##_args = { \ + (rw), \ + (desc), \ + }; \ + SYSINIT(name##_rw_sysinit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ + rw_sysinit, &name##_args); \ + SYSUNINIT(name##_mtx_sysuninit, SI_SUB_LOCK, SI_ORDER_MIDDLE, \ + rw_destroy, (rw)) + +/* + * The INVARIANTS-enabled rw_assert() functionality. + * + * The constants need to be defined for INVARIANT_SUPPORT infrastructure + * support as _rw_assert() itself uses them and the latter implies that + * _rw_assert() must build. + */ +#if defined(INVARIANTS) || defined(INVARIANT_SUPPORT) +#define RA_LOCKED LA_LOCKED +#define RA_RLOCKED LA_SLOCKED +#define RA_WLOCKED LA_XLOCKED +#define RA_UNLOCKED LA_UNLOCKED +#endif + +#ifdef INVARIANTS +#define rw_assert(rw, what) _rw_assert((sx), (what), LOCK_FILE, LOCK_LINE) +#else +#define rw_assert(rw, what) +#endif + +#endif /* _KERNEL */ +#endif /* !_SYS_RWLOCK_H_ */ |