diff options
Diffstat (limited to 'sys')
| -rw-r--r-- | sys/conf/NOTES | 6 | ||||
| -rw-r--r-- | sys/conf/options | 1 | ||||
| -rw-r--r-- | sys/kern/kern_mutex.c | 12 | ||||
| -rw-r--r-- | sys/kern/kern_rwlock.c | 52 |
4 files changed, 44 insertions, 27 deletions
diff --git a/sys/conf/NOTES b/sys/conf/NOTES index 8e2c447..88086cb 100644 --- a/sys/conf/NOTES +++ b/sys/conf/NOTES @@ -202,6 +202,12 @@ options SMP # Symmetric MultiProcessor Kernel # to disable it. options NO_ADAPTIVE_MUTEXES +# ADAPTIVE_RWLOCKS changes the behavior of reader/writer locks to spin +# if the thread that currently owns the rwlock is executing on another +# CPU. This behaviour is enabled by default, so this option can be used +# to disable it. +options NO_ADAPTIVE_RWLOCKS + # ADAPTIVE_GIANT causes the Giant lock to also be made adaptive when # running without NO_ADAPTIVE_MUTEXES. Normally, because Giant is assumed # to be held for extended periods, contention on Giant will cause a thread diff --git a/sys/conf/options b/sys/conf/options index 63de061..0496857 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -124,6 +124,7 @@ MFI_DECODE_LOG opt_mfi.h MPROF_BUFFERS opt_mprof.h MPROF_HASH_SIZE opt_mprof.h MUTEX_WAKE_ALL +NO_ADAPTIVE_RWLOCKS NSWBUF_MIN opt_swap.h PANIC_REBOOT_WAIT_TIME opt_panic.h PPC_DEBUG opt_ppc.h diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 987e5ec..9cff711 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -81,6 +81,10 @@ __FBSDID("$FreeBSD$"); #define MUTEX_WAKE_ALL #endif +#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) +#define ADAPTIVE_MUTEXES +#endif + /* * Internal utility macros. */ @@ -299,7 +303,7 @@ void _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, int line) { -#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) +#ifdef ADAPTIVE_MUTEXES volatile struct thread *owner; #endif #ifdef KTR @@ -365,7 +369,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, continue; } -#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) +#ifdef ADAPTIVE_MUTEXES /* * If the current owner of the lock is executing on another * CPU, spin instead of blocking. @@ -383,7 +387,7 @@ _mtx_lock_sleep(struct mtx *m, uintptr_t tid, int opts, const char *file, } continue; } -#endif /* SMP && !NO_ADAPTIVE_MUTEXES */ +#endif /* ADAPTIVE_MUTEXES */ /* * We definitely must sleep for this lock. @@ -498,7 +502,7 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) if (LOCK_LOG_TEST(&m->lock_object, opts)) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); -#if defined(SMP) && !defined(NO_ADAPTIVE_MUTEXES) +#ifdef ADAPTIVE_MUTEXES if (ts == NULL) { _release_lock_quick(m); if (LOCK_LOG_TEST(&m->lock_object, opts)) diff --git a/sys/kern/kern_rwlock.c b/sys/kern/kern_rwlock.c index 8f96e9b..f61da35 100644 --- a/sys/kern/kern_rwlock.c +++ b/sys/kern/kern_rwlock.c @@ -35,6 +35,7 @@ __FBSDID("$FreeBSD$"); #include "opt_ddb.h" +#include "opt_no_adaptive_rwlocks.h" #include <sys/param.h> #include <sys/ktr.h> @@ -47,6 +48,10 @@ __FBSDID("$FreeBSD$"); #include <sys/lock_profile.h> #include <machine/cpu.h> +#if defined(SMP) && !defined(NO_ADAPTIVE_RWLOCKS) +#define ADAPTIVE_RWLOCKS +#endif + #ifdef DDB #include <ddb/ddb.h> @@ -179,7 +184,7 @@ _rw_wunlock(struct rwlock *rw, const char *file, int line) void _rw_rlock(struct rwlock *rw, const char *file, int line) { -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS volatile struct thread *owner; #endif uint64_t waittime = 0; @@ -278,7 +283,7 @@ _rw_rlock(struct rwlock *rw, const char *file, int line) __func__, rw); } -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS /* * If the owner is running on another CPU, spin until * the owner stops running or the state of the lock @@ -446,7 +451,7 @@ _rw_runlock(struct rwlock *rw, const char *file, int line) void _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) { -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS volatile struct thread *owner; #endif uintptr_t v; @@ -510,7 +515,7 @@ _rw_wlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) __func__, rw); } -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS /* * If the lock is write locked and the owner is * running on another CPU, spin until the owner stops @@ -565,7 +570,7 @@ _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) turnstile_lock(&rw->lock_object); ts = turnstile_lookup(&rw->lock_object); -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS /* * There might not be a turnstile for this lock if all of * the waiters are adaptively spinning. In that case, just @@ -598,16 +603,16 @@ _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) * there that could be worked around either by waking both queues * of waiters or doing some complicated lock handoff gymnastics. * - * Note that in the SMP case, if both flags are set, there might - * not be any actual writers on the turnstile as they might all - * be spinning. In that case, we don't want to preserve the - * RW_LOCK_WRITE_WAITERS flag as the turnstile is going to go - * away once we wakeup all the readers. + * Note that in the ADAPTIVE_RWLOCKS case, if both flags are + * set, there might not be any actual writers on the turnstile + * as they might all be spinning. In that case, we don't want + * to preserve the RW_LOCK_WRITE_WAITERS flag as the turnstile + * is going to go away once we wakeup all the readers. */ v = RW_UNLOCKED; if (rw->rw_lock & RW_LOCK_READ_WAITERS) { queue = TS_SHARED_QUEUE; -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS if (rw->rw_lock & RW_LOCK_WRITE_WAITERS && !turnstile_empty(ts, TS_EXCLUSIVE_QUEUE)) v |= RW_LOCK_WRITE_WAITERS; @@ -617,7 +622,7 @@ _rw_wunlock_hard(struct rwlock *rw, uintptr_t tid, const char *file, int line) } else queue = TS_EXCLUSIVE_QUEUE; -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS /* * We have to make sure that we actually have waiters to * wakeup. If they are all spinning, then we just need to @@ -678,14 +683,15 @@ _rw_try_upgrade(struct rwlock *rw, const char *file, int line) * Try to switch from one reader to a writer again. This time * we honor the current state of the RW_LOCK_WRITE_WAITERS * flag. If we obtain the lock with the flag set, then claim - * ownership of the turnstile. In the SMP case it is possible - * for there to not be an associated turnstile even though there - * are waiters if all of the waiters are spinning. + * ownership of the turnstile. In the ADAPTIVE_RWLOCKS case + * it is possible for there to not be an associated turnstile + * even though there are waiters if all of the waiters are + * spinning. */ v = rw->rw_lock & RW_LOCK_WRITE_WAITERS; success = atomic_cmpset_acq_ptr(&rw->rw_lock, RW_READERS_LOCK(1) | v, tid | v); -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS if (success && v && turnstile_lookup(&rw->lock_object) != NULL) #else if (success && v) @@ -737,14 +743,14 @@ _rw_downgrade(struct rwlock *rw, const char *file, int line) * RW_LOCK_WRITE_WAITERS and give up ownership of the * turnstile. If there are any read waiters, wake them up. * - * For SMP, we have to allow for the fact that all of the - * read waiters might be spinning. In that case, act as if - * RW_LOCK_READ_WAITERS is not set. Also, only preserve - * the RW_LOCK_WRITE_WAITERS flag if at least one writer is - * blocked on the turnstile. + * For ADAPTIVE_RWLOCKS, we have to allow for the fact that + * all of the read waiters might be spinning. In that case, + * act as if RW_LOCK_READ_WAITERS is not set. Also, only + * preserve the RW_LOCK_WRITE_WAITERS flag if at least one + * writer is blocked on the turnstile. */ ts = turnstile_lookup(&rw->lock_object); -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS if (ts == NULL) v &= ~(RW_LOCK_READ_WAITERS | RW_LOCK_WRITE_WAITERS); else if (v & RW_LOCK_READ_WAITERS && @@ -762,7 +768,7 @@ _rw_downgrade(struct rwlock *rw, const char *file, int line) (v & RW_LOCK_WRITE_WAITERS)); if (v & RW_LOCK_READ_WAITERS) turnstile_unpend(ts, TS_EXCLUSIVE_LOCK); -#ifdef SMP +#ifdef ADAPTIVE_RWLOCKS else if (ts == NULL) turnstile_release(&rw->lock_object); #endif |
