diff options
-rw-r--r-- | sys/conf/NOTES | 5 | ||||
-rw-r--r-- | sys/conf/options | 1 | ||||
-rw-r--r-- | sys/kern/kern_mutex.c | 26 | ||||
-rw-r--r-- | sys/kern/subr_turnstile.c | 26 |
4 files changed, 58 insertions, 0 deletions
diff --git a/sys/conf/NOTES b/sys/conf/NOTES index 1d431e3..688de2f 100644 --- a/sys/conf/NOTES +++ b/sys/conf/NOTES @@ -120,6 +120,11 @@ options ROOTDEVNAME=\"ufs:da0s2e\" # Mandatory: options SMP # Symmetric MultiProcessor Kernel +# ADAPTIVE_MUTEXES changes the behavior of blocking mutexes to spin +# if the thread that currently owns the mutex is executing on another +# CPU. +options ADAPTIVE_MUTEXES + # SMP Debugging Options: # # MUTEX_DEBUG enables various extra assertions in the mutex code. diff --git a/sys/conf/options b/sys/conf/options index 14133e2..5caf9b5 100644 --- a/sys/conf/options +++ b/sys/conf/options @@ -57,6 +57,7 @@ ADW_ALLOW_MEMIO opt_adw.h # Allow PCI devices to use memory # Miscellaneous options. GEOM opt_geom.h +ADAPTIVE_MUTEXES COMPAT_43 opt_compat.h COMPAT_SUNOS opt_compat.h COMPILING_LINT opt_global.h diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index 2a98204..a46235f 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -34,6 +34,7 @@ * Machine independent bits of mutex implementation. */ +#include "opt_adaptive_mutexes.h" #include "opt_ddb.h" #include <sys/param.h> @@ -459,6 +460,9 @@ void _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) { struct thread *td = curthread; +#if defined(SMP) && defined(ADAPTIVE_MUTEXES) + struct thread *owner; +#endif if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)td) { m->mtx_recurse++; @@ -514,6 +518,19 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) continue; } +#if defined(SMP) && defined(ADAPTIVE_MUTEXES) + /* + * If the current owner of the lock is executing on another + * CPU, spin instead of blocking. + */ + owner = (struct thread *)(v & MTX_FLAGMASK); + if (m != &Giant && owner->td_kse != NULL && + owner->td_kse->ke_oncpu != NOCPU) { + mtx_unlock_spin(&sched_lock); + continue; + } +#endif /* SMP && ADAPTIVE_MUTEXES */ + /* * We definitely must sleep for this lock. */ @@ -651,6 +668,15 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); td1 = TAILQ_FIRST(&m->mtx_blocked); +#if defined(SMP) && defined(ADAPTIVE_MUTEXES) + if (td1 == NULL) { + _release_lock_quick(m); + if (LOCK_LOG_TEST(&m->mtx_object, opts)) + CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); + mtx_unlock_spin(&sched_lock); + return; + } +#endif MPASS(td->td_proc->p_magic == P_MAGIC); MPASS(td1->td_proc->p_magic == P_MAGIC); diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index 2a98204..a46235f 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -34,6 +34,7 @@ * Machine independent bits of mutex implementation. */ +#include "opt_adaptive_mutexes.h" #include "opt_ddb.h" #include <sys/param.h> @@ -459,6 +460,9 @@ void _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) { struct thread *td = curthread; +#if defined(SMP) && defined(ADAPTIVE_MUTEXES) + struct thread *owner; +#endif if ((m->mtx_lock & MTX_FLAGMASK) == (uintptr_t)td) { m->mtx_recurse++; @@ -514,6 +518,19 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) continue; } +#if defined(SMP) && defined(ADAPTIVE_MUTEXES) + /* + * If the current owner of the lock is executing on another + * CPU, spin instead of blocking. + */ + owner = (struct thread *)(v & MTX_FLAGMASK); + if (m != &Giant && owner->td_kse != NULL && + owner->td_kse->ke_oncpu != NOCPU) { + mtx_unlock_spin(&sched_lock); + continue; + } +#endif /* SMP && ADAPTIVE_MUTEXES */ + /* * We definitely must sleep for this lock. */ @@ -651,6 +668,15 @@ _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line) CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p contested", m); td1 = TAILQ_FIRST(&m->mtx_blocked); +#if defined(SMP) && defined(ADAPTIVE_MUTEXES) + if (td1 == NULL) { + _release_lock_quick(m); + if (LOCK_LOG_TEST(&m->mtx_object, opts)) + CTR1(KTR_LOCK, "_mtx_unlock_sleep: %p no sleepers", m); + mtx_unlock_spin(&sched_lock); + return; + } +#endif MPASS(td->td_proc->p_magic == P_MAGIC); MPASS(td1->td_proc->p_magic == P_MAGIC); |