Add a lwpid field into per-cpu structure, the lwpid represents current

running thread's id on each cpu. This allow us to add in-kernel adaptive
spin for user level mutex. While spinning in user space is possible,
without correct thread running state exported from kernel, it hardly
can be implemented efficiently without wasting cpu cycles, however
exporting thread running state unlikely will be implemented soon as
it has to design and stablize interfaces. This implementation is
transparent to user space, it can be disabled dynamically. With this
change, mutex ping-pong program's performance is improved massively on
SMP machine. performance of mysql super-smack select benchmark is increased
about 7% on Intel dual dual-core2 Xeon machine, it indicates on systems
which have bunch of cpus and system-call overhead is low (athlon64, opteron,
and core-2 are known to be fast), the adaptive spin does help performance.

Added sysctls:
    kern.threads.umtx_dflt_spins
        if the sysctl value is non-zero, a zero umutex.m_spincount will
        cause the sysctl value to be used a spin cycle count.
    kern.threads.umtx_max_spins
        the sysctl sets upper limit of spin cycle count.

Tested on: Athlon64 X2 3800+, Dual Xeon 5130

1 files changed, 70 insertions, 0 deletions

diff --git a/sys/kern/kern_umtx.c b/sys/kern/kern_umtx.c
index 645f02d..a43b4d4 100644
--- a/sys/kern/kern_umtx.c
+++ b/sys/kern/kern_umtx.c
@@ -38,6 +38,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/priv.h>
 #include <sys/proc.h>
 #include <sys/sched.h>
+#include <sys/smp.h>
 #include <sys/sysctl.h>
 #include <sys/sysent.h>
 #include <sys/systm.h>
@@ -51,6 +52,8 @@ __FBSDID("$FreeBSD$");
 #include <vm/vm_map.h>
 #include <vm/vm_object.h>
 
+#include <machine/cpu.h>
+
 #ifdef COMPAT_IA32
 #include <compat/freebsd32/freebsd32_proto.h>
 #endif
@@ -190,6 +193,13 @@ static int			umtx_pi_allocated;
 SYSCTL_NODE(_debug, OID_AUTO, umtx, CTLFLAG_RW, 0, "umtx debug");
 SYSCTL_INT(_debug_umtx, OID_AUTO, umtx_pi_allocated, CTLFLAG_RD,
     &umtx_pi_allocated, 0, "Allocated umtx_pi");
+SYSCTL_DECL(_kern_threads);
+static int			umtx_dflt_spins = 0;
+SYSCTL_INT(_kern_threads, OID_AUTO, umtx_dflt_spins, CTLFLAG_RW,
+    &umtx_dflt_spins, 0, "default umtx spin count");
+static int			umtx_max_spins = 3000;
+SYSCTL_INT(_kern_threads, OID_AUTO, umtx_max_spins, CTLFLAG_RW,
+    &umtx_max_spins, 0, "max umtx spin count");
 
 static void umtxq_sysinit(void *);
 static void umtxq_hash(struct umtx_key *key);
@@ -1012,16 +1022,33 @@ _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 {
 	struct umtx_q *uq;
 	uint32_t owner, old, id;
+#ifdef SMP
+	int spincount;
+#endif
 	int error = 0;
 
 	id = td->td_tid;
 	uq = td->td_umtxq;
 
+#ifdef SMP
+	if (smp_cpus > 1) {
+		spincount = fuword32(&m->m_spincount);
+		if (spincount == 0)
+			spincount = umtx_dflt_spins;
+		if (spincount > umtx_max_spins)
+			spincount = umtx_max_spins;
+	} else
+		spincount = 0;
+#endif
+
 	/*
 	 * Care must be exercised when dealing with umtx structure. It
 	 * can fault on any access.
 	 */
 	for (;;) {
+#ifdef SMP
+try_unowned:
+#endif
 		/*
 		 * Try the uncontested case.  This should be done in userland.
 		 */
@@ -1037,6 +1064,9 @@ _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 
 		/* If no one owns it but it is contested try to acquire it. */
 		if (owner == UMUTEX_CONTESTED) {
+#ifdef SMP
+try_contested:
+#endif
 			owner = casuword32(&m->m_owner,
 			    UMUTEX_CONTESTED, id | UMUTEX_CONTESTED);
 
@@ -1058,6 +1088,46 @@ _do_lock_normal(struct thread *td, struct umutex *m, uint32_t flags, int timo,
 		if (try != 0)
 			return (EBUSY);
 
+#ifdef SMP
+		if (spincount > 0 && (owner & ~UMUTEX_CONTESTED) != id) {
+			int i, found = 0;
+			struct pcpu *pcpu = NULL;
+
+			/* Look for a cpu the owner is running on */
+			for (i = 0; i < MAXCPU; i++) {
+				if (CPU_ABSENT(i))
+					continue;
+				pcpu = pcpu_find(i);
+				if ((owner & ~UMUTEX_CONTESTED) == pcpu->pc_curtid) {
+					found = 1;
+					break;
+				}
+			}
+
+			if (__predict_false(!found))
+				goto end_spin;
+
+			while ((owner & ~UMUTEX_CONTESTED) == pcpu->pc_curtid &&
+			       (owner & ~UMUTEX_CONTESTED) != id) {
+				if (--spincount <= 0)
+					break;
+				if ((td->td_flags &
+			    	    (TDF_NEEDRESCHED|TDF_ASTPENDING|TDF_NEEDSIGCHK)) ||
+				     P_SHOULDSTOP(td->td_proc))
+					break;
+				owner = fuword32(__DEVOLATILE(uint32_t *, &m->m_owner));
+				if (owner == UMUTEX_UNOWNED)
+					goto try_unowned;
+ 				if (owner == UMUTEX_CONTESTED)
+					goto try_contested;
+				cpu_spinwait();
+			}
+		}
+end_spin:
+		spincount = 0;
+
+#endif
+
 		/*
 		 * If we caught a signal, we have retried and now
 		 * exit immediately.