Add appropriate IA32 "pause" instructions to improve performanec on

Pentium 4's and newer IA32 processors.  The "pause" instruction has been
verified by Intel to be a NOP on all currently existing IA32 processors
prior to the Pentium 4.

1 files changed, 17 insertions, 1 deletions

diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index 153b505..e1428b3 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -488,6 +488,9 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line)
 		 */
 		if ((v = m->mtx_lock) == MTX_UNOWNED) {
 			mtx_unlock_spin(&sched_lock);
+#ifdef __i386__
+			cpu_pause();
+#endif
 			continue;
 		}
 
@@ -515,6 +518,9 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line)
 		    !atomic_cmpset_ptr(&m->mtx_lock, (void *)v,
 			(void *)(v | MTX_CONTESTED))) {
 			mtx_unlock_spin(&sched_lock);
+#ifdef __i386__
+			cpu_pause();
+#endif
 			continue;
 		}
 
@@ -527,6 +533,9 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line)
 		if (m != &Giant && owner->td_kse != NULL &&
 		    owner->td_kse->ke_oncpu != NOCPU) {
 			mtx_unlock_spin(&sched_lock);
+#ifdef __i386__
+			cpu_pause();
+#endif
 			continue;
 		}
 #endif	/* SMP && ADAPTIVE_MUTEXES */
@@ -619,8 +628,12 @@ _mtx_lock_spin(struct mtx *m, int opts, const char *file, int line)
 		/* Give interrupts a chance while we spin. */
 		critical_exit();
 		while (m->mtx_lock != MTX_UNOWNED) {
-			if (i++ < 10000000)
+			if (i++ < 10000000) {
+#ifdef __i386__
+				cpu_pause();
+#endif
 				continue;
+			}
 			if (i < 60000000)
 				DELAY(1);
 #ifdef DDB
@@ -630,6 +643,9 @@ _mtx_lock_spin(struct mtx *m, int opts, const char *file, int line)
 #endif
 				panic("spin lock %s held by %p for > 5 seconds",
 				    m->mtx_object.lo_name, (void *)m->mtx_lock);
+#ifdef __i386__
+			cpu_pause();
+#endif
 		}
 		critical_enter();
 	}