Add IPI support for preempting a thread on another CPU.

MFC after:	3 weeks

6 files changed, 40 insertions, 2 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 3bf6f31..c6f5a29 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -30,6 +30,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_cpu.h"
 #include "opt_kstack_pages.h"
 #include "opt_mp_watchdog.h"
+#include "opt_sched.h"
 
 #include <sys/param.h>
 #include <sys/systm.h>
@@ -956,6 +957,21 @@ ipi_bitmap_handler(struct clockframe frame)
 
 	ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
 
+#ifdef IPI_PREEMPTION
+	if (ipi_bitmap & IPI_PREEMPT) {
+		mtx_lock_spin(&sched_lock);
+		/* Don't preempt the idle thread */
+		if (curthread->td_priority <  PRI_MIN_IDLE) {
+			struct thread *running_thread = curthread;
+			if (running_thread->td_critnest > 1) 
+				running_thread->td_owepreempt = 1;
+			else 		
+				mi_switch(SW_INVOL | SW_PREEMPT, NULL);
+		}
+		mtx_unlock_spin(&sched_lock);
+	}
+#endif
+
 	/* Nothing to do for AST */
 }
 
diff --git a/sys/amd64/include/apicvar.h b/sys/amd64/include/apicvar.h
index 9daa0cf..eb05c6a 100644
--- a/sys/amd64/include/apicvar.h
+++ b/sys/amd64/include/apicvar.h
@@ -123,7 +123,8 @@
 
 /* IPIs handled by IPI_BITMAPED_VECTOR  (XXX ups is there a better place?) */
 #define	IPI_AST		0 	/* Generate software trap. */
-#define IPI_BITMAP_LAST IPI_AST
+#define IPI_PREEMPT     1
+#define IPI_BITMAP_LAST IPI_PREEMPT
 #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST)
 
 #define	IPI_STOP	(APIC_IPI_INTS + 6)	/* Stop CPU until restarted. */
diff --git a/sys/conf/NOTES b/sys/conf/NOTES
index a8dd7c7..d59901e 100644
--- a/sys/conf/NOTES
+++ b/sys/conf/NOTES
@@ -216,6 +216,8 @@ options 	MUTEX_WAKE_ALL
 #	  performance and increase the frequency of kernel panics by
 #	  design.  If you aren't sure that you need it then you don't.
 #	  Relies on the PREEMPTION option.  DON'T TURN THIS ON.
+# IPI_PREEMPTION instructs the kernel to preempt threads running on other
+#         CPUS if needed.  Relies on the PREEMPTION option
 # MUTEX_DEBUG enables various extra assertions in the mutex code.
 # SLEEPQUEUE_PROFILING enables rudimentary profiling of the hash table
 #	  used to hold active sleep queues.
@@ -229,6 +231,7 @@ options 	MUTEX_WAKE_ALL
 # WITNESS_SKIPSPIN disables the witness checks on spin mutexes.
 options 	PREEMPTION
 options 	FULL_PREEMPTION
+options		IPI_PREEMPTION
 options 	MUTEX_DEBUG
 options 	WITNESS
 options 	WITNESS_KDB
diff --git a/sys/conf/options b/sys/conf/options
index ba322b6..7fb5e06 100644
--- a/sys/conf/options
+++ b/sys/conf/options
@@ -69,6 +69,7 @@ CY_PCI_FASTINTR
 DIRECTIO	opt_directio.h
 FULL_PREEMPTION	opt_sched.h
 PREEMPTION	opt_sched.h
+IPI_PREEMPTION	opt_sched.h
 GEOM_AES	opt_geom.h
 GEOM_APPLE	opt_geom.h
 GEOM_BDE	opt_geom.h
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index e2812d6..b3404d0 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -30,6 +30,7 @@ __FBSDID("$FreeBSD$");
 #include "opt_cpu.h"
 #include "opt_kstack_pages.h"
 #include "opt_mp_watchdog.h"
+#include "opt_sched.h"
 
 #if !defined(lint)
 #if !defined(SMP)
@@ -1178,6 +1179,21 @@ ipi_bitmap_handler(struct clockframe frame)
 
 	ipi_bitmap = atomic_readandclear_int(&cpu_ipi_pending[cpu]);
 
+#ifdef IPI_PREEMPTION
+	if (ipi_bitmap & IPI_PREEMPT) {
+		mtx_lock_spin(&sched_lock);
+		/* Don't preempt the idle thread */
+		if (curthread->td_priority <  PRI_MIN_IDLE) {
+			struct thread *running_thread = curthread;
+			if (running_thread->td_critnest > 1) 
+				running_thread->td_owepreempt = 1;
+			else 		
+				mi_switch(SW_INVOL | SW_PREEMPT, NULL);
+		}
+		mtx_unlock_spin(&sched_lock);
+	}
+#endif
+
 	/* Nothing to do for AST */
 }
 
diff --git a/sys/i386/include/apicvar.h b/sys/i386/include/apicvar.h
index 1beb025..2c1fdf1 100644
--- a/sys/i386/include/apicvar.h
+++ b/sys/i386/include/apicvar.h
@@ -122,7 +122,8 @@
 
 /* IPIs handled by IPI_BITMAPED_VECTOR  (XXX ups is there a better place?) */
 #define	IPI_AST		0 	/* Generate software trap. */
-#define IPI_BITMAP_LAST IPI_AST
+#define IPI_PREEMPT     1
+#define IPI_BITMAP_LAST IPI_PREEMPT
 #define IPI_IS_BITMAPED(x) ((x) <= IPI_BITMAP_LAST)
 
 #define	IPI_STOP	(APIC_IPI_INTS + 6)	/* Stop CPU until restarted. */