Rename the KDB_STOP_NMI kernel option to STOP_NMI and make it apply to all

IPI_STOP IPIs.
- Change the i386 and amd64 MD IPI code to send an NMI if STOP_NMI is
  enabled if an attempt is made to send an IPI_STOP IPI.  If the kernel
  option is enabled, there is also a sysctl to change the behavior at
  runtime (debug.stop_cpus_with_nmi which defaults to enabled).  This
  includes removing stop_cpus_nmi() and making ipi_nmi_selected() a
  private function for i386 and amd64.
- Fix ipi_all(), ipi_all_but_self(), and ipi_self() on i386 and amd64 to
  properly handle bitmapped IPIs as well as IPI_STOP IPIs when STOP_NMI is
  enabled.
- Fix ipi_nmi_handler() to execute the restart function on the first CPU
  that is restarted making use of atomic_readandclear() rather than
  assuming that the BSP is always included in the set of restarted CPUs.
  Also, the NMI handler didn't clear the function pointer meaning that
  subsequent stop and restarts could execute the function again.
- Define a new macro HAVE_STOPPEDPCBS on i386 and amd64 to control the use
  of stoppedpcbs[] and always enable it for i386 and amd64 instead of
  being dependent on KDB_STOP_NMI.  It works fine in both the NMI and
  non-NMI cases.

15 files changed, 115 insertions, 109 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 286dae9..be4f878 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -28,7 +28,6 @@
 __FBSDID("$FreeBSD$");
 
 #include "opt_cpu.h"
-#include "opt_kdb.h"
 #include "opt_kstack_pages.h"
 #include "opt_mp_watchdog.h"
 #include "opt_sched.h"
@@ -113,10 +112,30 @@ volatile int smp_tlb_wait;
 
 extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
 
+#ifdef STOP_NMI
+volatile cpumask_t ipi_nmi_pending;
+
+static void	ipi_nmi_selected(u_int32_t cpus);
+#endif 
+
 /*
  * Local data and functions.
  */
 
+#ifdef STOP_NMI
+/* 
+ * Provide an alternate method of stopping other CPUs. If another CPU has
+ * disabled interrupts the conventional STOP IPI will be blocked. This 
+ * NMI-based stop should get through in that case.
+ */
+static int stop_cpus_with_nmi = 1;
+SYSCTL_INT(_debug, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW,
+    &stop_cpus_with_nmi, 0, "");
+TUNABLE_INT("debug.stop_cpus_with_nmi", &stop_cpus_with_nmi);
+#else
+#define	stop_cpus_with_nmi	0
+#endif
+
 static u_int logical_cpus;
 
 /* used to hold the AP's until we are ready to release them */
@@ -199,11 +218,6 @@ mp_topology(void)
 	smp_topology = &mp_top;
 }
 
-
-#ifdef KDB_STOP_NMI
-volatile cpumask_t ipi_nmi_pending;
-#endif 
-
 /*
  * Calculate usable address in base memory for AP trampoline code.
  */
@@ -944,6 +958,12 @@ ipi_selected(u_int32_t cpus, u_int ipi)
 		ipi = IPI_BITMAP_VECTOR;
 	}
 
+#ifdef STOP_NMI
+	if (ipi == IPI_STOP && stop_cpus_with_nmi) {
+		ipi_nmi_selected(cpus);
+		return;
+	}
+#endif
 	CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
 	while ((cpu = ffs(cpus)) != 0) {
 		cpu--;
@@ -974,6 +994,10 @@ void
 ipi_all(u_int ipi)
 {
 
+	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+		ipi_selected(all_cpus, ipi);
+		return;
+	}
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL);
 }
@@ -985,6 +1009,10 @@ void
 ipi_all_but_self(u_int ipi)
 {
 
+	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+		ipi_selected(PCPU_GET(other_cpus), ipi);
+		return;
+	}
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
 }
@@ -996,11 +1024,15 @@ void
 ipi_self(u_int ipi)
 {
 
+	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+		ipi_selected(PCPU_GET(cpumask), ipi);
+		return;
+	}
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
 }
 
-#ifdef KDB_STOP_NMI
+#ifdef STOP_NMI
 /*
  * send NMI IPI to selected CPUs
  */
@@ -1040,8 +1072,9 @@ ipi_nmi_handler()
 {
 	int cpu = PCPU_GET(cpuid);
 	int cpumask = PCPU_GET(cpumask);
+	void (*restartfunc)(void);
 
-	if (!(atomic_load_acq_int(&ipi_nmi_pending) & cpumask))
+	if (!(ipi_nmi_pending & cpumask))
 		return 1;
 
 	atomic_clear_int(&ipi_nmi_pending, cpumask);
@@ -1052,19 +1085,21 @@ ipi_nmi_handler()
 	atomic_set_int(&stopped_cpus, cpumask);
 
 	/* Wait for restart */
-	while (!(atomic_load_acq_int(&started_cpus) & cpumask))
+	while (!(started_cpus & cpumask))
 	    ia32_pause();
 
 	atomic_clear_int(&started_cpus, cpumask);
 	atomic_clear_int(&stopped_cpus, cpumask);
 
-	if (cpu == 0 && cpustop_restartfunc != NULL)
-		cpustop_restartfunc();
+	restartfunc = (void (*)(void))atomic_readandclear_long(
+		(u_long *)&cpustop_restartfunc);
+	if (restartfunc != NULL)
+		restartfunc();
 
 	return 0;
 }
      
-#endif /* KDB_STOP_NMI */
+#endif /* STOP_NMI */
 
 /*
  * This is called once the rest of the system is up and running and we're
diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c
index ed74171..173ee2c 100644
--- a/sys/amd64/amd64/trap.c
+++ b/sys/amd64/amd64/trap.c
@@ -167,13 +167,13 @@ trap(frame)
 	PCPU_LAZY_INC(cnt.v_trap);
 	type = frame.tf_trapno;
 
-#ifdef KDB_STOP_NMI
-	/* Handler for NMI IPIs used for debugging */
+#ifdef STOP_NMI
+	/* Handler for NMI IPIs used for stopping CPUs. */
 	if (type == T_NMI) {
 	         if (ipi_nmi_handler() == 0)
 	                   goto out;
 	}
-#endif /* KDB_STOP_NMI */
+#endif /* STOP_NMI */
 
 #ifdef KDB
 	if (kdb_active) {
diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES
index 8799252..4bf2e7c 100644
--- a/sys/amd64/conf/NOTES
+++ b/sys/amd64/conf/NOTES
@@ -39,7 +39,7 @@ options 	MP_WATCHDOG
 # 
 # Debugging options.
 #
-options		KDB_STOP_NMI		# Stop CPUS using NMI instead of IPI
+options		STOP_NMI		# Stop CPUS using NMI instead of IPI
 
 
 
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index 2c63dc9..ba967c6 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -63,9 +63,8 @@ void	smp_masked_invlpg_range(u_int mask, vm_offset_t startva,
 void	smp_invltlb(void);
 void	smp_masked_invltlb(u_int mask);
 
-#ifdef KDB_STOP_NMI
-int ipi_nmi_handler(void);
-void ipi_nmi_selected(u_int32_t cpus);
+#ifdef STOP_NMI
+int	ipi_nmi_handler(void);
 #endif
 
 #endif /* !LOCORE */
diff --git a/sys/conf/options.amd64 b/sys/conf/options.amd64
index fdbc8f8..7b115ef 100644
--- a/sys/conf/options.amd64
+++ b/sys/conf/options.amd64
@@ -57,4 +57,4 @@ PSM_DEBUG		opt_psm.h
 DEV_ATPIC		opt_atpic.h
 
 # Debugging
-KDB_STOP_NMI		opt_kdb.h
+STOP_NMI		opt_cpu.h
diff --git a/sys/conf/options.i386 b/sys/conf/options.i386
index 8ad0b8b..7dafd5e 100644
--- a/sys/conf/options.i386
+++ b/sys/conf/options.i386
@@ -161,5 +161,5 @@ DEV_NPX			opt_npx.h
 ASR_COMPAT		opt_asr.h
 
 # Debugging
-KDB_STOP_NMI		opt_kdb.h
+STOP_NMI		opt_cpu.h
 NPX_DEBUG		opt_npx.h
diff --git a/sys/conf/options.pc98 b/sys/conf/options.pc98
index f50c867..a1f3cc1 100644
--- a/sys/conf/options.pc98
+++ b/sys/conf/options.pc98
@@ -103,5 +103,5 @@ DEV_MECIA		opt_mecia.h
 DEV_NPX			opt_npx.h
 
 # Debugging
-KDB_STOP_NMI		opt_kdb.h
+STOP_NMI		opt_cpu.h
 NPX_DEBUG		opt_npx.h
diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES
index 4a3a5b4..9cbbab5 100644
--- a/sys/i386/conf/NOTES
+++ b/sys/i386/conf/NOTES
@@ -56,7 +56,7 @@ options 	MP_WATCHDOG
 
 # Debugging options.
 #
-options		KDB_STOP_NMI		# Stop CPUS using NMI instead of IPI
+options		STOP_NMI		# Stop CPUS using NMI instead of IPI
 
 
 
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 90ded0c..bfc3c5b 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -28,7 +28,6 @@ __FBSDID("$FreeBSD$");
 
 #include "opt_apic.h"
 #include "opt_cpu.h"
-#include "opt_kdb.h"
 #include "opt_kstack_pages.h"
 #include "opt_mp_watchdog.h"
 #include "opt_sched.h"
@@ -177,8 +176,10 @@ vm_offset_t smp_tlb_addr1;
 vm_offset_t smp_tlb_addr2;
 volatile int smp_tlb_wait;
 
-#ifdef KDB_STOP_NMI
+#ifdef STOP_NMI
 volatile cpumask_t ipi_nmi_pending;
+
+static void	ipi_nmi_selected(u_int32_t cpus);
 #endif 
 
 #ifdef COUNT_IPIS
@@ -198,6 +199,20 @@ u_long *ipi_lazypmap_counts[MAXCPU];
  * Local data and functions.
  */
 
+#ifdef STOP_NMI
+/* 
+ * Provide an alternate method of stopping other CPUs. If another CPU has
+ * disabled interrupts the conventional STOP IPI will be blocked. This 
+ * NMI-based stop should get through in that case.
+ */
+static int stop_cpus_with_nmi = 1;
+SYSCTL_INT(_debug, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW,
+    &stop_cpus_with_nmi, 0, "");
+TUNABLE_INT("debug.stop_cpus_with_nmi", &stop_cpus_with_nmi);
+#else
+#define	stop_cpus_with_nmi	0
+#endif
+
 static u_int logical_cpus;
 
 /* used to hold the AP's until we are ready to release them */
@@ -1182,6 +1197,12 @@ ipi_selected(u_int32_t cpus, u_int ipi)
 		ipi = IPI_BITMAP_VECTOR;
 	}
 
+#ifdef STOP_NMI
+	if (ipi == IPI_STOP && stop_cpus_with_nmi) {
+		ipi_nmi_selected(cpus);
+		return;
+	}
+#endif
 	CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi);
 	while ((cpu = ffs(cpus)) != 0) {
 		cpu--;
@@ -1212,6 +1233,10 @@ void
 ipi_all(u_int ipi)
 {
 
+	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+		ipi_selected(all_cpus, ipi);
+		return;
+	}
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL);
 }
@@ -1223,6 +1248,10 @@ void
 ipi_all_but_self(u_int ipi)
 {
 
+	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+		ipi_selected(PCPU_GET(other_cpus), ipi);
+		return;
+	}
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS);
 }
@@ -1234,11 +1263,15 @@ void
 ipi_self(u_int ipi)
 {
 
+	if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) {
+		ipi_selected(PCPU_GET(cpumask), ipi);
+		return;
+	}
 	CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi);
 	lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF);
 }
 
-#ifdef KDB_STOP_NMI
+#ifdef STOP_NMI
 /*
  * send NMI IPI to selected CPUs
  */
@@ -1273,14 +1306,14 @@ ipi_nmi_selected(u_int32_t cpus)
 	}
 }
 
-
 int
 ipi_nmi_handler()
 {
 	int cpu = PCPU_GET(cpuid);
 	int cpumask = PCPU_GET(cpumask);
+	void (*restartfunc)(void);
 
-	if (!(atomic_load_acq_int(&ipi_nmi_pending) & cpumask))
+	if (!(ipi_nmi_pending & cpumask))
 		return 1;
 
 	atomic_clear_int(&ipi_nmi_pending, cpumask);
@@ -1291,19 +1324,21 @@ ipi_nmi_handler()
 	atomic_set_int(&stopped_cpus, cpumask);
 
 	/* Wait for restart */
-	while (!(atomic_load_acq_int(&started_cpus) & cpumask))
+	while (!(started_cpus & cpumask))
 	    ia32_pause();
 
 	atomic_clear_int(&started_cpus, cpumask);
 	atomic_clear_int(&stopped_cpus, cpumask);
 
-	if (cpu == 0 && cpustop_restartfunc != NULL)
-		cpustop_restartfunc();
+	restartfunc = (void (*)(void))atomic_readandclear_int(
+		(u_int *)&cpustop_restartfunc);
+	if (restartfunc != NULL)
+		restartfunc();
 
 	return 0;
 }
      
-#endif /* KDB_STOP_NMI */
+#endif /* STOP_NMI */
 
 /*
  * This is called once the rest of the system is up and running and we're
diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c
index 4703537..4c1df2a 100644
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@@ -185,13 +185,13 @@ trap(frame)
 	PCPU_LAZY_INC(cnt.v_trap);
 	type = frame.tf_trapno;
 
-#ifdef KDB_STOP_NMI
-	/* Handler for NMI IPIs used for debugging */
+#ifdef STOP_NMI
+	/* Handler for NMI IPIs used for stopping CPUs. */
 	if (type == T_NMI) {
 	         if (ipi_nmi_handler() == 0)
 	                   goto out;
 	}
-#endif /* KDB_STOP_NMI */
+#endif /* STOP_NMI */
 
 #ifdef KDB
 	if (kdb_active) {
diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h
index 4c9ae76..d2482fe 100644
--- a/sys/i386/include/smp.h
+++ b/sys/i386/include/smp.h
@@ -79,9 +79,8 @@ void	smp_masked_invlpg_range(u_int mask, vm_offset_t startva,
 void	smp_invltlb(void);
 void	smp_masked_invltlb(u_int mask);
 
-#ifdef KDB_STOP_NMI
-int ipi_nmi_handler(void);
-void ipi_nmi_selected(u_int32_t cpus);
+#ifdef STOP_NMI
+int	ipi_nmi_handler(void);
 #endif
 
 #endif /* !LOCORE */
diff --git a/sys/kern/subr_kdb.c b/sys/kern/subr_kdb.c
index c4f14d8..3157578 100644
--- a/sys/kern/subr_kdb.c
+++ b/sys/kern/subr_kdb.c
@@ -42,16 +42,11 @@ __FBSDID("$FreeBSD$");
 #include <machine/kdb.h>
 #include <machine/pcb.h>
 
-#ifdef KDB_STOP_NMI
+#ifdef SMP
+#if defined (__i386__) || defined(__amd64__)
+#define	HAVE_STOPPEDPCBS
 #include <machine/smp.h>
 #endif
-
-/* 
- * KDB_STOP_NMI requires SMP to pick up the right dependencies
- * (And isn't useful on UP anyway) 
- */
-#if defined(KDB_STOP_NMI) && !defined(SMP)
-#error "options KDB_STOP_NMI" requires "options SMP"
 #endif
 
 int kdb_active = 0;
@@ -91,19 +86,6 @@ static int kdb_stop_cpus = 1;
 SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus, CTLTYPE_INT | CTLFLAG_RW,
     &kdb_stop_cpus, 0, "");
 TUNABLE_INT("debug.kdb.stop_cpus", &kdb_stop_cpus);
-
-#ifdef KDB_STOP_NMI
-/* 
- * Provide an alternate method of stopping other CPUs. If another CPU has
- * disabled interrupts the conventional STOP IPI will be blocked. This 
- * NMI-based stop should get through in that case.
- */
-static int kdb_stop_cpus_with_nmi = 1;
-SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW,
-    &kdb_stop_cpus_with_nmi, 0, "");
-TUNABLE_INT("debug.kdb.stop_cpus_with_nmi", &kdb_stop_cpus_with_nmi);
-#endif /* KDB_STOP_NMI */
-
 #endif
 
 static int
@@ -335,26 +317,24 @@ kdb_reenter(void)
 
 struct pcb *
 kdb_thr_ctx(struct thread *thr)
-#ifdef KDB_STOP_NMI
 {  
+#ifdef HAVE_STOPPEDPCBS
 	struct pcpu *pc;
 	u_int cpuid;
+#endif
   
 	if (thr == curthread) 
 		return (&kdb_pcb);
 
+#ifdef HAVE_STOPPEDPCBS
 	SLIST_FOREACH(pc, &cpuhead, pc_allcpu)  {
 		cpuid = pc->pc_cpuid;
-		if (pc->pc_curthread == thr && (atomic_load_acq_int(&stopped_cpus) & (1 << cpuid)))
+		if (pc->pc_curthread == thr && (stopped_cpus & (1 << cpuid)))
 			return (&stoppcbs[cpuid]);
 	}
+#endif
 	return (thr->td_pcb);
 }
-#else
-{
-	return ((thr == curthread) ? &kdb_pcb : thr->td_pcb);
-}
-#endif /* KDB_STOP_NMI */
 
 struct thread *
 kdb_thr_first(void)
@@ -451,14 +431,7 @@ kdb_trap(int type, int code, struct trapframe *tf)
 
 #ifdef SMP
 	if ((did_stop_cpus = kdb_stop_cpus) != 0)
-	  {
-#ifdef KDB_STOP_NMI
-	    if(kdb_stop_cpus_with_nmi)
-	      stop_cpus_nmi(PCPU_GET(other_cpus));
-	    else
-#endif /* KDB_STOP_NMI */
 		stop_cpus(PCPU_GET(other_cpus));
-	  }
 #endif
 
 	kdb_frame = tf;
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 6b11b47..f0c0994 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -35,8 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include "opt_kdb.h"
-
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -256,36 +254,6 @@ stop_cpus(cpumask_t map)
 	return 1;
 }
 
-#ifdef KDB_STOP_NMI
-int
-stop_cpus_nmi(cpumask_t map)
-{
-	int i;
-
-	if (!smp_started)
-		return 0;
-
-	CTR1(KTR_SMP, "stop_cpus(%x)", map);
-
-	/* send the stop IPI to all CPUs in map */
-	ipi_nmi_selected(map);
-
-	i = 0;
-	while ((atomic_load_acq_int(&stopped_cpus) & map) != map) {
-		/* spin */
-		i++;
-#ifdef DIAGNOSTIC
-		if (i == 100000) {
-			printf("timeout stopping cpus\n");
-			break;
-		}
-#endif
-	}
-
-	return 1;
-}
-#endif /* KDB_STOP_NMI */
-
 /*
  * Called by a CPU to restart stopped CPUs. 
  *
diff --git a/sys/pc98/conf/NOTES b/sys/pc98/conf/NOTES
index af8d271..6e410f6 100644
--- a/sys/pc98/conf/NOTES
+++ b/sys/pc98/conf/NOTES
@@ -39,7 +39,7 @@ options 	MP_WATCHDOG
 
 # Debugging options.
 #
-options		KDB_STOP_NMI		# Stop CPUS using NMI instead of IPI
+options		STOP_NMI		# Stop CPUS using NMI instead of IPI
 
 
 
diff --git a/sys/sys/smp.h b/sys/sys/smp.h
index 88c6ae2..f2b2f52 100644
--- a/sys/sys/smp.h
+++ b/sys/sys/smp.h
@@ -102,9 +102,6 @@ int	stop_cpus(cpumask_t);
 void	smp_rendezvous_action(void);
 extern	struct mtx smp_ipi_mtx;
 
-#ifdef KDB_STOP_NMI
-int	stop_cpus_nmi(cpumask_t);
-#endif 
 #endif /* SMP */
 void	smp_rendezvous(void (*)(void *), 
 		       void (*)(void *),