Rename the KDB_STOP_NMI kernel option to STOP_NMI and make it apply to all

IPI_STOP IPIs.
- Change the i386 and amd64 MD IPI code to send an NMI if STOP_NMI is
  enabled if an attempt is made to send an IPI_STOP IPI.  If the kernel
  option is enabled, there is also a sysctl to change the behavior at
  runtime (debug.stop_cpus_with_nmi which defaults to enabled).  This
  includes removing stop_cpus_nmi() and making ipi_nmi_selected() a
  private function for i386 and amd64.
- Fix ipi_all(), ipi_all_but_self(), and ipi_self() on i386 and amd64 to
  properly handle bitmapped IPIs as well as IPI_STOP IPIs when STOP_NMI is
  enabled.
- Fix ipi_nmi_handler() to execute the restart function on the first CPU
  that is restarted making use of atomic_readandclear() rather than
  assuming that the BSP is always included in the set of restarted CPUs.
  Also, the NMI handler didn't clear the function pointer meaning that
  subsequent stop and restarts could execute the function again.
- Define a new macro HAVE_STOPPEDPCBS on i386 and amd64 to control the use
  of stoppedpcbs[] and always enable it for i386 and amd64 instead of
  being dependent on KDB_STOP_NMI.  It works fine in both the NMI and
  non-NMI cases.

2 files changed, 8 insertions, 67 deletions

diff --git a/sys/kern/subr_kdb.c b/sys/kern/subr_kdb.c
index c4f14d8..3157578 100644
--- a/sys/kern/subr_kdb.c
+++ b/sys/kern/subr_kdb.c
@@ -42,16 +42,11 @@ __FBSDID("$FreeBSD$");
 #include <machine/kdb.h>
 #include <machine/pcb.h>
 
-#ifdef KDB_STOP_NMI
+#ifdef SMP
+#if defined (__i386__) || defined(__amd64__)
+#define	HAVE_STOPPEDPCBS
 #include <machine/smp.h>
 #endif
-
-/* 
- * KDB_STOP_NMI requires SMP to pick up the right dependencies
- * (And isn't useful on UP anyway) 
- */
-#if defined(KDB_STOP_NMI) && !defined(SMP)
-#error "options KDB_STOP_NMI" requires "options SMP"
 #endif
 
 int kdb_active = 0;
@@ -91,19 +86,6 @@ static int kdb_stop_cpus = 1;
 SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus, CTLTYPE_INT | CTLFLAG_RW,
     &kdb_stop_cpus, 0, "");
 TUNABLE_INT("debug.kdb.stop_cpus", &kdb_stop_cpus);
-
-#ifdef KDB_STOP_NMI
-/* 
- * Provide an alternate method of stopping other CPUs. If another CPU has
- * disabled interrupts the conventional STOP IPI will be blocked. This 
- * NMI-based stop should get through in that case.
- */
-static int kdb_stop_cpus_with_nmi = 1;
-SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW,
-    &kdb_stop_cpus_with_nmi, 0, "");
-TUNABLE_INT("debug.kdb.stop_cpus_with_nmi", &kdb_stop_cpus_with_nmi);
-#endif /* KDB_STOP_NMI */
-
 #endif
 
 static int
@@ -335,26 +317,24 @@ kdb_reenter(void)
 
 struct pcb *
 kdb_thr_ctx(struct thread *thr)
-#ifdef KDB_STOP_NMI
 {  
+#ifdef HAVE_STOPPEDPCBS
 	struct pcpu *pc;
 	u_int cpuid;
+#endif
   
 	if (thr == curthread) 
 		return (&kdb_pcb);
 
+#ifdef HAVE_STOPPEDPCBS
 	SLIST_FOREACH(pc, &cpuhead, pc_allcpu)  {
 		cpuid = pc->pc_cpuid;
-		if (pc->pc_curthread == thr && (atomic_load_acq_int(&stopped_cpus) & (1 << cpuid)))
+		if (pc->pc_curthread == thr && (stopped_cpus & (1 << cpuid)))
 			return (&stoppcbs[cpuid]);
 	}
+#endif
 	return (thr->td_pcb);
 }
-#else
-{
-	return ((thr == curthread) ? &kdb_pcb : thr->td_pcb);
-}
-#endif /* KDB_STOP_NMI */
 
 struct thread *
 kdb_thr_first(void)
@@ -451,14 +431,7 @@ kdb_trap(int type, int code, struct trapframe *tf)
 
 #ifdef SMP
 	if ((did_stop_cpus = kdb_stop_cpus) != 0)
-	  {
-#ifdef KDB_STOP_NMI
-	    if(kdb_stop_cpus_with_nmi)
-	      stop_cpus_nmi(PCPU_GET(other_cpus));
-	    else
-#endif /* KDB_STOP_NMI */
 		stop_cpus(PCPU_GET(other_cpus));
-	  }
 #endif
 
 	kdb_frame = tf;
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 6b11b47..f0c0994 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -35,8 +35,6 @@
 #include <sys/cdefs.h>
 __FBSDID("$FreeBSD$");
 
-#include "opt_kdb.h"
-
 #include <sys/param.h>
 #include <sys/systm.h>
 #include <sys/kernel.h>
@@ -256,36 +254,6 @@ stop_cpus(cpumask_t map)
 	return 1;
 }
 
-#ifdef KDB_STOP_NMI
-int
-stop_cpus_nmi(cpumask_t map)
-{
-	int i;
-
-	if (!smp_started)
-		return 0;
-
-	CTR1(KTR_SMP, "stop_cpus(%x)", map);
-
-	/* send the stop IPI to all CPUs in map */
-	ipi_nmi_selected(map);
-
-	i = 0;
-	while ((atomic_load_acq_int(&stopped_cpus) & map) != map) {
-		/* spin */
-		i++;
-#ifdef DIAGNOSTIC
-		if (i == 100000) {
-			printf("timeout stopping cpus\n");
-			break;
-		}
-#endif
-	}
-
-	return 1;
-}
-#endif /* KDB_STOP_NMI */
-
 /*
  * Called by a CPU to restart stopped CPUs. 
  *