diff options
| -rw-r--r-- | sys/amd64/amd64/mp_machdep.c | 59 | ||||
| -rw-r--r-- | sys/amd64/amd64/trap.c | 6 | ||||
| -rw-r--r-- | sys/amd64/conf/NOTES | 2 | ||||
| -rw-r--r-- | sys/amd64/include/smp.h | 5 | ||||
| -rw-r--r-- | sys/conf/options.amd64 | 2 | ||||
| -rw-r--r-- | sys/conf/options.i386 | 2 | ||||
| -rw-r--r-- | sys/conf/options.pc98 | 2 | ||||
| -rw-r--r-- | sys/i386/conf/NOTES | 2 | ||||
| -rw-r--r-- | sys/i386/i386/mp_machdep.c | 53 | ||||
| -rw-r--r-- | sys/i386/i386/trap.c | 6 | ||||
| -rw-r--r-- | sys/i386/include/smp.h | 5 | ||||
| -rw-r--r-- | sys/kern/subr_kdb.c | 43 | ||||
| -rw-r--r-- | sys/kern/subr_smp.c | 32 | ||||
| -rw-r--r-- | sys/pc98/conf/NOTES | 2 | ||||
| -rw-r--r-- | sys/sys/smp.h | 3 |
15 files changed, 115 insertions, 109 deletions
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 286dae9..be4f878 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -28,7 +28,6 @@ __FBSDID("$FreeBSD$"); #include "opt_cpu.h" -#include "opt_kdb.h" #include "opt_kstack_pages.h" #include "opt_mp_watchdog.h" #include "opt_sched.h" @@ -113,10 +112,30 @@ volatile int smp_tlb_wait; extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32); +#ifdef STOP_NMI +volatile cpumask_t ipi_nmi_pending; + +static void ipi_nmi_selected(u_int32_t cpus); +#endif + /* * Local data and functions. */ +#ifdef STOP_NMI +/* + * Provide an alternate method of stopping other CPUs. If another CPU has + * disabled interrupts the conventional STOP IPI will be blocked. This + * NMI-based stop should get through in that case. + */ +static int stop_cpus_with_nmi = 1; +SYSCTL_INT(_debug, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW, + &stop_cpus_with_nmi, 0, ""); +TUNABLE_INT("debug.stop_cpus_with_nmi", &stop_cpus_with_nmi); +#else +#define stop_cpus_with_nmi 0 +#endif + static u_int logical_cpus; /* used to hold the AP's until we are ready to release them */ @@ -199,11 +218,6 @@ mp_topology(void) smp_topology = &mp_top; } - -#ifdef KDB_STOP_NMI -volatile cpumask_t ipi_nmi_pending; -#endif - /* * Calculate usable address in base memory for AP trampoline code. */ @@ -944,6 +958,12 @@ ipi_selected(u_int32_t cpus, u_int ipi) ipi = IPI_BITMAP_VECTOR; } +#ifdef STOP_NMI + if (ipi == IPI_STOP && stop_cpus_with_nmi) { + ipi_nmi_selected(cpus); + return; + } +#endif CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); while ((cpu = ffs(cpus)) != 0) { cpu--; @@ -974,6 +994,10 @@ void ipi_all(u_int ipi) { + if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { + ipi_selected(all_cpus, ipi); + return; + } CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL); } @@ -985,6 +1009,10 @@ void ipi_all_but_self(u_int ipi) { + if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { + ipi_selected(PCPU_GET(other_cpus), ipi); + return; + } CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); } @@ -996,11 +1024,15 @@ void ipi_self(u_int ipi) { + if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { + ipi_selected(PCPU_GET(cpumask), ipi); + return; + } CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); } -#ifdef KDB_STOP_NMI +#ifdef STOP_NMI /* * send NMI IPI to selected CPUs */ @@ -1040,8 +1072,9 @@ ipi_nmi_handler() { int cpu = PCPU_GET(cpuid); int cpumask = PCPU_GET(cpumask); + void (*restartfunc)(void); - if (!(atomic_load_acq_int(&ipi_nmi_pending) & cpumask)) + if (!(ipi_nmi_pending & cpumask)) return 1; atomic_clear_int(&ipi_nmi_pending, cpumask); @@ -1052,19 +1085,21 @@ ipi_nmi_handler() atomic_set_int(&stopped_cpus, cpumask); /* Wait for restart */ - while (!(atomic_load_acq_int(&started_cpus) & cpumask)) + while (!(started_cpus & cpumask)) ia32_pause(); atomic_clear_int(&started_cpus, cpumask); atomic_clear_int(&stopped_cpus, cpumask); - if (cpu == 0 && cpustop_restartfunc != NULL) - cpustop_restartfunc(); + restartfunc = (void (*)(void))atomic_readandclear_long( + (u_long *)&cpustop_restartfunc); + if (restartfunc != NULL) + restartfunc(); return 0; } -#endif /* KDB_STOP_NMI */ +#endif /* STOP_NMI */ /* * This is called once the rest of the system is up and running and we're diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index ed74171..173ee2c 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -167,13 +167,13 @@ trap(frame) PCPU_LAZY_INC(cnt.v_trap); type = frame.tf_trapno; -#ifdef KDB_STOP_NMI - /* Handler for NMI IPIs used for debugging */ +#ifdef STOP_NMI + /* Handler for NMI IPIs used for stopping CPUs. */ if (type == T_NMI) { if (ipi_nmi_handler() == 0) goto out; } -#endif /* KDB_STOP_NMI */ +#endif /* STOP_NMI */ #ifdef KDB if (kdb_active) { diff --git a/sys/amd64/conf/NOTES b/sys/amd64/conf/NOTES index 8799252..4bf2e7c 100644 --- a/sys/amd64/conf/NOTES +++ b/sys/amd64/conf/NOTES @@ -39,7 +39,7 @@ options MP_WATCHDOG # # Debugging options. # -options KDB_STOP_NMI # Stop CPUS using NMI instead of IPI +options STOP_NMI # Stop CPUS using NMI instead of IPI diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 2c63dc9..ba967c6 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -63,9 +63,8 @@ void smp_masked_invlpg_range(u_int mask, vm_offset_t startva, void smp_invltlb(void); void smp_masked_invltlb(u_int mask); -#ifdef KDB_STOP_NMI -int ipi_nmi_handler(void); -void ipi_nmi_selected(u_int32_t cpus); +#ifdef STOP_NMI +int ipi_nmi_handler(void); #endif #endif /* !LOCORE */ diff --git a/sys/conf/options.amd64 b/sys/conf/options.amd64 index fdbc8f8..7b115ef 100644 --- a/sys/conf/options.amd64 +++ b/sys/conf/options.amd64 @@ -57,4 +57,4 @@ PSM_DEBUG opt_psm.h DEV_ATPIC opt_atpic.h # Debugging -KDB_STOP_NMI opt_kdb.h +STOP_NMI opt_cpu.h diff --git a/sys/conf/options.i386 b/sys/conf/options.i386 index 8ad0b8b..7dafd5e 100644 --- a/sys/conf/options.i386 +++ b/sys/conf/options.i386 @@ -161,5 +161,5 @@ DEV_NPX opt_npx.h ASR_COMPAT opt_asr.h # Debugging -KDB_STOP_NMI opt_kdb.h +STOP_NMI opt_cpu.h NPX_DEBUG opt_npx.h diff --git a/sys/conf/options.pc98 b/sys/conf/options.pc98 index f50c867..a1f3cc1 100644 --- a/sys/conf/options.pc98 +++ b/sys/conf/options.pc98 @@ -103,5 +103,5 @@ DEV_MECIA opt_mecia.h DEV_NPX opt_npx.h # Debugging -KDB_STOP_NMI opt_kdb.h +STOP_NMI opt_cpu.h NPX_DEBUG opt_npx.h diff --git a/sys/i386/conf/NOTES b/sys/i386/conf/NOTES index 4a3a5b4..9cbbab5 100644 --- a/sys/i386/conf/NOTES +++ b/sys/i386/conf/NOTES @@ -56,7 +56,7 @@ options MP_WATCHDOG # Debugging options. # -options KDB_STOP_NMI # Stop CPUS using NMI instead of IPI +options STOP_NMI # Stop CPUS using NMI instead of IPI diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index 90ded0c..bfc3c5b 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -28,7 +28,6 @@ __FBSDID("$FreeBSD$"); #include "opt_apic.h" #include "opt_cpu.h" -#include "opt_kdb.h" #include "opt_kstack_pages.h" #include "opt_mp_watchdog.h" #include "opt_sched.h" @@ -177,8 +176,10 @@ vm_offset_t smp_tlb_addr1; vm_offset_t smp_tlb_addr2; volatile int smp_tlb_wait; -#ifdef KDB_STOP_NMI +#ifdef STOP_NMI volatile cpumask_t ipi_nmi_pending; + +static void ipi_nmi_selected(u_int32_t cpus); #endif #ifdef COUNT_IPIS @@ -198,6 +199,20 @@ u_long *ipi_lazypmap_counts[MAXCPU]; * Local data and functions. */ +#ifdef STOP_NMI +/* + * Provide an alternate method of stopping other CPUs. If another CPU has + * disabled interrupts the conventional STOP IPI will be blocked. This + * NMI-based stop should get through in that case. + */ +static int stop_cpus_with_nmi = 1; +SYSCTL_INT(_debug, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW, + &stop_cpus_with_nmi, 0, ""); +TUNABLE_INT("debug.stop_cpus_with_nmi", &stop_cpus_with_nmi); +#else +#define stop_cpus_with_nmi 0 +#endif + static u_int logical_cpus; /* used to hold the AP's until we are ready to release them */ @@ -1182,6 +1197,12 @@ ipi_selected(u_int32_t cpus, u_int ipi) ipi = IPI_BITMAP_VECTOR; } +#ifdef STOP_NMI + if (ipi == IPI_STOP && stop_cpus_with_nmi) { + ipi_nmi_selected(cpus); + return; + } +#endif CTR3(KTR_SMP, "%s: cpus: %x ipi: %x", __func__, cpus, ipi); while ((cpu = ffs(cpus)) != 0) { cpu--; @@ -1212,6 +1233,10 @@ void ipi_all(u_int ipi) { + if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { + ipi_selected(all_cpus, ipi); + return; + } CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_ALL); } @@ -1223,6 +1248,10 @@ void ipi_all_but_self(u_int ipi) { + if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { + ipi_selected(PCPU_GET(other_cpus), ipi); + return; + } CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_OTHERS); } @@ -1234,11 +1263,15 @@ void ipi_self(u_int ipi) { + if (IPI_IS_BITMAPED(ipi) || (ipi == IPI_STOP && stop_cpus_with_nmi)) { + ipi_selected(PCPU_GET(cpumask), ipi); + return; + } CTR2(KTR_SMP, "%s: ipi: %x", __func__, ipi); lapic_ipi_vectored(ipi, APIC_IPI_DEST_SELF); } -#ifdef KDB_STOP_NMI +#ifdef STOP_NMI /* * send NMI IPI to selected CPUs */ @@ -1273,14 +1306,14 @@ ipi_nmi_selected(u_int32_t cpus) } } - int ipi_nmi_handler() { int cpu = PCPU_GET(cpuid); int cpumask = PCPU_GET(cpumask); + void (*restartfunc)(void); - if (!(atomic_load_acq_int(&ipi_nmi_pending) & cpumask)) + if (!(ipi_nmi_pending & cpumask)) return 1; atomic_clear_int(&ipi_nmi_pending, cpumask); @@ -1291,19 +1324,21 @@ ipi_nmi_handler() atomic_set_int(&stopped_cpus, cpumask); /* Wait for restart */ - while (!(atomic_load_acq_int(&started_cpus) & cpumask)) + while (!(started_cpus & cpumask)) ia32_pause(); atomic_clear_int(&started_cpus, cpumask); atomic_clear_int(&stopped_cpus, cpumask); - if (cpu == 0 && cpustop_restartfunc != NULL) - cpustop_restartfunc(); + restartfunc = (void (*)(void))atomic_readandclear_int( + (u_int *)&cpustop_restartfunc); + if (restartfunc != NULL) + restartfunc(); return 0; } -#endif /* KDB_STOP_NMI */ +#endif /* STOP_NMI */ /* * This is called once the rest of the system is up and running and we're diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 4703537..4c1df2a 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -185,13 +185,13 @@ trap(frame) PCPU_LAZY_INC(cnt.v_trap); type = frame.tf_trapno; -#ifdef KDB_STOP_NMI - /* Handler for NMI IPIs used for debugging */ +#ifdef STOP_NMI + /* Handler for NMI IPIs used for stopping CPUs. */ if (type == T_NMI) { if (ipi_nmi_handler() == 0) goto out; } -#endif /* KDB_STOP_NMI */ +#endif /* STOP_NMI */ #ifdef KDB if (kdb_active) { diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h index 4c9ae76..d2482fe 100644 --- a/sys/i386/include/smp.h +++ b/sys/i386/include/smp.h @@ -79,9 +79,8 @@ void smp_masked_invlpg_range(u_int mask, vm_offset_t startva, void smp_invltlb(void); void smp_masked_invltlb(u_int mask); -#ifdef KDB_STOP_NMI -int ipi_nmi_handler(void); -void ipi_nmi_selected(u_int32_t cpus); +#ifdef STOP_NMI +int ipi_nmi_handler(void); #endif #endif /* !LOCORE */ diff --git a/sys/kern/subr_kdb.c b/sys/kern/subr_kdb.c index c4f14d8..3157578 100644 --- a/sys/kern/subr_kdb.c +++ b/sys/kern/subr_kdb.c @@ -42,16 +42,11 @@ __FBSDID("$FreeBSD$"); #include <machine/kdb.h> #include <machine/pcb.h> -#ifdef KDB_STOP_NMI +#ifdef SMP +#if defined (__i386__) || defined(__amd64__) +#define HAVE_STOPPEDPCBS #include <machine/smp.h> #endif - -/* - * KDB_STOP_NMI requires SMP to pick up the right dependencies - * (And isn't useful on UP anyway) - */ -#if defined(KDB_STOP_NMI) && !defined(SMP) -#error "options KDB_STOP_NMI" requires "options SMP" #endif int kdb_active = 0; @@ -91,19 +86,6 @@ static int kdb_stop_cpus = 1; SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus, CTLTYPE_INT | CTLFLAG_RW, &kdb_stop_cpus, 0, ""); TUNABLE_INT("debug.kdb.stop_cpus", &kdb_stop_cpus); - -#ifdef KDB_STOP_NMI -/* - * Provide an alternate method of stopping other CPUs. If another CPU has - * disabled interrupts the conventional STOP IPI will be blocked. This - * NMI-based stop should get through in that case. - */ -static int kdb_stop_cpus_with_nmi = 1; -SYSCTL_INT(_debug_kdb, OID_AUTO, stop_cpus_with_nmi, CTLTYPE_INT | CTLFLAG_RW, - &kdb_stop_cpus_with_nmi, 0, ""); -TUNABLE_INT("debug.kdb.stop_cpus_with_nmi", &kdb_stop_cpus_with_nmi); -#endif /* KDB_STOP_NMI */ - #endif static int @@ -335,26 +317,24 @@ kdb_reenter(void) struct pcb * kdb_thr_ctx(struct thread *thr) -#ifdef KDB_STOP_NMI { +#ifdef HAVE_STOPPEDPCBS struct pcpu *pc; u_int cpuid; +#endif if (thr == curthread) return (&kdb_pcb); +#ifdef HAVE_STOPPEDPCBS SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { cpuid = pc->pc_cpuid; - if (pc->pc_curthread == thr && (atomic_load_acq_int(&stopped_cpus) & (1 << cpuid))) + if (pc->pc_curthread == thr && (stopped_cpus & (1 << cpuid))) return (&stoppcbs[cpuid]); } +#endif return (thr->td_pcb); } -#else -{ - return ((thr == curthread) ? &kdb_pcb : thr->td_pcb); -} -#endif /* KDB_STOP_NMI */ struct thread * kdb_thr_first(void) @@ -451,14 +431,7 @@ kdb_trap(int type, int code, struct trapframe *tf) #ifdef SMP if ((did_stop_cpus = kdb_stop_cpus) != 0) - { -#ifdef KDB_STOP_NMI - if(kdb_stop_cpus_with_nmi) - stop_cpus_nmi(PCPU_GET(other_cpus)); - else -#endif /* KDB_STOP_NMI */ stop_cpus(PCPU_GET(other_cpus)); - } #endif kdb_frame = tf; diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index 6b11b47..f0c0994 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -35,8 +35,6 @@ #include <sys/cdefs.h> __FBSDID("$FreeBSD$"); -#include "opt_kdb.h" - #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> @@ -256,36 +254,6 @@ stop_cpus(cpumask_t map) return 1; } -#ifdef KDB_STOP_NMI -int -stop_cpus_nmi(cpumask_t map) -{ - int i; - - if (!smp_started) - return 0; - - CTR1(KTR_SMP, "stop_cpus(%x)", map); - - /* send the stop IPI to all CPUs in map */ - ipi_nmi_selected(map); - - i = 0; - while ((atomic_load_acq_int(&stopped_cpus) & map) != map) { - /* spin */ - i++; -#ifdef DIAGNOSTIC - if (i == 100000) { - printf("timeout stopping cpus\n"); - break; - } -#endif - } - - return 1; -} -#endif /* KDB_STOP_NMI */ - /* * Called by a CPU to restart stopped CPUs. * diff --git a/sys/pc98/conf/NOTES b/sys/pc98/conf/NOTES index af8d271..6e410f6 100644 --- a/sys/pc98/conf/NOTES +++ b/sys/pc98/conf/NOTES @@ -39,7 +39,7 @@ options MP_WATCHDOG # Debugging options. # -options KDB_STOP_NMI # Stop CPUS using NMI instead of IPI +options STOP_NMI # Stop CPUS using NMI instead of IPI diff --git a/sys/sys/smp.h b/sys/sys/smp.h index 88c6ae2..f2b2f52 100644 --- a/sys/sys/smp.h +++ b/sys/sys/smp.h @@ -102,9 +102,6 @@ int stop_cpus(cpumask_t); void smp_rendezvous_action(void); extern struct mtx smp_ipi_mtx; -#ifdef KDB_STOP_NMI -int stop_cpus_nmi(cpumask_t); -#endif #endif /* SMP */ void smp_rendezvous(void (*)(void *), void (*)(void *), |
