summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--sys/kern/subr_witness.c3
-rw-r--r--sys/sparc64/include/smp.h50
-rw-r--r--sys/sparc64/sparc64/machdep.c2
-rw-r--r--sys/sparc64/sparc64/mp_machdep.c99
-rw-r--r--sys/sparc64/sparc64/tick.c7
5 files changed, 81 insertions, 80 deletions
diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c
index 8b1e3a0..122a246 100644
--- a/sys/kern/subr_witness.c
+++ b/sys/kern/subr_witness.c
@@ -666,6 +666,9 @@ static struct witness_order_list_entry order_lists[] = {
*/
{ "intrcnt", &lock_class_mtx_spin },
{ "icu", &lock_class_mtx_spin },
+#if defined(SMP) && defined(__sparc64__)
+ { "ipi", &lock_class_mtx_spin },
+#endif
#ifdef __i386__
{ "allpmaps", &lock_class_mtx_spin },
{ "descriptor tables", &lock_class_mtx_spin },
diff --git a/sys/sparc64/include/smp.h b/sys/sparc64/include/smp.h
index 2e1110b..c46c4f8 100644
--- a/sys/sparc64/include/smp.h
+++ b/sys/sparc64/include/smp.h
@@ -39,13 +39,15 @@
#ifndef LOCORE
+#include <sys/param.h>
#include <sys/cpuset.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
#include <sys/proc.h>
#include <sys/sched.h>
#include <sys/smp.h>
#include <machine/intr_machdep.h>
-#include <machine/pcb.h>
#include <machine/tte.h>
#define IDR_BUSY 0x0000000000000001ULL
@@ -96,6 +98,7 @@ struct ipi_tlb_args {
};
#define ita_va ita_start
+struct pcb;
struct pcpu;
extern struct pcb stoppcbs[];
@@ -108,8 +111,9 @@ extern cpu_ipi_selected_t *cpu_ipi_selected;
typedef void cpu_ipi_single_t(u_int, u_long, u_long, u_long);
extern cpu_ipi_single_t *cpu_ipi_single;
-void mp_init(u_int cpu_impl);
+void mp_init(void);
+extern struct mtx ipi_mtx;
extern struct ipi_cache_args ipi_cache_args;
extern struct ipi_rd_args ipi_rd_args;
extern struct ipi_tlb_args ipi_tlb_args;
@@ -139,23 +143,37 @@ ipi_all_but_self(u_int ipi)
{
cpuset_t cpus;
+ if (__predict_false(smp_started == 0))
+ return;
cpus = all_cpus;
+ sched_pin();
CPU_CLR(PCPU_GET(cpuid), &cpus);
+ mtx_lock_spin(&ipi_mtx);
cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi);
+ mtx_unlock_spin(&ipi_mtx);
+ sched_unpin();
}
static __inline void
ipi_selected(cpuset_t cpus, u_int ipi)
{
+ if (__predict_false(smp_started == 0 || CPU_EMPTY(&cpus)))
+ return;
+ mtx_lock_spin(&ipi_mtx);
cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi);
+ mtx_unlock_spin(&ipi_mtx);
}
static __inline void
ipi_cpu(int cpu, u_int ipi)
{
+ if (__predict_false(smp_started == 0))
+ return;
+ mtx_lock_spin(&ipi_mtx);
cpu_ipi_single(cpu, 0, (u_long)tl_ipi_level, ipi);
+ mtx_unlock_spin(&ipi_mtx);
}
#if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_)
@@ -165,11 +183,11 @@ ipi_dcache_page_inval(void *func, vm_paddr_t pa)
{
struct ipi_cache_args *ica;
- if (smp_cpus == 1)
+ if (__predict_false(smp_started == 0))
return (NULL);
sched_pin();
ica = &ipi_cache_args;
- mtx_lock_spin(&smp_ipi_mtx);
+ mtx_lock_spin(&ipi_mtx);
ica->ica_mask = all_cpus;
CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask);
ica->ica_pa = pa;
@@ -182,11 +200,11 @@ ipi_icache_page_inval(void *func, vm_paddr_t pa)
{
struct ipi_cache_args *ica;
- if (smp_cpus == 1)
+ if (__predict_false(smp_started == 0))
return (NULL);
sched_pin();
ica = &ipi_cache_args;
- mtx_lock_spin(&smp_ipi_mtx);
+ mtx_lock_spin(&ipi_mtx);
ica->ica_mask = all_cpus;
CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask);
ica->ica_pa = pa;
@@ -199,11 +217,11 @@ ipi_rd(u_int cpu, void *func, u_long *val)
{
struct ipi_rd_args *ira;
- if (smp_cpus == 1)
+ if (__predict_false(smp_started == 0))
return (NULL);
sched_pin();
ira = &ipi_rd_args;
- mtx_lock_spin(&smp_ipi_mtx);
+ mtx_lock_spin(&ipi_mtx);
CPU_SETOF(cpu, &ira->ira_mask);
ira->ira_val = val;
cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira);
@@ -216,7 +234,7 @@ ipi_tlb_context_demap(struct pmap *pm)
struct ipi_tlb_args *ita;
cpuset_t cpus;
- if (smp_cpus == 1)
+ if (__predict_false(smp_started == 0))
return (NULL);
sched_pin();
cpus = pm->pm_active;
@@ -227,7 +245,7 @@ ipi_tlb_context_demap(struct pmap *pm)
return (NULL);
}
ita = &ipi_tlb_args;
- mtx_lock_spin(&smp_ipi_mtx);
+ mtx_lock_spin(&ipi_mtx);
ita->ita_mask = cpus;
ita->ita_pmap = pm;
cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_context_demap,
@@ -241,7 +259,7 @@ ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va)
struct ipi_tlb_args *ita;
cpuset_t cpus;
- if (smp_cpus == 1)
+ if (__predict_false(smp_started == 0))
return (NULL);
sched_pin();
cpus = pm->pm_active;
@@ -252,7 +270,7 @@ ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va)
return (NULL);
}
ita = &ipi_tlb_args;
- mtx_lock_spin(&smp_ipi_mtx);
+ mtx_lock_spin(&ipi_mtx);
ita->ita_mask = cpus;
ita->ita_pmap = pm;
ita->ita_va = va;
@@ -266,7 +284,7 @@ ipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end)
struct ipi_tlb_args *ita;
cpuset_t cpus;
- if (smp_cpus == 1)
+ if (__predict_false(smp_started == 0))
return (NULL);
sched_pin();
cpus = pm->pm_active;
@@ -277,7 +295,7 @@ ipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end)
return (NULL);
}
ita = &ipi_tlb_args;
- mtx_lock_spin(&smp_ipi_mtx);
+ mtx_lock_spin(&ipi_mtx);
ita->ita_mask = cpus;
ita->ita_pmap = pm;
ita->ita_start = start;
@@ -292,10 +310,10 @@ ipi_wait(void *cookie)
{
volatile cpuset_t *mask;
- if ((mask = cookie) != NULL) {
+ if (__predict_false((mask = cookie) != NULL)) {
while (!CPU_EMPTY(mask))
;
- mtx_unlock_spin(&smp_ipi_mtx);
+ mtx_unlock_spin(&ipi_mtx);
sched_unpin();
}
}
diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c
index af8c619..f016eaa 100644
--- a/sys/sparc64/sparc64/machdep.c
+++ b/sys/sparc64/sparc64/machdep.c
@@ -503,7 +503,7 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec)
}
#ifdef SMP
- mp_init(cpu_impl);
+ mp_init();
#endif
/*
diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index 0f977b3..c2c4e3e 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$");
#include <machine/atomic.h>
#include <machine/bus.h>
#include <machine/cpu.h>
+#include <machine/cpufunc.h>
#include <machine/md_var.h>
#include <machine/metadata.h>
#include <machine/ofw_machdep.h>
@@ -113,12 +114,13 @@ struct ipi_rd_args ipi_rd_args;
struct ipi_tlb_args ipi_tlb_args;
struct pcb stoppcbs[MAXCPU];
+struct mtx ipi_mtx;
+
cpu_ipi_selected_t *cpu_ipi_selected;
cpu_ipi_single_t *cpu_ipi_single;
static vm_offset_t mp_tramp;
static u_int cpuid_to_mid[MAXCPU];
-static int isjbus;
static volatile cpuset_t shutdown_cpus;
static void ap_count(phandle_t node, u_int mid, u_int cpu_impl);
@@ -138,7 +140,7 @@ static cpu_ipi_single_t spitfire_ipi_single;
SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL);
void
-mp_init(u_int cpu_impl)
+mp_init(void)
{
struct tte *tp;
int i;
@@ -157,24 +159,6 @@ mp_init(u_int cpu_impl)
}
for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t))
flush(mp_tramp + i);
-
- /*
- * On UP systems cpu_ipi_selected() can be called while
- * cpu_mp_start() wasn't so initialize these here.
- */
- if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi ||
- cpu_impl == CPU_IMPL_ULTRASPARCIIIip) {
- isjbus = 1;
- cpu_ipi_selected = jalapeno_ipi_selected;
- cpu_ipi_single = jalapeno_ipi_single;
- } else if (cpu_impl == CPU_IMPL_SPARC64V ||
- cpu_impl >= CPU_IMPL_ULTRASPARCIII) {
- cpu_ipi_selected = cheetah_ipi_selected;
- cpu_ipi_single = cheetah_ipi_single;
- } else {
- cpu_ipi_selected = spitfire_ipi_selected;
- cpu_ipi_single = spitfire_ipi_single;
- }
}
static void
@@ -219,7 +203,7 @@ foreach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid,
* Probe for other CPUs.
*/
void
-cpu_mp_setmaxid()
+cpu_mp_setmaxid(void)
{
CPU_SETOF(curcpu, &all_cpus);
@@ -277,6 +261,25 @@ sun4u_startcpu(phandle_t cpu, void *func, u_long arg)
void
cpu_mp_start(void)
{
+ u_int cpu_impl, isjbus;
+
+ mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN);
+
+ isjbus = 0;
+ cpu_impl = PCPU_GET(impl);
+ if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi ||
+ cpu_impl == CPU_IMPL_ULTRASPARCIIIip) {
+ isjbus = 1;
+ cpu_ipi_selected = jalapeno_ipi_selected;
+ cpu_ipi_single = jalapeno_ipi_single;
+ } else if (cpu_impl == CPU_IMPL_SPARC64V ||
+ cpu_impl >= CPU_IMPL_ULTRASPARCIII) {
+ cpu_ipi_selected = cheetah_ipi_selected;
+ cpu_ipi_single = cheetah_ipi_single;
+ } else {
+ cpu_ipi_selected = spitfire_ipi_selected;
+ cpu_ipi_single = spitfire_ipi_single;
+ }
intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL);
intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action,
@@ -360,7 +363,7 @@ cpu_mp_announce(void)
}
static void
-cpu_mp_unleash(void *v)
+cpu_mp_unleash(void *v __unused)
{
volatile struct cpu_start_args *csa;
struct pcpu *pc;
@@ -407,7 +410,6 @@ cpu_mp_unleash(void *v)
membar(StoreLoad);
csa->csa_count = 0;
- smp_started = 1;
}
void
@@ -464,6 +466,9 @@ cpu_mp_bootstrap(struct pcpu *pc)
while (csa->csa_count != 0)
;
+ if (smp_cpus == mp_ncpus)
+ atomic_store_rel_int(&smp_started, 1);
+
/* Start per-CPU event timers. */
cpu_initclocks_ap();
@@ -530,7 +535,7 @@ cpu_ipi_stop(struct trapframe *tf __unused)
}
static void
-cpu_ipi_preempt(struct trapframe *tf)
+cpu_ipi_preempt(struct trapframe *tf __unused)
{
sched_preempt(curthread);
@@ -573,9 +578,11 @@ spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2)
u_int mid;
int i;
+ mtx_assert(&ipi_mtx, MA_OWNED);
KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__));
KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0,
("%s: outstanding dispatch", __func__));
+
mid = cpuid_to_mid[cpu];
for (i = 0; i < IPI_RETRIES; i++) {
s = intr_disable();
@@ -601,12 +608,6 @@ spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2)
intr_restore(s);
if ((ids & (IDR_BUSY | IDR_NACK)) == 0)
return;
- /*
- * Leave interrupts enabled for a bit before retrying
- * in order to avoid deadlocks if the other CPU is also
- * trying to send an IPI.
- */
- DELAY(2);
}
if (kdb_active != 0 || panicstr != NULL)
printf("%s: couldn't send IPI to module 0x%u\n",
@@ -624,10 +625,12 @@ cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2)
u_int mid;
int i;
+ mtx_assert(&ipi_mtx, MA_OWNED);
KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__));
KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) &
IDR_CHEETAH_ALL_BUSY) == 0,
("%s: outstanding dispatch", __func__));
+
mid = cpuid_to_mid[cpu];
for (i = 0; i < IPI_RETRIES; i++) {
s = intr_disable();
@@ -644,12 +647,6 @@ cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2)
intr_restore(s);
if ((ids & (IDR_BUSY | IDR_NACK)) == 0)
return;
- /*
- * Leave interrupts enabled for a bit before retrying
- * in order to avoid deadlocks if the other CPU is also
- * trying to send an IPI.
- */
- DELAY(2);
}
if (kdb_active != 0 || panicstr != NULL)
printf("%s: couldn't send IPI to module 0x%u\n",
@@ -669,13 +666,14 @@ cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2)
u_int cpu;
int i;
+ mtx_assert(&ipi_mtx, MA_OWNED);
+ KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__));
KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself",
__func__));
KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) &
IDR_CHEETAH_ALL_BUSY) == 0,
("%s: outstanding dispatch", __func__));
- if (CPU_EMPTY(&cpus))
- return;
+
ids = 0;
for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) {
s = intr_disable();
@@ -709,12 +707,6 @@ cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2)
}
if (CPU_EMPTY(&cpus))
return;
- /*
- * Leave interrupts enabled for a bit before retrying
- * in order to avoid deadlocks if the other CPUs are
- * also trying to send IPIs.
- */
- DELAY(2 * mp_ncpus);
}
if (kdb_active != 0 || panicstr != NULL)
printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n",
@@ -732,10 +724,12 @@ jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2)
u_int busy, busynack, mid;
int i;
+ mtx_assert(&ipi_mtx, MA_OWNED);
KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__));
KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) &
IDR_CHEETAH_ALL_BUSY) == 0,
("%s: outstanding dispatch", __func__));
+
mid = cpuid_to_mid[cpu];
busy = IDR_BUSY << (2 * mid);
busynack = (IDR_BUSY | IDR_NACK) << (2 * mid);
@@ -754,12 +748,6 @@ jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2)
intr_restore(s);
if ((ids & busynack) == 0)
return;
- /*
- * Leave interrupts enabled for a bit before retrying
- * in order to avoid deadlocks if the other CPU is also
- * trying to send an IPI.
- */
- DELAY(2);
}
if (kdb_active != 0 || panicstr != NULL)
printf("%s: couldn't send IPI to module 0x%u\n",
@@ -778,13 +766,14 @@ jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2)
u_int cpu;
int i;
+ mtx_assert(&ipi_mtx, MA_OWNED);
+ KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__));
KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself",
__func__));
KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) &
IDR_CHEETAH_ALL_BUSY) == 0,
("%s: outstanding dispatch", __func__));
- if (CPU_EMPTY(&cpus))
- return;
+
ids = 0;
for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) {
s = intr_disable();
@@ -811,12 +800,6 @@ jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2)
if ((ids & (IDR_NACK <<
(2 * cpuid_to_mid[cpu]))) == 0)
CPU_CLR(cpu, &cpus);
- /*
- * Leave interrupts enabled for a bit before retrying
- * in order to avoid deadlocks if the other CPUs are
- * also trying to send IPIs.
- */
- DELAY(2 * mp_ncpus);
}
if (kdb_active != 0 || panicstr != NULL)
printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n",
diff --git a/sys/sparc64/sparc64/tick.c b/sys/sparc64/sparc64/tick.c
index 130efa6..2072b18 100644
--- a/sys/sparc64/sparc64/tick.c
+++ b/sys/sparc64/sparc64/tick.c
@@ -31,8 +31,6 @@ __FBSDID("$FreeBSD$");
#include <sys/param.h>
#include <sys/systm.h>
#include <sys/kernel.h>
-#include <sys/lock.h>
-#include <sys/mutex.h>
#include <sys/pcpu.h>
#include <sys/proc.h>
#include <sys/sched.h>
@@ -46,7 +44,6 @@ __FBSDID("$FreeBSD$");
#include <vm/vm.h>
#include <vm/pmap.h>
-#include <machine/cpu.h>
#include <machine/frame.h>
#include <machine/intr_machdep.h>
#include <machine/smp.h>
@@ -326,7 +323,7 @@ tick_get_timecount_up(struct timecounter *tc)
static u_int
stick_get_timecount_mp(struct timecounter *tc)
{
- u_long stick;
+ static u_long stick;
sched_pin();
if (curcpu == 0)
@@ -340,7 +337,7 @@ stick_get_timecount_mp(struct timecounter *tc)
static u_int
tick_get_timecount_mp(struct timecounter *tc)
{
- u_long tick;
+ static u_long tick;
sched_pin();
if (curcpu == 0)
OpenPOWER on IntegriCloud