diff options
| -rw-r--r-- | sys/kern/subr_witness.c | 3 | ||||
| -rw-r--r-- | sys/sparc64/include/smp.h | 50 | ||||
| -rw-r--r-- | sys/sparc64/sparc64/machdep.c | 2 | ||||
| -rw-r--r-- | sys/sparc64/sparc64/mp_machdep.c | 99 | ||||
| -rw-r--r-- | sys/sparc64/sparc64/tick.c | 7 |
5 files changed, 81 insertions, 80 deletions
diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index ab494ce..70e2ca9 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -661,6 +661,9 @@ static struct witness_order_list_entry order_lists[] = { */ { "intrcnt", &lock_class_mtx_spin }, { "icu", &lock_class_mtx_spin }, +#if defined(SMP) && defined(__sparc64__) + { "ipi", &lock_class_mtx_spin }, +#endif #ifdef __i386__ { "allpmaps", &lock_class_mtx_spin }, { "descriptor tables", &lock_class_mtx_spin }, diff --git a/sys/sparc64/include/smp.h b/sys/sparc64/include/smp.h index 2e1110b..c46c4f8 100644 --- a/sys/sparc64/include/smp.h +++ b/sys/sparc64/include/smp.h @@ -39,13 +39,15 @@ #ifndef LOCORE +#include <sys/param.h> #include <sys/cpuset.h> +#include <sys/lock.h> +#include <sys/mutex.h> #include <sys/proc.h> #include <sys/sched.h> #include <sys/smp.h> #include <machine/intr_machdep.h> -#include <machine/pcb.h> #include <machine/tte.h> #define IDR_BUSY 0x0000000000000001ULL @@ -96,6 +98,7 @@ struct ipi_tlb_args { }; #define ita_va ita_start +struct pcb; struct pcpu; extern struct pcb stoppcbs[]; @@ -108,8 +111,9 @@ extern cpu_ipi_selected_t *cpu_ipi_selected; typedef void cpu_ipi_single_t(u_int, u_long, u_long, u_long); extern cpu_ipi_single_t *cpu_ipi_single; -void mp_init(u_int cpu_impl); +void mp_init(void); +extern struct mtx ipi_mtx; extern struct ipi_cache_args ipi_cache_args; extern struct ipi_rd_args ipi_rd_args; extern struct ipi_tlb_args ipi_tlb_args; @@ -139,23 +143,37 @@ ipi_all_but_self(u_int ipi) { cpuset_t cpus; + if (__predict_false(smp_started == 0)) + return; cpus = all_cpus; + sched_pin(); CPU_CLR(PCPU_GET(cpuid), &cpus); + mtx_lock_spin(&ipi_mtx); cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); + mtx_unlock_spin(&ipi_mtx); + sched_unpin(); } static __inline void ipi_selected(cpuset_t cpus, u_int ipi) { + if (__predict_false(smp_started == 0 || CPU_EMPTY(&cpus))) + return; + mtx_lock_spin(&ipi_mtx); cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_level, ipi); + mtx_unlock_spin(&ipi_mtx); } static __inline void ipi_cpu(int cpu, u_int ipi) { + if (__predict_false(smp_started == 0)) + return; + mtx_lock_spin(&ipi_mtx); cpu_ipi_single(cpu, 0, (u_long)tl_ipi_level, ipi); + mtx_unlock_spin(&ipi_mtx); } #if defined(_MACHINE_PMAP_H_) && defined(_SYS_MUTEX_H_) @@ -165,11 +183,11 @@ ipi_dcache_page_inval(void *func, vm_paddr_t pa) { struct ipi_cache_args *ica; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); ica = &ipi_cache_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ica->ica_mask = all_cpus; CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); ica->ica_pa = pa; @@ -182,11 +200,11 @@ ipi_icache_page_inval(void *func, vm_paddr_t pa) { struct ipi_cache_args *ica; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); ica = &ipi_cache_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ica->ica_mask = all_cpus; CPU_CLR(PCPU_GET(cpuid), &ica->ica_mask); ica->ica_pa = pa; @@ -199,11 +217,11 @@ ipi_rd(u_int cpu, void *func, u_long *val) { struct ipi_rd_args *ira; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); ira = &ipi_rd_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); CPU_SETOF(cpu, &ira->ira_mask); ira->ira_val = val; cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira); @@ -216,7 +234,7 @@ ipi_tlb_context_demap(struct pmap *pm) struct ipi_tlb_args *ita; cpuset_t cpus; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); cpus = pm->pm_active; @@ -227,7 +245,7 @@ ipi_tlb_context_demap(struct pmap *pm) return (NULL); } ita = &ipi_tlb_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ita->ita_mask = cpus; ita->ita_pmap = pm; cpu_ipi_selected(cpus, 0, (u_long)tl_ipi_tlb_context_demap, @@ -241,7 +259,7 @@ ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va) struct ipi_tlb_args *ita; cpuset_t cpus; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); cpus = pm->pm_active; @@ -252,7 +270,7 @@ ipi_tlb_page_demap(struct pmap *pm, vm_offset_t va) return (NULL); } ita = &ipi_tlb_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ita->ita_mask = cpus; ita->ita_pmap = pm; ita->ita_va = va; @@ -266,7 +284,7 @@ ipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end) struct ipi_tlb_args *ita; cpuset_t cpus; - if (smp_cpus == 1) + if (__predict_false(smp_started == 0)) return (NULL); sched_pin(); cpus = pm->pm_active; @@ -277,7 +295,7 @@ ipi_tlb_range_demap(struct pmap *pm, vm_offset_t start, vm_offset_t end) return (NULL); } ita = &ipi_tlb_args; - mtx_lock_spin(&smp_ipi_mtx); + mtx_lock_spin(&ipi_mtx); ita->ita_mask = cpus; ita->ita_pmap = pm; ita->ita_start = start; @@ -292,10 +310,10 @@ ipi_wait(void *cookie) { volatile cpuset_t *mask; - if ((mask = cookie) != NULL) { + if (__predict_false((mask = cookie) != NULL)) { while (!CPU_EMPTY(mask)) ; - mtx_unlock_spin(&smp_ipi_mtx); + mtx_unlock_spin(&ipi_mtx); sched_unpin(); } } diff --git a/sys/sparc64/sparc64/machdep.c b/sys/sparc64/sparc64/machdep.c index a95bc63..f0e54a5 100644 --- a/sys/sparc64/sparc64/machdep.c +++ b/sys/sparc64/sparc64/machdep.c @@ -499,7 +499,7 @@ sparc64_init(caddr_t mdp, u_long o1, u_long o2, u_long o3, ofw_vec_t *vec) } #ifdef SMP - mp_init(cpu_impl); + mp_init(); #endif /* diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c index 0f977b3..c2c4e3e 100644 --- a/sys/sparc64/sparc64/mp_machdep.c +++ b/sys/sparc64/sparc64/mp_machdep.c @@ -82,6 +82,7 @@ __FBSDID("$FreeBSD$"); #include <machine/atomic.h> #include <machine/bus.h> #include <machine/cpu.h> +#include <machine/cpufunc.h> #include <machine/md_var.h> #include <machine/metadata.h> #include <machine/ofw_machdep.h> @@ -113,12 +114,13 @@ struct ipi_rd_args ipi_rd_args; struct ipi_tlb_args ipi_tlb_args; struct pcb stoppcbs[MAXCPU]; +struct mtx ipi_mtx; + cpu_ipi_selected_t *cpu_ipi_selected; cpu_ipi_single_t *cpu_ipi_single; static vm_offset_t mp_tramp; static u_int cpuid_to_mid[MAXCPU]; -static int isjbus; static volatile cpuset_t shutdown_cpus; static void ap_count(phandle_t node, u_int mid, u_int cpu_impl); @@ -138,7 +140,7 @@ static cpu_ipi_single_t spitfire_ipi_single; SYSINIT(cpu_mp_unleash, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); void -mp_init(u_int cpu_impl) +mp_init(void) { struct tte *tp; int i; @@ -157,24 +159,6 @@ mp_init(u_int cpu_impl) } for (i = 0; i < PAGE_SIZE; i += sizeof(vm_offset_t)) flush(mp_tramp + i); - - /* - * On UP systems cpu_ipi_selected() can be called while - * cpu_mp_start() wasn't so initialize these here. - */ - if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || - cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { - isjbus = 1; - cpu_ipi_selected = jalapeno_ipi_selected; - cpu_ipi_single = jalapeno_ipi_single; - } else if (cpu_impl == CPU_IMPL_SPARC64V || - cpu_impl >= CPU_IMPL_ULTRASPARCIII) { - cpu_ipi_selected = cheetah_ipi_selected; - cpu_ipi_single = cheetah_ipi_single; - } else { - cpu_ipi_selected = spitfire_ipi_selected; - cpu_ipi_single = spitfire_ipi_single; - } } static void @@ -219,7 +203,7 @@ foreach_ap(phandle_t node, void (*func)(phandle_t node, u_int mid, * Probe for other CPUs. */ void -cpu_mp_setmaxid() +cpu_mp_setmaxid(void) { CPU_SETOF(curcpu, &all_cpus); @@ -277,6 +261,25 @@ sun4u_startcpu(phandle_t cpu, void *func, u_long arg) void cpu_mp_start(void) { + u_int cpu_impl, isjbus; + + mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN); + + isjbus = 0; + cpu_impl = PCPU_GET(impl); + if (cpu_impl == CPU_IMPL_ULTRASPARCIIIi || + cpu_impl == CPU_IMPL_ULTRASPARCIIIip) { + isjbus = 1; + cpu_ipi_selected = jalapeno_ipi_selected; + cpu_ipi_single = jalapeno_ipi_single; + } else if (cpu_impl == CPU_IMPL_SPARC64V || + cpu_impl >= CPU_IMPL_ULTRASPARCIII) { + cpu_ipi_selected = cheetah_ipi_selected; + cpu_ipi_single = cheetah_ipi_single; + } else { + cpu_ipi_selected = spitfire_ipi_selected; + cpu_ipi_single = spitfire_ipi_single; + } intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL); intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action, @@ -360,7 +363,7 @@ cpu_mp_announce(void) } static void -cpu_mp_unleash(void *v) +cpu_mp_unleash(void *v __unused) { volatile struct cpu_start_args *csa; struct pcpu *pc; @@ -407,7 +410,6 @@ cpu_mp_unleash(void *v) membar(StoreLoad); csa->csa_count = 0; - smp_started = 1; } void @@ -464,6 +466,9 @@ cpu_mp_bootstrap(struct pcpu *pc) while (csa->csa_count != 0) ; + if (smp_cpus == mp_ncpus) + atomic_store_rel_int(&smp_started, 1); + /* Start per-CPU event timers. */ cpu_initclocks_ap(); @@ -530,7 +535,7 @@ cpu_ipi_stop(struct trapframe *tf __unused) } static void -cpu_ipi_preempt(struct trapframe *tf) +cpu_ipi_preempt(struct trapframe *tf __unused) { sched_preempt(curthread); @@ -573,9 +578,11 @@ spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) u_int mid; int i; + mtx_assert(&ipi_mtx, MA_OWNED); KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_BUSY) == 0, ("%s: outstanding dispatch", __func__)); + mid = cpuid_to_mid[cpu]; for (i = 0; i < IPI_RETRIES; i++) { s = intr_disable(); @@ -601,12 +608,6 @@ spitfire_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) intr_restore(s); if ((ids & (IDR_BUSY | IDR_NACK)) == 0) return; - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPU is also - * trying to send an IPI. - */ - DELAY(2); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI to module 0x%u\n", @@ -624,10 +625,12 @@ cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) u_int mid; int i; + mtx_assert(&ipi_mtx, MA_OWNED); KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_CHEETAH_ALL_BUSY) == 0, ("%s: outstanding dispatch", __func__)); + mid = cpuid_to_mid[cpu]; for (i = 0; i < IPI_RETRIES; i++) { s = intr_disable(); @@ -644,12 +647,6 @@ cheetah_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) intr_restore(s); if ((ids & (IDR_BUSY | IDR_NACK)) == 0) return; - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPU is also - * trying to send an IPI. - */ - DELAY(2); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI to module 0x%u\n", @@ -669,13 +666,14 @@ cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) u_int cpu; int i; + mtx_assert(&ipi_mtx, MA_OWNED); + KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", __func__)); KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_CHEETAH_ALL_BUSY) == 0, ("%s: outstanding dispatch", __func__)); - if (CPU_EMPTY(&cpus)) - return; + ids = 0; for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { s = intr_disable(); @@ -709,12 +707,6 @@ cheetah_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) } if (CPU_EMPTY(&cpus)) return; - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPUs are - * also trying to send IPIs. - */ - DELAY(2 * mp_ncpus); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", @@ -732,10 +724,12 @@ jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) u_int busy, busynack, mid; int i; + mtx_assert(&ipi_mtx, MA_OWNED); KASSERT(cpu != curcpu, ("%s: CPU can't IPI itself", __func__)); KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_CHEETAH_ALL_BUSY) == 0, ("%s: outstanding dispatch", __func__)); + mid = cpuid_to_mid[cpu]; busy = IDR_BUSY << (2 * mid); busynack = (IDR_BUSY | IDR_NACK) << (2 * mid); @@ -754,12 +748,6 @@ jalapeno_ipi_single(u_int cpu, u_long d0, u_long d1, u_long d2) intr_restore(s); if ((ids & busynack) == 0) return; - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPU is also - * trying to send an IPI. - */ - DELAY(2); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI to module 0x%u\n", @@ -778,13 +766,14 @@ jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) u_int cpu; int i; + mtx_assert(&ipi_mtx, MA_OWNED); + KASSERT(!CPU_EMPTY(&cpus), ("%s: no CPUs to IPI", __func__)); KASSERT(!CPU_ISSET(curcpu, &cpus), ("%s: CPU can't IPI itself", __func__)); KASSERT((ldxa(0, ASI_INTR_DISPATCH_STATUS) & IDR_CHEETAH_ALL_BUSY) == 0, ("%s: outstanding dispatch", __func__)); - if (CPU_EMPTY(&cpus)) - return; + ids = 0; for (i = 0; i < IPI_RETRIES * mp_ncpus; i++) { s = intr_disable(); @@ -811,12 +800,6 @@ jalapeno_ipi_selected(cpuset_t cpus, u_long d0, u_long d1, u_long d2) if ((ids & (IDR_NACK << (2 * cpuid_to_mid[cpu]))) == 0) CPU_CLR(cpu, &cpus); - /* - * Leave interrupts enabled for a bit before retrying - * in order to avoid deadlocks if the other CPUs are - * also trying to send IPIs. - */ - DELAY(2 * mp_ncpus); } if (kdb_active != 0 || panicstr != NULL) printf("%s: couldn't send IPI (cpus=%s ids=0x%lu)\n", diff --git a/sys/sparc64/sparc64/tick.c b/sys/sparc64/sparc64/tick.c index 130efa6..2072b18 100644 --- a/sys/sparc64/sparc64/tick.c +++ b/sys/sparc64/sparc64/tick.c @@ -31,8 +31,6 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> -#include <sys/lock.h> -#include <sys/mutex.h> #include <sys/pcpu.h> #include <sys/proc.h> #include <sys/sched.h> @@ -46,7 +44,6 @@ __FBSDID("$FreeBSD$"); #include <vm/vm.h> #include <vm/pmap.h> -#include <machine/cpu.h> #include <machine/frame.h> #include <machine/intr_machdep.h> #include <machine/smp.h> @@ -326,7 +323,7 @@ tick_get_timecount_up(struct timecounter *tc) static u_int stick_get_timecount_mp(struct timecounter *tc) { - u_long stick; + static u_long stick; sched_pin(); if (curcpu == 0) @@ -340,7 +337,7 @@ stick_get_timecount_mp(struct timecounter *tc) static u_int tick_get_timecount_mp(struct timecounter *tc) { - u_long tick; + static u_long tick; sched_pin(); if (curcpu == 0) |
