diff options
Diffstat (limited to 'sys/kern/subr_smp.c')
| -rw-r--r-- | sys/kern/subr_smp.c | 53 |
1 files changed, 37 insertions, 16 deletions
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index a884288..bc30b2a 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -104,10 +104,10 @@ SYSCTL_INT(_kern_smp, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, "Forwarding of roundrobin to all other CPUs"); /* Variables needed for SMP rendezvous. */ -static void (*smp_rv_setup_func)(void *arg); -static void (*smp_rv_action_func)(void *arg); -static void (*smp_rv_teardown_func)(void *arg); -static void *smp_rv_func_arg; +static void (*volatile smp_rv_setup_func)(void *arg); +static void (*volatile smp_rv_action_func)(void *arg); +static void (* volatile smp_rv_teardown_func)(void *arg); +static void * volatile smp_rv_func_arg; static volatile int smp_rv_waiters[3]; /* @@ -286,6 +286,13 @@ restart_cpus(cpumask_t map) return 1; } +void smp_no_rendevous_barrier(void *dummy) +{ +#ifdef SMP + KASSERT((!smp_started),("smp_no_rendevous called and smp is started")); +#endif +} + /* * All-CPU rendezvous. CPUs are signalled, all execute the setup function * (if specified), rendezvous, execute the action function (if specified), @@ -298,33 +305,41 @@ restart_cpus(cpumask_t map) void smp_rendezvous_action(void) { - + void* local_func_arg = smp_rv_func_arg; + void (*local_setup_func)(void*) = smp_rv_setup_func; + void (*local_action_func)(void*) = smp_rv_action_func; + void (*local_teardown_func)(void*) = smp_rv_teardown_func; + /* Ensure we have up-to-date values. */ atomic_add_acq_int(&smp_rv_waiters[0], 1); while (smp_rv_waiters[0] < mp_ncpus) cpu_spinwait(); /* setup function */ - if (smp_rv_setup_func != NULL) - smp_rv_setup_func(smp_rv_func_arg); + if (local_setup_func != smp_no_rendevous_barrier) { + if (smp_rv_setup_func != NULL) + smp_rv_setup_func(smp_rv_func_arg); + /* spin on entry rendezvous */ + atomic_add_int(&smp_rv_waiters[1], 1); + while (smp_rv_waiters[1] < mp_ncpus) + cpu_spinwait(); + } - /* spin on entry rendezvous */ - atomic_add_int(&smp_rv_waiters[1], 1); - while (smp_rv_waiters[1] < mp_ncpus) - cpu_spinwait(); /* action function */ - if (smp_rv_action_func != NULL) - smp_rv_action_func(smp_rv_func_arg); + if (local_action_func != NULL) + local_action_func(local_func_arg); + /* spin on exit rendezvous */ atomic_add_int(&smp_rv_waiters[2], 1); + if (local_teardown_func == smp_no_rendevous_barrier) + return; while (smp_rv_waiters[2] < mp_ncpus) cpu_spinwait(); - /* teardown function */ - if (smp_rv_teardown_func != NULL) - smp_rv_teardown_func(smp_rv_func_arg); + if (local_teardown_func != NULL) + local_teardown_func(local_func_arg); } void @@ -356,12 +371,18 @@ smp_rendezvous(void (* setup_func)(void *), smp_rv_waiters[2] = 0; atomic_store_rel_int(&smp_rv_waiters[0], 0); + + /* signal other processors, which will enter the IPI with interrupts off */ ipi_all_but_self(IPI_RENDEZVOUS); /* call executor function */ smp_rendezvous_action(); + if (teardown_func == smp_no_rendevous_barrier) { + while (atomic_load_acq_int(&smp_rv_waiters[2]) < mp_ncpus) + cpu_spinwait(); + } /* release lock */ mtx_unlock_spin(&smp_ipi_mtx); } |
