diff options
author | msmith <msmith@FreeBSD.org> | 1999-07-20 06:52:35 +0000 |
---|---|---|
committer | msmith <msmith@FreeBSD.org> | 1999-07-20 06:52:35 +0000 |
commit | 84c939ef118a545b74e42d1ab9013ea9ba8b1b3f (patch) | |
tree | f285b114a76c38187b7d9910bdede9788d00ae9e /sys/amd64 | |
parent | 905bcc118433dc6747bf9091a2bae04370334c34 (diff) | |
download | FreeBSD-src-84c939ef118a545b74e42d1ab9013ea9ba8b1b3f.zip FreeBSD-src-84c939ef118a545b74e42d1ab9013ea9ba8b1b3f.tar.gz |
Implement an all-CPU shootdown-style rendezvous facility. This allows
the caller to specify a function to be guarded between an entry and exit
barrier, as well as pre- and post-barrier functions.
The primary use for this function is synchronised update of per-cpu private
data. The implementation is almost (but not quite) MI; with a better
mechanism for masking per-CPU interrupts it could probably be hoisted.
Reviewed by: peter (partially)
Diffstat (limited to 'sys/amd64')
-rw-r--r-- | sys/amd64/amd64/apic_vector.S | 25 | ||||
-rw-r--r-- | sys/amd64/amd64/mp_machdep.c | 86 | ||||
-rw-r--r-- | sys/amd64/amd64/mptable.c | 86 | ||||
-rw-r--r-- | sys/amd64/include/mptable.h | 86 | ||||
-rw-r--r-- | sys/amd64/include/smp.h | 7 | ||||
-rw-r--r-- | sys/amd64/isa/intr_machdep.h | 8 |
6 files changed, 291 insertions, 7 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index b99b93a..13b116b 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.41 1999/07/03 06:33:47 alc Exp $ + * $Id: apic_vector.s,v 1.42 1999/07/10 15:27:59 bde Exp $ */ @@ -952,6 +952,29 @@ MCOUNT_LABEL(bintr) INTR(23,intr23,) MCOUNT_LABEL(eintr) +/* + * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU. + * + * - Calls the generic rendezvous action function. + */ + .text + SUPERALIGN_TEXT + .globl _Xrendezvous +_Xrendezvous: + PUSH_FRAME + movl $KDSEL, %eax + movl %ax, %ds /* use KERNEL data segment */ + movl %ax, %es + movl $KPSEL, %eax + movl %ax, %fs + + call smp_rendezvous_action + + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + POP_FRAME + iret + + .data /* * Addresses of interrupt handlers. diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 9d8be53..e81efef 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.104 1999/06/23 21:47:22 luoqi Exp $ + * $Id: mp_machdep.c,v 1.105 1999/06/23 23:02:38 msmith Exp $ */ #include "opt_smp.h" @@ -62,7 +62,10 @@ #include <machine/smp.h> #include <machine/apic.h> +#include <machine/atomic.h> +#include <machine/cpufunc.h> #include <machine/mpapic.h> +#include <machine/psl.h> #include <machine/segments.h> #include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */ #include <machine/tss.h> @@ -591,6 +594,10 @@ mp_enable(u_int boot_addr) SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif + /* install an inter-CPU IPI for all-CPU rendezvous */ + setidt(XRENDEZVOUS_OFFSET, Xrendezvous, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); @@ -1698,6 +1705,9 @@ struct simplelock com_lock; struct simplelock clock_lock; #endif /* USE_CLOCKLOCK */ +/* lock around the MP rendezvous */ +static struct simplelock smp_rv_lock; + static void init_locks(void) { @@ -1722,6 +1732,7 @@ init_locks(void) s_lock_init((struct simplelock*)&intr_lock); s_lock_init((struct simplelock*)&imen_lock); s_lock_init((struct simplelock*)&cpl_lock); + s_lock_init(&smp_rv_lock); #ifdef USE_COMLOCK s_lock_init((struct simplelock*)&com_lock); @@ -2604,3 +2615,76 @@ set_lapic_isrloc(int intr, int vector) apic_isrbit_location[intr].bit = (1<<(vector & 31)); } #endif + +/* + * All-CPU rendezvous. CPUs are signalled, all execute the setup function + * (if specified), rendezvous, execute the action function (if specified), + * rendezvous again, execute the teardown function (if specified), and then + * resume. + * + * Note that the supplied external functions _must_ be reentrant and aware + * that they are running in parallel and in an unknown lock context. + */ +static void (*smp_rv_setup_func)(void *arg); +static void (*smp_rv_action_func)(void *arg); +static void (*smp_rv_teardown_func)(void *arg); +static void *smp_rv_func_arg; +static volatile int smp_rv_waiters[2]; + +void +smp_rendezvous_action(void) +{ + /* setup function */ + if (smp_rv_setup_func != NULL) + smp_rv_setup_func(smp_rv_func_arg); + /* spin on entry rendezvous */ + atomic_add_int(&smp_rv_waiters[0], 1); + while (smp_rv_waiters[0] < mp_ncpus) + ; + /* action function */ + if (smp_rv_action_func != NULL) + smp_rv_action_func(smp_rv_func_arg); + /* spin on exit rendezvous */ + atomic_add_int(&smp_rv_waiters[1], 1); + while (smp_rv_waiters[1] < mp_ncpus) + ; + /* teardown function */ + if (smp_rv_teardown_func != NULL) + smp_rv_teardown_func(smp_rv_func_arg); +} + +void +smp_rendezvous(void (* setup_func)(void *), + void (* action_func)(void *), + void (* teardown_func)(void *), + void *arg) +{ + u_int efl; + + /* obtain rendezvous lock */ + s_lock(&smp_rv_lock); /* XXX sleep here? NOWAIT flag? */ + + /* set static function pointers */ + smp_rv_setup_func = setup_func; + smp_rv_action_func = action_func; + smp_rv_teardown_func = teardown_func; + smp_rv_func_arg = arg; + smp_rv_waiters[0] = 0; + smp_rv_waiters[1] = 0; + + /* disable interrupts on this CPU, save interrupt status */ + efl = read_eflags(); + write_eflags(efl & ~PSL_I); + + /* signal other processors, which will enter the IPI with interrupts off */ + all_but_self_ipi(XRENDEZVOUS_OFFSET); + + /* call executor function */ + smp_rendezvous_action(); + + /* restore interrupt flag */ + write_eflags(efl); + + /* release lock */ + s_unlock(&smp_rv_lock); +} diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c index 9d8be53..e81efef 100644 --- a/sys/amd64/amd64/mptable.c +++ b/sys/amd64/amd64/mptable.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.104 1999/06/23 21:47:22 luoqi Exp $ + * $Id: mp_machdep.c,v 1.105 1999/06/23 23:02:38 msmith Exp $ */ #include "opt_smp.h" @@ -62,7 +62,10 @@ #include <machine/smp.h> #include <machine/apic.h> +#include <machine/atomic.h> +#include <machine/cpufunc.h> #include <machine/mpapic.h> +#include <machine/psl.h> #include <machine/segments.h> #include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */ #include <machine/tss.h> @@ -591,6 +594,10 @@ mp_enable(u_int boot_addr) SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif + /* install an inter-CPU IPI for all-CPU rendezvous */ + setidt(XRENDEZVOUS_OFFSET, Xrendezvous, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); @@ -1698,6 +1705,9 @@ struct simplelock com_lock; struct simplelock clock_lock; #endif /* USE_CLOCKLOCK */ +/* lock around the MP rendezvous */ +static struct simplelock smp_rv_lock; + static void init_locks(void) { @@ -1722,6 +1732,7 @@ init_locks(void) s_lock_init((struct simplelock*)&intr_lock); s_lock_init((struct simplelock*)&imen_lock); s_lock_init((struct simplelock*)&cpl_lock); + s_lock_init(&smp_rv_lock); #ifdef USE_COMLOCK s_lock_init((struct simplelock*)&com_lock); @@ -2604,3 +2615,76 @@ set_lapic_isrloc(int intr, int vector) apic_isrbit_location[intr].bit = (1<<(vector & 31)); } #endif + +/* + * All-CPU rendezvous. CPUs are signalled, all execute the setup function + * (if specified), rendezvous, execute the action function (if specified), + * rendezvous again, execute the teardown function (if specified), and then + * resume. + * + * Note that the supplied external functions _must_ be reentrant and aware + * that they are running in parallel and in an unknown lock context. + */ +static void (*smp_rv_setup_func)(void *arg); +static void (*smp_rv_action_func)(void *arg); +static void (*smp_rv_teardown_func)(void *arg); +static void *smp_rv_func_arg; +static volatile int smp_rv_waiters[2]; + +void +smp_rendezvous_action(void) +{ + /* setup function */ + if (smp_rv_setup_func != NULL) + smp_rv_setup_func(smp_rv_func_arg); + /* spin on entry rendezvous */ + atomic_add_int(&smp_rv_waiters[0], 1); + while (smp_rv_waiters[0] < mp_ncpus) + ; + /* action function */ + if (smp_rv_action_func != NULL) + smp_rv_action_func(smp_rv_func_arg); + /* spin on exit rendezvous */ + atomic_add_int(&smp_rv_waiters[1], 1); + while (smp_rv_waiters[1] < mp_ncpus) + ; + /* teardown function */ + if (smp_rv_teardown_func != NULL) + smp_rv_teardown_func(smp_rv_func_arg); +} + +void +smp_rendezvous(void (* setup_func)(void *), + void (* action_func)(void *), + void (* teardown_func)(void *), + void *arg) +{ + u_int efl; + + /* obtain rendezvous lock */ + s_lock(&smp_rv_lock); /* XXX sleep here? NOWAIT flag? */ + + /* set static function pointers */ + smp_rv_setup_func = setup_func; + smp_rv_action_func = action_func; + smp_rv_teardown_func = teardown_func; + smp_rv_func_arg = arg; + smp_rv_waiters[0] = 0; + smp_rv_waiters[1] = 0; + + /* disable interrupts on this CPU, save interrupt status */ + efl = read_eflags(); + write_eflags(efl & ~PSL_I); + + /* signal other processors, which will enter the IPI with interrupts off */ + all_but_self_ipi(XRENDEZVOUS_OFFSET); + + /* call executor function */ + smp_rendezvous_action(); + + /* restore interrupt flag */ + write_eflags(efl); + + /* release lock */ + s_unlock(&smp_rv_lock); +} diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h index 9d8be53..e81efef 100644 --- a/sys/amd64/include/mptable.h +++ b/sys/amd64/include/mptable.h @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.104 1999/06/23 21:47:22 luoqi Exp $ + * $Id: mp_machdep.c,v 1.105 1999/06/23 23:02:38 msmith Exp $ */ #include "opt_smp.h" @@ -62,7 +62,10 @@ #include <machine/smp.h> #include <machine/apic.h> +#include <machine/atomic.h> +#include <machine/cpufunc.h> #include <machine/mpapic.h> +#include <machine/psl.h> #include <machine/segments.h> #include <machine/smptests.h> /** TEST_DEFAULT_CONFIG, TEST_TEST1 */ #include <machine/tss.h> @@ -591,6 +594,10 @@ mp_enable(u_int boot_addr) SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); #endif + /* install an inter-CPU IPI for all-CPU rendezvous */ + setidt(XRENDEZVOUS_OFFSET, Xrendezvous, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); + /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); @@ -1698,6 +1705,9 @@ struct simplelock com_lock; struct simplelock clock_lock; #endif /* USE_CLOCKLOCK */ +/* lock around the MP rendezvous */ +static struct simplelock smp_rv_lock; + static void init_locks(void) { @@ -1722,6 +1732,7 @@ init_locks(void) s_lock_init((struct simplelock*)&intr_lock); s_lock_init((struct simplelock*)&imen_lock); s_lock_init((struct simplelock*)&cpl_lock); + s_lock_init(&smp_rv_lock); #ifdef USE_COMLOCK s_lock_init((struct simplelock*)&com_lock); @@ -2604,3 +2615,76 @@ set_lapic_isrloc(int intr, int vector) apic_isrbit_location[intr].bit = (1<<(vector & 31)); } #endif + +/* + * All-CPU rendezvous. CPUs are signalled, all execute the setup function + * (if specified), rendezvous, execute the action function (if specified), + * rendezvous again, execute the teardown function (if specified), and then + * resume. + * + * Note that the supplied external functions _must_ be reentrant and aware + * that they are running in parallel and in an unknown lock context. + */ +static void (*smp_rv_setup_func)(void *arg); +static void (*smp_rv_action_func)(void *arg); +static void (*smp_rv_teardown_func)(void *arg); +static void *smp_rv_func_arg; +static volatile int smp_rv_waiters[2]; + +void +smp_rendezvous_action(void) +{ + /* setup function */ + if (smp_rv_setup_func != NULL) + smp_rv_setup_func(smp_rv_func_arg); + /* spin on entry rendezvous */ + atomic_add_int(&smp_rv_waiters[0], 1); + while (smp_rv_waiters[0] < mp_ncpus) + ; + /* action function */ + if (smp_rv_action_func != NULL) + smp_rv_action_func(smp_rv_func_arg); + /* spin on exit rendezvous */ + atomic_add_int(&smp_rv_waiters[1], 1); + while (smp_rv_waiters[1] < mp_ncpus) + ; + /* teardown function */ + if (smp_rv_teardown_func != NULL) + smp_rv_teardown_func(smp_rv_func_arg); +} + +void +smp_rendezvous(void (* setup_func)(void *), + void (* action_func)(void *), + void (* teardown_func)(void *), + void *arg) +{ + u_int efl; + + /* obtain rendezvous lock */ + s_lock(&smp_rv_lock); /* XXX sleep here? NOWAIT flag? */ + + /* set static function pointers */ + smp_rv_setup_func = setup_func; + smp_rv_action_func = action_func; + smp_rv_teardown_func = teardown_func; + smp_rv_func_arg = arg; + smp_rv_waiters[0] = 0; + smp_rv_waiters[1] = 0; + + /* disable interrupts on this CPU, save interrupt status */ + efl = read_eflags(); + write_eflags(efl & ~PSL_I); + + /* signal other processors, which will enter the IPI with interrupts off */ + all_but_self_ipi(XRENDEZVOUS_OFFSET); + + /* call executor function */ + smp_rendezvous_action(); + + /* restore interrupt flag */ + write_eflags(efl); + + /* release lock */ + s_unlock(&smp_rv_lock); +} diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 3750c72..c425db6 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -6,7 +6,7 @@ * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * - * $Id: smp.h,v 1.44 1998/09/06 22:41:40 tegge Exp $ + * $Id: smp.h,v 1.45 1999/04/28 01:04:07 luoqi Exp $ * */ @@ -146,6 +146,11 @@ void forward_roundrobin __P((void)); #ifdef APIC_INTR_REORDER void set_lapic_isrloc __P((int, int)); #endif /* APIC_INTR_REORDER */ +void smp_rendezvous_action __P((void)); +void smp_rendezvous __P((void (*)(void *), + void (*)(void *), + void (*)(void *), + void *arg)); /* global data in mpapic.c */ extern volatile lapic_t lapic; diff --git a/sys/amd64/isa/intr_machdep.h b/sys/amd64/isa/intr_machdep.h index 675714d..06593cf 100644 --- a/sys/amd64/isa/intr_machdep.h +++ b/sys/amd64/isa/intr_machdep.h @@ -30,7 +30,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: intr_machdep.h,v 1.14 1999/04/21 07:26:27 peter Exp $ + * $Id: intr_machdep.h,v 1.15 1999/06/03 20:41:00 peter Exp $ */ #ifndef _I386_ISA_INTR_MACHDEP_H_ @@ -112,6 +112,9 @@ #define XCPUCHECKSTATE_OFFSET (ICU_OFFSET + 113) #endif +/* inter-CPU rendezvous */ +#define XRENDEZVOUS_OFFSET (ICU_OFFSET + 114) + /* IPI to generate an additional software trap at the target CPU */ #define XCPUAST_OFFSET (ICU_OFFSET + 48) @@ -178,7 +181,8 @@ inthand_t Xcpuast, /* Additional software trap on other cpu */ Xforward_irq, /* Forward irq to cpu holding ISR lock */ Xcpustop, /* CPU stops & waits for another CPU to restart it */ - Xspuriousint; /* handle APIC "spurious INTs" */ + Xspuriousint, /* handle APIC "spurious INTs" */ + Xrendezvous; /* handle CPU rendezvous */ #ifdef TEST_TEST1 inthand_t |