Implement an all-CPU shootdown-style rendezvous facility. This allows

the caller to specify a function to be guarded between an entry and exit
barrier, as well as pre- and post-barrier functions.

The primary use for this function is synchronised update of per-cpu private
data.  The implementation is almost (but not quite) MI; with a better
mechanism for masking per-CPU interrupts it could probably be hoisted.

Reviewed by:	peter (partially)

6 files changed, 291 insertions, 7 deletions

diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S
index b99b93a..13b116b 100644
--- a/sys/amd64/amd64/apic_vector.S
+++ b/sys/amd64/amd64/apic_vector.S
@@ -1,6 +1,6 @@
 /*
  *	from: vector.s, 386BSD 0.1 unknown origin
- *	$Id: apic_vector.s,v 1.41 1999/07/03 06:33:47 alc Exp $
+ *	$Id: apic_vector.s,v 1.42 1999/07/10 15:27:59 bde Exp $
  */
 
 
@@ -952,6 +952,29 @@ MCOUNT_LABEL(bintr)
 	INTR(23,intr23,)
 MCOUNT_LABEL(eintr)
 
+/*
+ * Executed by a CPU when it receives a RENDEZVOUS IPI from another CPU.
+ *
+ * - Calls the generic rendezvous action function.
+ */
+	.text
+	SUPERALIGN_TEXT
+	.globl	_Xrendezvous
+_Xrendezvous:
+	PUSH_FRAME
+	movl	$KDSEL, %eax
+	movl	%ax, %ds		/* use KERNEL data segment */
+	movl	%ax, %es
+	movl	$KPSEL, %eax
+	movl	%ax, %fs
+
+	call	smp_rendezvous_action
+
+	movl	$0, lapic_eoi		/* End Of Interrupt to APIC */
+	POP_FRAME
+	iret
+	
+	
 	.data
 /*
  * Addresses of interrupt handlers.
diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 9d8be53..e81efef 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.104 1999/06/23 21:47:22 luoqi Exp $
+ *	$Id: mp_machdep.c,v 1.105 1999/06/23 23:02:38 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -62,7 +62,10 @@
 
 #include <machine/smp.h>
 #include <machine/apic.h>
+#include <machine/atomic.h>
+#include <machine/cpufunc.h>
 #include <machine/mpapic.h>
+#include <machine/psl.h>
 #include <machine/segments.h>
 #include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG, TEST_TEST1 */
 #include <machine/tss.h>
@@ -591,6 +594,10 @@ mp_enable(u_int boot_addr)
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 #endif
 	
+	/* install an inter-CPU IPI for all-CPU rendezvous */
+	setidt(XRENDEZVOUS_OFFSET, Xrendezvous,
+	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+
 	/* install an inter-CPU IPI for forcing an additional software trap */
 	setidt(XCPUAST_OFFSET, Xcpuast,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
@@ -1698,6 +1705,9 @@ struct simplelock	com_lock;
 struct simplelock	clock_lock;
 #endif /* USE_CLOCKLOCK */
 
+/* lock around the MP rendezvous */
+static struct simplelock smp_rv_lock;
+
 static void
 init_locks(void)
 {
@@ -1722,6 +1732,7 @@ init_locks(void)
 	s_lock_init((struct simplelock*)&intr_lock);
 	s_lock_init((struct simplelock*)&imen_lock);
 	s_lock_init((struct simplelock*)&cpl_lock);
+	s_lock_init(&smp_rv_lock);
 
 #ifdef USE_COMLOCK
 	s_lock_init((struct simplelock*)&com_lock);
@@ -2604,3 +2615,76 @@ set_lapic_isrloc(int intr, int vector)
 	apic_isrbit_location[intr].bit = (1<<(vector & 31));
 }
 #endif
+
+/*
+ * All-CPU rendezvous.  CPUs are signalled, all execute the setup function 
+ * (if specified), rendezvous, execute the action function (if specified),
+ * rendezvous again, execute the teardown function (if specified), and then
+ * resume.
+ *
+ * Note that the supplied external functions _must_ be reentrant and aware
+ * that they are running in parallel and in an unknown lock context.
+ */
+static void (*smp_rv_setup_func)(void *arg);
+static void (*smp_rv_action_func)(void *arg);
+static void (*smp_rv_teardown_func)(void *arg);
+static void *smp_rv_func_arg;
+static volatile int smp_rv_waiters[2];
+
+void
+smp_rendezvous_action(void)
+{
+	/* setup function */
+	if (smp_rv_setup_func != NULL)
+		smp_rv_setup_func(smp_rv_func_arg);
+	/* spin on entry rendezvous */
+	atomic_add_int(&smp_rv_waiters[0], 1);
+	while (smp_rv_waiters[0] < mp_ncpus)
+		;
+	/* action function */
+	if (smp_rv_action_func != NULL)
+		smp_rv_action_func(smp_rv_func_arg);
+	/* spin on exit rendezvous */
+	atomic_add_int(&smp_rv_waiters[1], 1);
+	while (smp_rv_waiters[1] < mp_ncpus)
+		;
+	/* teardown function */
+	if (smp_rv_teardown_func != NULL)
+		smp_rv_teardown_func(smp_rv_func_arg);
+}
+
+void
+smp_rendezvous(void (* setup_func)(void *), 
+	       void (* action_func)(void *),
+	       void (* teardown_func)(void *),
+	       void *arg)
+{
+	u_int	efl;
+	
+	/* obtain rendezvous lock */
+	s_lock(&smp_rv_lock);		/* XXX sleep here? NOWAIT flag? */
+
+	/* set static function pointers */
+	smp_rv_setup_func = setup_func;
+	smp_rv_action_func = action_func;
+	smp_rv_teardown_func = teardown_func;
+	smp_rv_func_arg = arg;
+	smp_rv_waiters[0] = 0;
+	smp_rv_waiters[1] = 0;
+
+	/* disable interrupts on this CPU, save interrupt status */
+	efl = read_eflags();
+	write_eflags(efl & ~PSL_I);
+
+	/* signal other processors, which will enter the IPI with interrupts off */
+	all_but_self_ipi(XRENDEZVOUS_OFFSET);
+
+	/* call executor function */
+	smp_rendezvous_action();
+
+	/* restore interrupt flag */
+	write_eflags(efl);
+
+	/* release lock */
+	s_unlock(&smp_rv_lock);
+}
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index 9d8be53..e81efef 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.104 1999/06/23 21:47:22 luoqi Exp $
+ *	$Id: mp_machdep.c,v 1.105 1999/06/23 23:02:38 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -62,7 +62,10 @@
 
 #include <machine/smp.h>
 #include <machine/apic.h>
+#include <machine/atomic.h>
+#include <machine/cpufunc.h>
 #include <machine/mpapic.h>
+#include <machine/psl.h>
 #include <machine/segments.h>
 #include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG, TEST_TEST1 */
 #include <machine/tss.h>
@@ -591,6 +594,10 @@ mp_enable(u_int boot_addr)
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 #endif
 	
+	/* install an inter-CPU IPI for all-CPU rendezvous */
+	setidt(XRENDEZVOUS_OFFSET, Xrendezvous,
+	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+
 	/* install an inter-CPU IPI for forcing an additional software trap */
 	setidt(XCPUAST_OFFSET, Xcpuast,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
@@ -1698,6 +1705,9 @@ struct simplelock	com_lock;
 struct simplelock	clock_lock;
 #endif /* USE_CLOCKLOCK */
 
+/* lock around the MP rendezvous */
+static struct simplelock smp_rv_lock;
+
 static void
 init_locks(void)
 {
@@ -1722,6 +1732,7 @@ init_locks(void)
 	s_lock_init((struct simplelock*)&intr_lock);
 	s_lock_init((struct simplelock*)&imen_lock);
 	s_lock_init((struct simplelock*)&cpl_lock);
+	s_lock_init(&smp_rv_lock);
 
 #ifdef USE_COMLOCK
 	s_lock_init((struct simplelock*)&com_lock);
@@ -2604,3 +2615,76 @@ set_lapic_isrloc(int intr, int vector)
 	apic_isrbit_location[intr].bit = (1<<(vector & 31));
 }
 #endif
+
+/*
+ * All-CPU rendezvous.  CPUs are signalled, all execute the setup function 
+ * (if specified), rendezvous, execute the action function (if specified),
+ * rendezvous again, execute the teardown function (if specified), and then
+ * resume.
+ *
+ * Note that the supplied external functions _must_ be reentrant and aware
+ * that they are running in parallel and in an unknown lock context.
+ */
+static void (*smp_rv_setup_func)(void *arg);
+static void (*smp_rv_action_func)(void *arg);
+static void (*smp_rv_teardown_func)(void *arg);
+static void *smp_rv_func_arg;
+static volatile int smp_rv_waiters[2];
+
+void
+smp_rendezvous_action(void)
+{
+	/* setup function */
+	if (smp_rv_setup_func != NULL)
+		smp_rv_setup_func(smp_rv_func_arg);
+	/* spin on entry rendezvous */
+	atomic_add_int(&smp_rv_waiters[0], 1);
+	while (smp_rv_waiters[0] < mp_ncpus)
+		;
+	/* action function */
+	if (smp_rv_action_func != NULL)
+		smp_rv_action_func(smp_rv_func_arg);
+	/* spin on exit rendezvous */
+	atomic_add_int(&smp_rv_waiters[1], 1);
+	while (smp_rv_waiters[1] < mp_ncpus)
+		;
+	/* teardown function */
+	if (smp_rv_teardown_func != NULL)
+		smp_rv_teardown_func(smp_rv_func_arg);
+}
+
+void
+smp_rendezvous(void (* setup_func)(void *), 
+	       void (* action_func)(void *),
+	       void (* teardown_func)(void *),
+	       void *arg)
+{
+	u_int	efl;
+	
+	/* obtain rendezvous lock */
+	s_lock(&smp_rv_lock);		/* XXX sleep here? NOWAIT flag? */
+
+	/* set static function pointers */
+	smp_rv_setup_func = setup_func;
+	smp_rv_action_func = action_func;
+	smp_rv_teardown_func = teardown_func;
+	smp_rv_func_arg = arg;
+	smp_rv_waiters[0] = 0;
+	smp_rv_waiters[1] = 0;
+
+	/* disable interrupts on this CPU, save interrupt status */
+	efl = read_eflags();
+	write_eflags(efl & ~PSL_I);
+
+	/* signal other processors, which will enter the IPI with interrupts off */
+	all_but_self_ipi(XRENDEZVOUS_OFFSET);
+
+	/* call executor function */
+	smp_rendezvous_action();
+
+	/* restore interrupt flag */
+	write_eflags(efl);
+
+	/* release lock */
+	s_unlock(&smp_rv_lock);
+}
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index 9d8be53..e81efef 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.104 1999/06/23 21:47:22 luoqi Exp $
+ *	$Id: mp_machdep.c,v 1.105 1999/06/23 23:02:38 msmith Exp $
  */
 
 #include "opt_smp.h"
@@ -62,7 +62,10 @@
 
 #include <machine/smp.h>
 #include <machine/apic.h>
+#include <machine/atomic.h>
+#include <machine/cpufunc.h>
 #include <machine/mpapic.h>
+#include <machine/psl.h>
 #include <machine/segments.h>
 #include <machine/smptests.h>	/** TEST_DEFAULT_CONFIG, TEST_TEST1 */
 #include <machine/tss.h>
@@ -591,6 +594,10 @@ mp_enable(u_int boot_addr)
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 #endif
 	
+	/* install an inter-CPU IPI for all-CPU rendezvous */
+	setidt(XRENDEZVOUS_OFFSET, Xrendezvous,
+	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+
 	/* install an inter-CPU IPI for forcing an additional software trap */
 	setidt(XCPUAST_OFFSET, Xcpuast,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
@@ -1698,6 +1705,9 @@ struct simplelock	com_lock;
 struct simplelock	clock_lock;
 #endif /* USE_CLOCKLOCK */
 
+/* lock around the MP rendezvous */
+static struct simplelock smp_rv_lock;
+
 static void
 init_locks(void)
 {
@@ -1722,6 +1732,7 @@ init_locks(void)
 	s_lock_init((struct simplelock*)&intr_lock);
 	s_lock_init((struct simplelock*)&imen_lock);
 	s_lock_init((struct simplelock*)&cpl_lock);
+	s_lock_init(&smp_rv_lock);
 
 #ifdef USE_COMLOCK
 	s_lock_init((struct simplelock*)&com_lock);
@@ -2604,3 +2615,76 @@ set_lapic_isrloc(int intr, int vector)
 	apic_isrbit_location[intr].bit = (1<<(vector & 31));
 }
 #endif
+
+/*
+ * All-CPU rendezvous.  CPUs are signalled, all execute the setup function 
+ * (if specified), rendezvous, execute the action function (if specified),
+ * rendezvous again, execute the teardown function (if specified), and then
+ * resume.
+ *
+ * Note that the supplied external functions _must_ be reentrant and aware
+ * that they are running in parallel and in an unknown lock context.
+ */
+static void (*smp_rv_setup_func)(void *arg);
+static void (*smp_rv_action_func)(void *arg);
+static void (*smp_rv_teardown_func)(void *arg);
+static void *smp_rv_func_arg;
+static volatile int smp_rv_waiters[2];
+
+void
+smp_rendezvous_action(void)
+{
+	/* setup function */
+	if (smp_rv_setup_func != NULL)
+		smp_rv_setup_func(smp_rv_func_arg);
+	/* spin on entry rendezvous */
+	atomic_add_int(&smp_rv_waiters[0], 1);
+	while (smp_rv_waiters[0] < mp_ncpus)
+		;
+	/* action function */
+	if (smp_rv_action_func != NULL)
+		smp_rv_action_func(smp_rv_func_arg);
+	/* spin on exit rendezvous */
+	atomic_add_int(&smp_rv_waiters[1], 1);
+	while (smp_rv_waiters[1] < mp_ncpus)
+		;
+	/* teardown function */
+	if (smp_rv_teardown_func != NULL)
+		smp_rv_teardown_func(smp_rv_func_arg);
+}
+
+void
+smp_rendezvous(void (* setup_func)(void *), 
+	       void (* action_func)(void *),
+	       void (* teardown_func)(void *),
+	       void *arg)
+{
+	u_int	efl;
+	
+	/* obtain rendezvous lock */
+	s_lock(&smp_rv_lock);		/* XXX sleep here? NOWAIT flag? */
+
+	/* set static function pointers */
+	smp_rv_setup_func = setup_func;
+	smp_rv_action_func = action_func;
+	smp_rv_teardown_func = teardown_func;
+	smp_rv_func_arg = arg;
+	smp_rv_waiters[0] = 0;
+	smp_rv_waiters[1] = 0;
+
+	/* disable interrupts on this CPU, save interrupt status */
+	efl = read_eflags();
+	write_eflags(efl & ~PSL_I);
+
+	/* signal other processors, which will enter the IPI with interrupts off */
+	all_but_self_ipi(XRENDEZVOUS_OFFSET);
+
+	/* call executor function */
+	smp_rendezvous_action();
+
+	/* restore interrupt flag */
+	write_eflags(efl);
+
+	/* release lock */
+	s_unlock(&smp_rv_lock);
+}
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index 3750c72..c425db6 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -6,7 +6,7 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
- * $Id: smp.h,v 1.44 1998/09/06 22:41:40 tegge Exp $
+ * $Id: smp.h,v 1.45 1999/04/28 01:04:07 luoqi Exp $
  *
  */
 
@@ -146,6 +146,11 @@ void	forward_roundrobin	__P((void));
 #ifdef	APIC_INTR_REORDER
 void	set_lapic_isrloc	__P((int, int));
 #endif /* APIC_INTR_REORDER */
+void	smp_rendezvous_action	__P((void));
+void	smp_rendezvous		__P((void (*)(void *), 
+				     void (*)(void *),
+				     void (*)(void *),
+				     void *arg));
 
 /* global data in mpapic.c */
 extern volatile lapic_t		lapic;
diff --git a/sys/amd64/isa/intr_machdep.h b/sys/amd64/isa/intr_machdep.h
index 675714d..06593cf 100644
--- a/sys/amd64/isa/intr_machdep.h
+++ b/sys/amd64/isa/intr_machdep.h
@@ -30,7 +30,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: intr_machdep.h,v 1.14 1999/04/21 07:26:27 peter Exp $
+ *	$Id: intr_machdep.h,v 1.15 1999/06/03 20:41:00 peter Exp $
  */
 
 #ifndef _I386_ISA_INTR_MACHDEP_H_
@@ -112,6 +112,9 @@
 #define XCPUCHECKSTATE_OFFSET	(ICU_OFFSET + 113)
 #endif
 
+/* inter-CPU rendezvous */
+#define XRENDEZVOUS_OFFSET	(ICU_OFFSET + 114)
+
 /* IPI to generate an additional software trap at the target CPU */
 #define XCPUAST_OFFSET		(ICU_OFFSET +  48)
 
@@ -178,7 +181,8 @@ inthand_t
 	Xcpuast,	/* Additional software trap on other cpu */ 
 	Xforward_irq,	/* Forward irq to cpu holding ISR lock */
 	Xcpustop,	/* CPU stops & waits for another CPU to restart it */
-	Xspuriousint;	/* handle APIC "spurious INTs" */
+	Xspuriousint,	/* handle APIC "spurious INTs" */
+	Xrendezvous;	/* handle CPU rendezvous */
 
 #ifdef TEST_TEST1
 inthand_t