Add forwarding of roundrobin to other cpus. This gives a more regular

update of cpu usage as shown by top when one process is cpu bound
(no system calls) while the system is otherwise idle (except for top).

Don't attempt to switch to the BSP in boot().  If the system was idle when
an interrupt caused a panic, this won't work.  Instead, switch to the BSP
in cpu_reset.

Remove some spurious forward_statclock/forward_hardclock warnings.

3 files changed, 59 insertions, 27 deletions

diff --git a/sys/kern/kern_shutdown.c b/sys/kern/kern_shutdown.c
index 448bcc0..e48c1ad 100644
--- a/sys/kern/kern_shutdown.c
+++ b/sys/kern/kern_shutdown.c
@@ -36,7 +36,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)kern_shutdown.c	8.3 (Berkeley) 1/21/94
- * $Id: kern_shutdown.c,v 1.30 1998/05/06 22:14:48 julian Exp $
+ * $Id: kern_shutdown.c,v 1.31 1998/05/12 17:34:02 bde Exp $
  */
 
 #include "opt_ddb.h"
@@ -176,23 +176,8 @@ boot(howto)
 	sle_p ep;
 
 #ifdef SMP
-	int c, spins;
-
-	/* The MPSPEC says that the BSP must do the shutdown */
 	if (smp_active) {
-		smp_active = 0;
-
-		spins = 100;
-
 		printf("boot() called on cpu#%d\n", cpuid);
-		while ((c = cpuid) != 0) {
-			if (spins-- < 1) {
-				printf("timeout waiting for cpu #0!\n");
-				break;
-			}
-			printf("I'm on cpu#%d, I need to be on cpu#0, sleeping..\n", c);
-			tsleep((caddr_t)&smp_active, PZERO, "cpu0wt", 10);
-		}
 	}
 #endif
 	/*
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index cd3ef44..342cb3a 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -36,7 +36,7 @@
  * SUCH DAMAGE.
  *
  *	@(#)kern_synch.c	8.9 (Berkeley) 5/19/95
- * $Id: kern_synch.c,v 1.54 1998/04/04 13:25:20 phk Exp $
+ * $Id: kern_synch.c,v 1.55 1998/05/17 11:52:45 phk Exp $
  */
 
 #include "opt_ktrace.h"
@@ -57,6 +57,10 @@
 #endif
 
 #include <machine/cpu.h>
+#ifdef SMP
+#include <machine/smp.h>
+#endif
+#include <machine/ipl.h>
 #include <machine/limits.h>	/* for UCHAR_MAX = typeof(p_priority)_MAX */
 
 static void rqinit __P((void *));
@@ -130,8 +134,13 @@ roundrobin(arg)
 {
  	struct proc *p = curproc; /* XXX */
  
+#ifdef SMP
+	need_resched();
+	forward_roundrobin();
+#else 
  	if (p == 0 || RTP_PRIO_NEED_RR(p->p_rtprio.type))
  		need_resched();
+#endif
 
  	timeout(roundrobin, NULL, ROUNDROBIN_INTERVAL);
 }
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 7b58b87..c5e392a 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.74 1998/05/11 01:06:06 dyson Exp $
+ *	$Id: mp_machdep.c,v 1.75 1998/05/17 18:53:17 tegge Exp $
  */
 
 #include "opt_smp.h"
@@ -312,6 +312,8 @@ extern pt_entry_t SMP_prvpt[];
 /* Private page pointer to curcpu's PTD, used during BSP init */
 extern pd_entry_t *my_idlePTD;
 
+struct pcb stoppcbs[NCPU];
+
 static int smp_started;		/* has the system started? */
 
 /*
@@ -2009,13 +2011,10 @@ stop_cpus(u_int map)
 	if (!smp_started)
 		return 0;
 
-	/* send IPI to all CPUs in map */
-	stopped_cpus = 0;
-
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
-
-	while (stopped_cpus != map)
+	
+	while ((stopped_cpus & map) != map)
 		/* spin */ ;
 
 	return 1;
@@ -2043,9 +2042,8 @@ restart_cpus(u_int map)
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while (started_cpus)		/* wait for each to clear its bit */
+	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
 		/* spin */ ;
-	stopped_cpus = 0;
 
 	return 1;
 }
@@ -2075,6 +2073,11 @@ int forward_signal_enabled = 1;
 SYSCTL_INT(_machdep, OID_AUTO, forward_signal_enabled, CTLFLAG_RW,
 	   &forward_signal_enabled, 0, "");
 
+/* Enable forwarding of roundrobin to all other cpus */
+int forward_roundrobin_enabled = 1;
+SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW,
+	   &forward_roundrobin_enabled, 0, "");
+
 /*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.
@@ -2296,9 +2299,11 @@ forward_statclock(int pscnt)
 	while (checkstate_probed_cpus != map) {
 		/* spin */
 		i++;
-		if (i == 1000000) {
+		if (i == 100000) {
+#ifdef BETTER_CLOCK_DIAGNOSTIC
 			printf("forward_statclock: checkstate %x\n",
 			       checkstate_probed_cpus);
+#endif
 			break;
 		}
 	}
@@ -2369,9 +2374,11 @@ forward_hardclock(int pscnt)
 	while (checkstate_probed_cpus != map) {
 		/* spin */
 		i++;
-		if (i == 1000000) {
+		if (i == 100000) {
+#ifdef BETTER_CLOCK_DIAGNOSTIC
 			printf("forward_hardclock: checkstate %x\n",
 			       checkstate_probed_cpus);
+#endif
 			break;
 		}
 	}
@@ -2475,6 +2482,37 @@ forward_signal(struct proc *p)
 	}
 }
 
+void
+forward_roundrobin(void)
+{
+	u_int map;
+	int i;
+
+	if (!smp_started || !invltlb_ok || cold || panicstr)
+		return;
+	if (!forward_roundrobin_enabled)
+		return;
+	resched_cpus |= other_cpus;
+	map = other_cpus & ~stopped_cpus ;
+#if 1
+	selected_apic_ipi(map, XCPUAST_OFFSET, APIC_DELMODE_FIXED);
+#else
+	(void) all_but_self_ipi(XCPUAST_OFFSET);
+#endif
+	i = 0;
+	while ((checkstate_need_ast & map) != 0) {
+		/* spin */
+		i++;
+		if (i > 100000) {
+#if 0
+			printf("forward_roundrobin: dropped ast 0x%x\n",
+			       checkstate_need_ast & map);
+#endif
+			break;
+		}
+	}
+}
+
 
 #ifdef APIC_INTR_REORDER
 /*