- According to comments in OpenBSD, E{2,4}50 tend to have fragile

  firmware versions which wedge when using the OFW test service,
  so given that we don't really depend on SUNW,stop-self just nuke
  it altogether instead of risking problems.
- At least Fire V880 have a small hardware glitch which causes the
  reception of IDR_NACKs for CPUs we actually haven't tried to send
  an IPI to, even not as part of the initial try. According to tests
  this apparently can be safely ignored though, so just return if
  checking for the individual IDR_NACKs indicates no outstanding
  dispatch. Serializing the sending of IPIs between MD and MI code
  by the combined usage of smp_ipi_mtx makes no difference to this
  phenomenon. [1]
- Provide relevant debugging bits already with the initial panic
  in case of problems with the IPI dispatch, which would have
  allowed to diagnose the above problem without a specially built
  kernel.
- In case of cheetah_ipi_selected() base the delay we wait for
  other CPUs which also might want to dispatch IPIs on the total
  amount of CPUs instead of just the number of CPUs we let this
  CPU send IPIs to because in the worst case all CPUs also want
  to IPI us at the same time.

Reported and access for extensive tests provided by:	Beat Gaetzi [1]

1 files changed, 12 insertions, 28 deletions

diff --git a/sys/sparc64/sparc64/mp_machdep.c b/sys/sparc64/sparc64/mp_machdep.c
index bf79adf..5470c11 100644
--- a/sys/sparc64/sparc64/mp_machdep.c
+++ b/sys/sparc64/sparc64/mp_machdep.c
@@ -116,14 +116,12 @@ cpu_ipi_selected_t *cpu_ipi_selected;
 
 static vm_offset_t mp_tramp;
 static u_int cpuid_to_mid[MAXCPU];
-static int has_stopself;
 static int isjbus;
 static volatile u_int shutdown_cpus;
 
 static void cpu_mp_unleash(void *v);
 static void spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2);
 static void sun4u_startcpu(phandle_t cpu, void *func, u_long arg);
-static void sun4u_stopself(void);
 
 static cpu_ipi_selected_t cheetah_ipi_selected;
 static cpu_ipi_selected_t spitfire_ipi_selected;
@@ -225,24 +223,6 @@ sun4u_startcpu(phandle_t cpu, void *func, u_long arg)
 }
 
 /*
- * Stop the calling CPU.
- */
-static void
-sun4u_stopself(void)
-{
-	static struct {
-		cell_t	name;
-		cell_t	nargs;
-		cell_t	nreturns;
-	} args = {
-		(cell_t)SUNW_STOPSELF,
-	};
-
-	ofw_exit(&args);
-	panic("%s: failed.", __func__);
-}
-
-/*
  * Fire up any non-boot processors.
  */
 void
@@ -260,9 +240,6 @@ cpu_mp_start(void)
 
 	mtx_init(&ipi_mtx, "ipi", NULL, MTX_SPIN);
 
-	if (OF_test(SUNW_STOPSELF) == 0)
-		has_stopself = 1;
-
 	intr_setup(PIL_AST, cpu_ipi_ast, -1, NULL, NULL);
 	intr_setup(PIL_RENDEZVOUS, (ih_func_t *)smp_rendezvous_action,
 	    -1, NULL, NULL);
@@ -457,8 +434,6 @@ cpu_ipi_stop(struct trapframe *tf)
 	while ((started_cpus & PCPU_GET(cpumask)) == 0) {
 		if ((shutdown_cpus & PCPU_GET(cpumask)) != 0) {
 			atomic_clear_int(&shutdown_cpus, PCPU_GET(cpumask));
-			if (has_stopself != 0)
-				sun4u_stopself();
 			(void)intr_disable();
 			for (;;)
 				;
@@ -538,7 +513,8 @@ spitfire_ipi_send(u_int mid, u_long d0, u_long d1, u_long d2)
 		printf("%s: couldn't send IPI to module 0x%u\n",
 		    __func__, mid);
 	else
-		panic("%s: couldn't send IPI", __func__);
+		panic("%s: couldn't send IPI to module 0x%u",
+		    __func__, mid);
 }
 
 static void
@@ -592,11 +568,18 @@ cheetah_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2)
 			}
 		}
 		/*
+		 * On at least Fire V880 we may receive IDR_NACKs for
+		 * CPUs we actually haven't tried to send an IPI to,
+		 * but which apparently can be safely ignored.
+		 */
+		if (cpus == 0)
+			return;
+		/*
 		 * Leave interrupts enabled for a bit before retrying
 		 * in order to avoid deadlocks if the other CPUs are
 		 * also trying to send IPIs.
 		 */
-		DELAY(2 * bnp);
+		DELAY(2 * mp_ncpus);
 	}
 	if (
 #ifdef KDB
@@ -606,5 +589,6 @@ cheetah_ipi_selected(u_int cpus, u_long d0, u_long d1, u_long d2)
 		printf("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)\n",
 		    __func__, cpus, ids);
 	else
-		panic("%s: couldn't send IPI", __func__);
+		panic("%s: couldn't send IPI (cpus=0x%u ids=0x%lu)",
+		    __func__, cpus, ids);
 }