Revert the part of r239864 which removed obtaining the SMP mutex around

reading registers from other CPUs. As it turns out, the hardware doesn't
really like concurrent IPI'ing causing adverse effects. Also the thought
deadlock when using this spin lock here and the targeted CPU(s) are also
holding or in case of nested locks can't actually happen. This is due to
the fact that on sparc64, spinlock_enter() only raises the PIL but doesn't
disable interrupts completely. Thus direct cross calls as used for the
register reading (and all other MD IPI needs) still will be executed by
the targeted CPU(s) in that case.

MFC after:	3 days

2 files changed, 4 insertions, 20 deletions

diff --git a/sys/sparc64/include/smp.h b/sys/sparc64/include/smp.h
index db17309..2e1110b 100644
--- a/sys/sparc64/include/smp.h
+++ b/sys/sparc64/include/smp.h
@@ -42,6 +42,7 @@
 #include <sys/cpuset.h>
 #include <sys/proc.h>
 #include <sys/sched.h>
+#include <sys/smp.h>
 
 #include <machine/intr_machdep.h>
 #include <machine/pcb.h>
@@ -202,6 +203,7 @@ ipi_rd(u_int cpu, void *func, u_long *val)
 		return (NULL);
 	sched_pin();
 	ira = &ipi_rd_args;
+	mtx_lock_spin(&smp_ipi_mtx);
 	CPU_SETOF(cpu, &ira->ira_mask);
 	ira->ira_val = val;
 	cpu_ipi_single(cpu, 0, (u_long)func, (u_long)ira);
@@ -298,18 +300,6 @@ ipi_wait(void *cookie)
 	}
 }
 
-static __inline void
-ipi_wait_unlocked(void *cookie)
-{
-	volatile cpuset_t *mask;
-
-	if ((mask = cookie) != NULL) {
-		while (!CPU_EMPTY(mask))
-			;
-		sched_unpin();
-	}
-}
-
 #endif /* _MACHINE_PMAP_H_ && _SYS_MUTEX_H_ */
 
 #endif /* !LOCORE */
@@ -368,12 +358,6 @@ ipi_wait(void *cookie __unused)
 }
 
 static __inline void
-ipi_wait_unlocked(void *cookie __unused)
-{
-
-}
-
-static __inline void
 tl_ipi_cheetah_dcache_page_inval(void)
 {
 
diff --git a/sys/sparc64/sparc64/tick.c b/sys/sparc64/sparc64/tick.c
index 69062ec..cfd5776 100644
--- a/sys/sparc64/sparc64/tick.c
+++ b/sys/sparc64/sparc64/tick.c
@@ -334,7 +334,7 @@ stick_get_timecount_mp(struct timecounter *tc)
 	if (curcpu == 0)
 		stick = rdstick();
 	else
-		ipi_wait_unlocked(ipi_rd(0, tl_ipi_stick_rd, &stick));
+		ipi_wait(ipi_rd(0, tl_ipi_stick_rd, &stick));
 	sched_unpin();
 	return (stick);
 }
@@ -348,7 +348,7 @@ tick_get_timecount_mp(struct timecounter *tc)
 	if (curcpu == 0)
 		tick = rd(tick);
 	else
-		ipi_wait_unlocked(ipi_rd(0, tl_ipi_tick_rd, &tick));
+		ipi_wait(ipi_rd(0, tl_ipi_tick_rd, &tick));
 	sched_unpin();
 	return (tick);
 }