MFC r265900:

Repair some races in IPI handling:
1. Make sure IPI mask is set before sending the IPI
2. Operate atomically on PS3 PIC outstanding interrupt list
3. Make sure IPIs are EOI'ed before, not after, processing. Without this,
   a second IPI could be sent partway through processing the first one,
   get erroneously acknowledge by the EOI to the first, and be lost. In
   particular in the case of smp_rendezvous(), this can be fatal.

In combination, this makes the PS3 boot SMP again. It probably also fixes
some latent bugs elsewhere.

3 files changed, 23 insertions, 2 deletions

diff --git a/sys/powerpc/powerpc/intr_machdep.c b/sys/powerpc/powerpc/intr_machdep.c
index 0b13191..af3fec8 100644
--- a/sys/powerpc/powerpc/intr_machdep.c
+++ b/sys/powerpc/powerpc/intr_machdep.c
@@ -103,6 +103,7 @@ struct powerpc_intr {
 	enum intr_trigger trig;
 	enum intr_polarity pol;
 	int	fwcode;
+	int	ipi;
 };
 
 struct pic {
@@ -203,6 +204,8 @@ intr_lookup(u_int irq)
 	i->irq = irq;
 	i->pic = NULL;
 	i->vector = -1;
+	i->fwcode = 0;
+	i->ipi = 0;
 
 #ifdef SMP
 	i->cpu = all_cpus;
@@ -415,6 +418,15 @@ powerpc_enable_intr(void)
 				printf("unable to setup IPI handler\n");
 				return (error);
 			}
+
+			/*
+			 * Some subterfuge: disable late EOI and mark this
+			 * as an IPI to the dispatch layer.
+			 */
+			i = intr_lookup(MAP_IRQ(piclist[n].node,
+			    piclist[n].irqs));
+			i->event->ie_post_filter = NULL;
+			i->ipi = 1;
 		}
 	}
 #endif
@@ -568,6 +580,13 @@ powerpc_dispatch_intr(u_int vector, struct trapframe *tf)
 	ie = i->event;
 	KASSERT(ie != NULL, ("%s: interrupt without an event", __func__));
 
+	/*
+	 * IPIs are magical and need to be EOI'ed before filtering.
+	 * This prevents races in IPI handling.
+	 */
+	if (i->ipi)
+		PIC_EOI(i->pic, i->intline);
+
 	if (intr_event_handle(ie, tf) != 0) {
 		goto stray;
 	}
diff --git a/sys/powerpc/powerpc/mp_machdep.c b/sys/powerpc/powerpc/mp_machdep.c
index 555daf1..9d26e95 100644
--- a/sys/powerpc/powerpc/mp_machdep.c
+++ b/sys/powerpc/powerpc/mp_machdep.c
@@ -336,6 +336,7 @@ ipi_send(struct pcpu *pc, int ipi)
 	    pc, pc->pc_cpuid, ipi);
 
 	atomic_set_32(&pc->pc_ipimask, (1 << ipi));
+	powerpc_sync();
 	PIC_IPI(root_pic, pc->pc_cpuid);
 
 	CTR1(KTR_SMP, "%s: sent", __func__);
diff --git a/sys/powerpc/ps3/ps3pic.c b/sys/powerpc/ps3/ps3pic.c
index 9b40f6c..374d96e 100644
--- a/sys/powerpc/ps3/ps3pic.c
+++ b/sys/powerpc/ps3/ps3pic.c
@@ -166,12 +166,13 @@ ps3pic_dispatch(device_t dev, struct trapframe *tf)
 	sc = device_get_softc(dev);
 
 	if (PCPU_GET(cpuid) == 0) {
-		bitmap = sc->bitmap_thread0[0];
+		bitmap = atomic_readandclear_64(&sc->bitmap_thread0[0]);
 		mask = sc->mask_thread0[0];
 	} else {
-		bitmap = sc->bitmap_thread1[0];
+		bitmap = atomic_readandclear_64(&sc->bitmap_thread1[0]);
 		mask = sc->mask_thread1[0];
 	}
+	powerpc_sync();
 
 	while ((irq = ffsl(bitmap & mask) - 1) != -1) {
 		bitmap &= ~(1UL << irq);