Fix the MP IPI code to differentiate between bitmapped IPIs and function IPIs.

This attempts to fix the IPI handling code to correctly differentiate
between bitmapped IPIs and function IPIs. The Xen IPIs were on low numbers
which clashed with the bitmapped IPIs.

This commit bumps those IPI numbers up to 240 and above (just like in the i386
code) and fiddles with the ipi_vectors[] logic to call the correct function.

This still isn't "right". Specifically, the IPI code may work fine for TLB
shootdown events but the rendezvous/lazypmap IPIs are thrown by calling ipi_*()
routines which don't set the call_func stuff (function id, addr1, addr2) that
the TLB shootdown events are. So the Xen SMP support is still broken.

PR:		135069

2 files changed, 14 insertions, 19 deletions

diff --git a/sys/i386/include/apicvar.h b/sys/i386/include/apicvar.h
index 056010a..9f419e2 100644
--- a/sys/i386/include/apicvar.h
+++ b/sys/i386/include/apicvar.h
@@ -114,14 +114,14 @@
 
 #define	APIC_IPI_INTS	(APIC_LOCAL_INTS + 2)
 #ifdef XEN
-#define	IPI_RENDEZVOUS		(2)	/* Inter-CPU rendezvous. */
-#define	IPI_INVLTLB		(3)	/* TLB Shootdown IPIs */
-#define	IPI_INVLPG		(4)
-#define	IPI_INVLRNG		(5)
-#define	IPI_INVLCACHE		(6)
-#define	IPI_LAZYPMAP		(7)	/* Lazy pmap release. */
+#define	IPI_RENDEZVOUS		(APIC_IPI_INTS)	/* Inter-CPU rendezvous. */
+#define	IPI_INVLTLB		(APIC_IPI_INTS + 1)	/* TLB Shootdown IPIs */
+#define	IPI_INVLPG		(APIC_IPI_INTS + 2)
+#define	IPI_INVLRNG		(APIC_IPI_INTS + 3)
+#define	IPI_INVLCACHE		(APIC_IPI_INTS + 4)
+#define	IPI_LAZYPMAP		(APIC_IPI_INTS + 5)	/* Lazy pmap release. */
 /* Vector to handle bitmap based IPIs */
-#define	IPI_BITMAP_VECTOR	(8)
+#define	IPI_BITMAP_VECTOR	(APIC_IPI_INTS + 6)
 
 #else
 #define	IPI_RENDEZVOUS	(APIC_IPI_INTS)		/* Inter-CPU rendezvous. */
diff --git a/sys/i386/xen/mp_machdep.c b/sys/i386/xen/mp_machdep.c
index d389189..f16d25f 100644
--- a/sys/i386/xen/mp_machdep.c
+++ b/sys/i386/xen/mp_machdep.c
@@ -350,17 +350,11 @@ iv_lazypmap(uintptr_t a, uintptr_t b)
 	atomic_add_int(&smp_tlb_wait, 1);
 }
 
-
-static void
-iv_noop(uintptr_t a, uintptr_t b)
-{
-	atomic_add_int(&smp_tlb_wait, 1);
-}
-
-static call_data_func_t *ipi_vectors[IPI_BITMAP_VECTOR] = 
+/*
+ * These start from "IPI offset" APIC_IPI_INTS
+ */
+static call_data_func_t *ipi_vectors[6] = 
 {
-  iv_noop,
-  iv_noop,
   iv_rendezvous,
   iv_invltlb,
   iv_invlpg,
@@ -419,10 +413,11 @@ smp_call_function_interrupt(void *unused)
 	atomic_t *started = &call_data->started;
 	atomic_t *finished = &call_data->finished;
 
-	if (call_data->func_id > IPI_BITMAP_VECTOR)
+	/* We only handle function IPIs, not bitmap IPIs */
+	if (call_data->func_id < APIC_IPI_INTS || call_data->func_id > IPI_BITMAP_VECTOR)
 		panic("invalid function id %u", call_data->func_id);
 	
-	func = ipi_vectors[call_data->func_id];
+	func = ipi_vectors[call_data->func_id - APIC_IPI_INTS];
 	/*
 	 * Notify initiating CPU that I've grabbed the data and am
 	 * about to execute the function