amd64/i386: switch IPI handlers to C code.

Move asm IPIs handlers to C code, so both Xen and native IPI handlers
share the same code.

Reviewed by: jhb
Approved by: gibbs
Sponsored by: Citrix Systems R&D

amd64/amd64/apic_vector.S:
i386/i386/apic_vector.s:
 - Remove asm coded IPI handlers and instead call the newly introduced
   C variants.

amd64/amd64/mp_machdep.c:
i386/i386/mp_machdep.c:
 - Add C coded clones to the asm IPI handlers (moved from
   x86/xen/hvm.c).

i386/include/smp.h:
amd64/include/smp.h:
 - Add prototypes for the C IPI handlers.

x86/xen/hvm.c:
 - Move the C IPI handlers to mp_machdep and call those in the Xen IPI
   handlers.

i386/xen/mp_machdep.c:
 - Add dummy IPI handlers to the i386 Xen PV port (this port doesn't
   support SMP).

4 files changed, 121 insertions, 119 deletions

diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s
index adedbc4..3d1999e 100644
--- a/sys/i386/i386/apic_vector.s
+++ b/sys/i386/i386/apic_vector.s
@@ -163,39 +163,21 @@ IDTVEC(xen_intr_upcall)
  */
 	.text
 	SUPERALIGN_TEXT
-IDTVEC(invltlb)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	movl	%eax, %ds
-
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	movl	%eax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_gbl(,%eax,4)
-#endif
-#ifdef COUNT_IPIS
-	movl	ipi_invltlb_counts(,%eax,4),%eax
-	incl	(%eax)
-#endif
-#endif
-
-	movl	%cr3, %eax		/* invalidate the TLB */
-	movl	%eax, %cr3
-
+invltlb_ret:
 	movl	lapic, %eax
 	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	POP_FRAME
+	iret
+
+	SUPERALIGN_TEXT
+IDTVEC(invltlb)
+	PUSH_FRAME
+	SET_KERNEL_SREGS
+	cld
 
-	lock
-	incl	smp_tlb_wait
+	call	invltlb_handler
 
-	popl	%ds
-	popl	%eax
-	iret
+	jmp	invltlb_ret
 
 /*
  * Single page TLB shootdown
@@ -203,38 +185,13 @@ IDTVEC(invltlb)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlpg)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	movl	%eax, %ds
-
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	movl	%eax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_pg(,%eax,4)
-#endif
-#ifdef COUNT_IPIS
-	movl	ipi_invlpg_counts(,%eax,4),%eax
-	incl	(%eax)
-#endif
-#endif
-
-	movl	smp_tlb_addr1, %eax
-	invlpg	(%eax)			/* invalidate single page */
-
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	PUSH_FRAME
+	SET_KERNEL_SREGS
+	cld
 
-	lock
-	incl	smp_tlb_wait
+	call	invlpg_handler
 
-	popl	%ds
-	popl	%eax
-	iret
+	jmp	invltlb_ret
 
 /*
  * Page range TLB shootdown.
@@ -242,44 +199,13 @@ IDTVEC(invlpg)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlrng)
-	pushl	%eax
-	pushl	%edx
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	movl	%eax, %ds
-
-#if defined(COUNT_XINVLTLB_HITS) || defined(COUNT_IPIS)
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	movl	%eax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-#ifdef COUNT_XINVLTLB_HITS
-	incl	xhits_rng(,%eax,4)
-#endif
-#ifdef COUNT_IPIS
-	movl	ipi_invlrng_counts(,%eax,4),%eax
-	incl	(%eax)
-#endif
-#endif
+	PUSH_FRAME
+	SET_KERNEL_SREGS
+	cld
 
-	movl	smp_tlb_addr1, %edx
-	movl	smp_tlb_addr2, %eax
-1:	invlpg	(%edx)			/* invalidate single page */
-	addl	$PAGE_SIZE, %edx
-	cmpl	%eax, %edx
-	jb	1b
+	call	invlrng_handler
 
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
-
-	lock
-	incl	smp_tlb_wait
-
-	popl	%ds
-	popl	%edx
-	popl	%eax
-	iret
+	jmp	invltlb_ret
 
 /*
  * Invalidate cache.
@@ -287,32 +213,13 @@ IDTVEC(invlrng)
 	.text
 	SUPERALIGN_TEXT
 IDTVEC(invlcache)
-	pushl	%eax
-	pushl	%ds
-	movl	$KDSEL, %eax		/* Kernel data selector */
-	movl	%eax, %ds
-
-#ifdef COUNT_IPIS
-	pushl	%fs
-	movl	$KPSEL, %eax		/* Private space selector */
-	movl	%eax, %fs
-	movl	PCPU(CPUID), %eax
-	popl	%fs
-	movl	ipi_invlcache_counts(,%eax,4),%eax
-	incl	(%eax)
-#endif
-
-	wbinvd
-
-	movl	lapic, %eax
-	movl	$0, LA_EOI(%eax)	/* End Of Interrupt to APIC */
+	PUSH_FRAME
+	SET_KERNEL_SREGS
+	cld
 
-	lock
-	incl	smp_tlb_wait
+	call	invlcache_handler
 
-	popl	%ds
-	popl	%eax
-	iret
+	jmp	invltlb_ret
 
 /*
  * Handler for IPIs sent via the per-cpu IPI bitmap.
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 8ee8653..53ac6df 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -1551,6 +1551,72 @@ cpususpend_handler(void)
 	CPU_CLR_ATOMIC(cpu, &suspended_cpus);
 	CPU_CLR_ATOMIC(cpu, &started_cpus);
 }
+
+/*
+ * Handlers for TLB related IPIs
+ */
+void
+invltlb_handler(void)
+{
+	uint64_t cr3;
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_gbl[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	cr3 = rcr3();
+	load_cr3(cr3);
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlpg_handler(void)
+{
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_pg[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	invlpg(smp_tlb_addr1);
+
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlrng_handler(void)
+{
+	vm_offset_t addr;
+#ifdef COUNT_XINVLTLB_HITS
+	xhits_rng[PCPU_GET(cpuid)]++;
+#endif /* COUNT_XINVLTLB_HITS */
+#ifdef COUNT_IPIS
+	(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	addr = smp_tlb_addr1;
+	do {
+		invlpg(addr);
+		addr += PAGE_SIZE;
+	} while (addr < smp_tlb_addr2);
+
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
+void
+invlcache_handler(void)
+{
+#ifdef COUNT_IPIS
+	(*ipi_invlcache_counts[PCPU_GET(cpuid)])++;
+#endif /* COUNT_IPIS */
+
+	wbinvd();
+	atomic_add_int(&smp_tlb_wait, 1);
+}
+
 /*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.
diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h
index 064c480..cc9a8ea 100644
--- a/sys/i386/include/smp.h
+++ b/sys/i386/include/smp.h
@@ -63,6 +63,10 @@ void	cpustop_handler(void);
 #ifndef XEN
 void	cpususpend_handler(void);
 #endif
+void	invltlb_handler(void);
+void	invlpg_handler(void);
+void	invlrng_handler(void);
+void	invlcache_handler(void);
 void	init_secondary(void);
 void	ipi_startup(int apic_id, int vector);
 void	ipi_all_but_self(u_int ipi);
diff --git a/sys/i386/xen/mp_machdep.c b/sys/i386/xen/mp_machdep.c
index c48fcb2..1bc6572 100644
--- a/sys/i386/xen/mp_machdep.c
+++ b/sys/i386/xen/mp_machdep.c
@@ -1276,6 +1276,31 @@ cpustop_handler(void)
 }
 
 /*
+ * Handlers for TLB related IPIs
+ *
+ * On i386 Xen PV this are no-ops since this port doesn't support SMP.
+ */
+void
+invltlb_handler(void)
+{
+}
+
+void
+invlpg_handler(void)
+{
+}
+
+void
+invlrng_handler(void)
+{
+}
+
+void
+invlcache_handler(void)
+{
+}
+
+/*
  * This is called once the rest of the system is up and running and we're
  * ready to let the AP's out of the pen.
  */