Work-in-progress commit syncing up pmap cleanups that I have been working

on for a while:
- fine grained TLB shootdown for SMP on i386
- ranged TLB shootdowns.. eg: specify a range of pages to shoot down with
  a single IPI, since the IPI is very expensive.  Adjust some callers
  that used to trigger this inside tight loops to do a ranged shootdown
  at the end instead.
- PG_G support for SMP on i386 (options ENABLE_PG_G)
- defer PG_G activation till after we decide what we are going to do with
  PSE and the 4MB pages at the start of the kernel.  This should solve
  some rumored strangeness about stale PG_G entries getting stuck
  underneath the 4MB pages.
- add some instrumentation for the fine TLB shootdown
- convert some asm instruction wrappers from functions to inlines.  gcc
  seems to do a fair bit better with this.
- [temporarily!] pessimize the tlb shootdown IPI handlers.  I will fix
  this again shortly.

This has been working fairly well for me for a while, but I have tweaked
it again prior to commit since my last major testing round.  The only
outstanding problem that I know of is PG_G related, which is why there
is an option for it (not on by default for SMP).  I have seen a world
speedups by a few percent (as much as 4 or 5% in one case) but I have
*not* accurately measured this - I am a bit sceptical of these numbers.

1 files changed, 83 insertions, 11 deletions

diff --git a/sys/i386/isa/apic_vector.s b/sys/i386/isa/apic_vector.s
index 95c9133..e3a37e1 100644
--- a/sys/i386/isa/apic_vector.s
+++ b/sys/i386/isa/apic_vector.s
@@ -181,30 +181,108 @@ Xspuriousint:
 	iret
 
 /*
- * Handle TLB shootdowns.
+ * Global address space TLB shootdown.
  */
 	.text
 	SUPERALIGN_TEXT
 	.globl	Xinvltlb
 Xinvltlb:
 	pushl	%eax
+	pushl	%ds
+	movl	$KDSEL, %eax		/* Kernel data selector */
+	mov	%ax, %ds
 
 #ifdef COUNT_XINVLTLB_HITS
 	pushl	%fs
-	movl	$KPSEL, %eax
+	movl	$KPSEL, %eax		/* Private space selector */
 	mov	%ax, %fs
 	movl	PCPU(CPUID), %eax
 	popl	%fs
-	ss
-	incl	_xhits(,%eax,4)
+	incl	xhits_gbl(,%eax,4)
 #endif /* COUNT_XINVLTLB_HITS */
 
 	movl	%cr3, %eax		/* invalidate the TLB */
 	movl	%eax, %cr3
 
-	ss				/* stack segment, avoid %ds load */
 	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
 
+	lock
+	incl	smp_tlb_wait
+
+	popl	%ds
+	popl	%eax
+	iret
+
+/*
+ * Single page TLB shootdown
+ */
+	.text
+	SUPERALIGN_TEXT
+	.globl	Xinvlpg
+Xinvlpg:
+	pushl	%eax
+	pushl	%ds
+	movl	$KDSEL, %eax		/* Kernel data selector */
+	mov	%ax, %ds
+
+#ifdef COUNT_XINVLTLB_HITS
+	pushl	%fs
+	movl	$KPSEL, %eax		/* Private space selector */
+	mov	%ax, %fs
+	movl	PCPU(CPUID), %eax
+	popl	%fs
+	ss
+	incl	xhits_pg(,%eax,4)
+#endif /* COUNT_XINVLTLB_HITS */
+
+	movl	smp_tlb_addr1, %eax
+	invlpg	(%eax)			/* invalidate single page */
+
+	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+
+	lock
+	incl	smp_tlb_wait
+
+	popl	%ds
+	popl	%eax
+	iret
+
+/*
+ * Page range TLB shootdown.
+ */
+	.text
+	SUPERALIGN_TEXT
+	.globl	Xinvlrng
+Xinvlrng:
+	pushl	%eax
+	pushl	%edx
+	pushl	%ds
+	movl	$KDSEL, %eax		/* Kernel data selector */
+	mov	%ax, %ds
+
+#ifdef COUNT_XINVLTLB_HITS
+	pushl	%fs
+	movl	$KPSEL, %eax		/* Private space selector */
+	mov	%ax, %fs
+	movl	PCPU(CPUID), %eax
+	popl	%fs
+	incl	xhits_rng(,%eax,4)
+#endif /* COUNT_XINVLTLB_HITS */
+
+	movl	smp_tlb_addr1, %edx
+	movl	smp_tlb_addr2, %eax
+1:	invlpg	(%edx)			/* invalidate single page */
+	addl	$PAGE_SIZE, %edx
+	cmpl	%edx, %eax
+	jb	1b
+
+	movl	$0, lapic+LA_EOI	/* End Of Interrupt to APIC */
+
+	lock
+	incl	smp_tlb_wait
+
+	popl	%ds
+	popl	%edx
 	popl	%eax
 	iret
 
@@ -443,12 +521,6 @@ Xrendezvous:
 	
 	.data
 
-#ifdef COUNT_XINVLTLB_HITS
-	.globl	_xhits
-_xhits:
-	.space	(NCPU * 4), 0
-#endif /* COUNT_XINVLTLB_HITS */
-
 	.globl	apic_pin_trigger
 apic_pin_trigger:
 	.long	0