Back out all the pmap related stuff I've touched over the last few days.

There is some unresolved badness that has been eluding me, particularly
affecting uniprocessor kernels.  Turning off PG_G helped (which is a bad
sign) but didn't solve it entirely.  Userland programs still crashed.

5 files changed, 104 insertions, 365 deletions

diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h
index 94d5c3a..969541f 100644
--- a/sys/i386/include/cpufunc.h
+++ b/sys/i386/include/cpufunc.h
@@ -227,6 +227,62 @@ invd(void)
 	__asm __volatile("invd");
 }
 
+#if defined(SMP) && defined(_KERNEL)
+
+/*
+ * When using APIC IPI's, invlpg() is not simply the invlpg instruction
+ * (this is a bug) and the inlining cost is prohibitive since the call
+ * executes into the IPI transmission system.
+ */
+void	invlpg		__P((u_int addr));
+void	invltlb		__P((void));
+
+static __inline void
+cpu_invlpg(void *addr)
+{
+	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
+}
+
+static __inline void
+cpu_invltlb(void)
+{
+	u_int	temp;
+	/*
+	 * This should be implemented as load_cr3(rcr3()) when load_cr3()
+	 * is inlined.
+	 */
+	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
+			 : : "memory");
+#if defined(SWTCH_OPTIM_STATS)
+	++tlb_flush_count;
+#endif
+}
+
+#else /* !(SMP && _KERNEL) */
+
+static __inline void
+invlpg(u_int addr)
+{
+	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
+}
+
+static __inline void
+invltlb(void)
+{
+	u_int	temp;
+	/*
+	 * This should be implemented as load_cr3(rcr3()) when load_cr3()
+	 * is inlined.
+	 */
+	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3" : "=r" (temp)
+			 : : "memory");
+#ifdef SWTCH_OPTIM_STATS
+	++tlb_flush_count;
+#endif
+}
+
+#endif /* SMP && _KERNEL */
+
 static __inline u_short
 inw(u_int port)
 {
@@ -292,6 +348,15 @@ outw(u_int port, u_short data)
 }
 
 static __inline u_int
+rcr2(void)
+{
+	u_int	data;
+
+	__asm __volatile("movl %%cr2,%0" : "=r" (data));
+	return (data);
+}
+
+static __inline u_int
 read_eflags(void)
 {
 	u_int	ef;
@@ -355,162 +420,6 @@ wrmsr(u_int msr, u_int64_t newval)
 	__asm __volatile("wrmsr" : : "A" (newval), "c" (msr));
 }
 
-static __inline void
-load_cr0(u_int data)
-{
-
-	__asm __volatile("movl %0,%%cr0" : : "r" (data));
-}
-
-static __inline u_int
-rcr0(void)
-{
-	u_int	data;
-
-	__asm __volatile("movl %%cr0,%0" : "=r" (data));
-	return (data);
-}
-
-static __inline u_int
-rcr2(void)
-{
-	u_int	data;
-
-	__asm __volatile("movl %%cr2,%0" : "=r" (data));
-	return (data);
-}
-
-static __inline void
-load_cr3(u_int data)
-{
-
-	__asm __volatile("movl %0,%%cr3" : : "r" (data) : "memory");
-#if defined(SWTCH_OPTIM_STATS)
-	++tlb_flush_count;
-#endif
-}
-
-static __inline u_int
-rcr3(void)
-{
-	u_int	data;
-
-	__asm __volatile("movl %%cr3,%0" : "=r" (data));
-	return (data);
-}
-
-static __inline void
-load_cr4(u_int data)
-{
-	__asm __volatile("movl %0,%%cr4" : : "r" (data));
-}
-
-static __inline u_int
-rcr4(void)
-{
-	u_int	data;
-
-	__asm __volatile("movl %%cr4,%0" : "=r" (data));
-	return (data);
-}
-
-/*
- * Global TLB flush (except for thise for pages marked PG_G)
- */
-static __inline void
-cpu_invltlb(void)
-{
-
-	load_cr3(rcr3());
-}
-
-/*
- * TLB flush for an individual page (even if it has PG_G).
- * Only works on 486+ CPUs (i386 does not have PG_G).
- */
-static __inline void
-cpu_invlpg(u_int addr)
-{
-
-#ifndef I386_CPU
-	__asm __volatile("invlpg %0" : : "m" (*(char *)addr) : "memory");
-#else
-	cpu_invltlb();
-#endif
-}
-
-#ifdef PAGE_SIZE	/* Avoid this file depending on sys/param.h */
-/*
- * Same as above but for a range of pages.
- */
-static __inline void
-cpu_invlpg_range(u_int startva, u_int endva)
-{
-#ifndef I386_CPU
-	u_int addr;
-
-	for (addr = startva; addr < endva; addr += PAGE_SIZE)
-		__asm __volatile("invlpg %0" : : "m" (*(char *)addr));
-	__asm __volatile("" : : : "memory");
-#else
-	cpu_invltlb();
-#endif
-}
-#endif
-
-#ifdef SMP
-extern void	smp_invlpg(u_int addr);
-extern void	smp_masked_invlpg(u_int mask, u_int addr);
-#ifdef PAGE_SIZE	/* Avoid this file depending on sys/param.h */
-extern void	smp_invlpg_range(u_int startva, u_int endva);
-extern void	smp_masked_invlpg_range(u_int mask, u_int startva, u_int endva);
-#endif
-extern void	smp_invltlb(void);
-extern void	smp_masked_invltlb(u_int mask);
-#endif
-
-/*
- * Generic page TLB flush.  Takes care of SMP.
- */
-static __inline void
-invlpg(u_int addr)
-{
-
-	cpu_invlpg(addr);
-#ifdef SMP
-	smp_invlpg(addr);
-#endif
-}
-
-#ifdef PAGE_SIZE	/* Avoid this file depending on sys/param.h */
-/*
- * Generic TLB flush for a range of pages. Takes care of SMP.
- * Saves many IPIs for SMP mode.
- */
-static __inline void
-invlpg_range(u_int startva, u_int endva)
-{
-
-	cpu_invlpg_range(startva, endva);
-#ifdef SMP
-	smp_invlpg_range(startva, endva);
-#endif
-}
-#endif
-
-/*
- * Generic global TLB flush (except for thise for pages marked PG_G)
- */
-static __inline void
-invltlb(void)
-{
-
-	cpu_invltlb();
-#ifdef SMP
-	smp_invltlb();
-#endif
-}
-
 static __inline u_int
 rfs(void)
 {
@@ -672,8 +581,6 @@ cpu_critical_exit(critical_t eflags)
 int	breakpoint	__P((void));
 u_int	bsfl		__P((u_int mask));
 u_int	bsrl		__P((u_int mask));
-void	cpu_invlpg	__P((u_int addr));
-void	cpu_invlpg_range __P((u_int start, u_int end));
 void	disable_intr	__P((void));
 void	do_cpuid	__P((u_int ax, u_int *p));
 void	enable_intr	__P((void));
@@ -684,26 +591,15 @@ void	insl		__P((u_int port, void *addr, size_t cnt));
 void	insw		__P((u_int port, void *addr, size_t cnt));
 void	invd		__P((void));
 void	invlpg		__P((u_int addr));
-void	invlpg_range	__P((u_int start, u_int end));
 void	invltlb		__P((void));
 u_short	inw		__P((u_int port));
-void	load_cr0	__P((u_int cr0));
-void	load_cr3	__P((u_int cr3));
-void	load_cr4	__P((u_int cr4));
-void	load_fs		__P((u_int sel));
-void	load_gs		__P((u_int sel));
 void	outb		__P((u_int port, u_char data));
 void	outl		__P((u_int port, u_int data));
 void	outsb		__P((u_int port, void *addr, size_t cnt));
 void	outsl		__P((u_int port, void *addr, size_t cnt));
 void	outsw		__P((u_int port, void *addr, size_t cnt));
 void	outw		__P((u_int port, u_short data));
-u_int	rcr0		__P((void));
 u_int	rcr2		__P((void));
-u_int	rcr3		__P((void));
-u_int	rcr4		__P((void));
-u_int	rfs		__P((void));
-u_int	rgs		__P((void));
 u_int64_t rdmsr		__P((u_int msr));
 u_int64_t rdpmc		__P((u_int pmc));
 u_int64_t rdtsc		__P((void));
@@ -711,12 +607,22 @@ u_int	read_eflags	__P((void));
 void	wbinvd		__P((void));
 void	write_eflags	__P((u_int ef));
 void	wrmsr		__P((u_int msr, u_int64_t newval));
+u_int	rfs		__P((void));
+u_int	rgs		__P((void));
+void	load_fs		__P((u_int sel));
+void	load_gs		__P((u_int sel));
 critical_t cpu_critical_enter __P((void));
 void	cpu_critical_exit __P((critical_t eflags));
 
 #endif	/* __GNUC__ */
 
+void	load_cr0	__P((u_int cr0));
+void	load_cr3	__P((u_int cr3));
+void	load_cr4	__P((u_int cr4));
 void	ltr		__P((u_short sel));
+u_int	rcr0		__P((void));
+u_int	rcr3		__P((void));
+u_int	rcr4		__P((void));
 void    reset_dbregs    __P((void));
 __END_DECLS
 
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index 008dfc5..27ee7ae 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -287,14 +287,6 @@ extern pt_entry_t *SMPpt;
 
 struct pcb stoppcbs[MAXCPU];
 
-#ifdef APIC_IO
-/* Variables needed for SMP tlb shootdown. */
-u_int smp_tlb_addr1;
-u_int smp_tlb_addr2;
-volatile int smp_tlb_wait;
-static struct mtx smp_tlb_mtx;
-#endif
-
 /*
  * Local data and functions.
  */
@@ -343,9 +335,6 @@ init_locks(void)
 #ifdef USE_COMLOCK
 	mtx_init(&com_mtx, "com", MTX_SPIN);
 #endif /* USE_COMLOCK */
-#ifdef APIC_IO
-	mtx_init(&smp_tlb_mtx, "tlb", MTX_SPIN);
-#endif
 }
 
 /*
@@ -615,10 +604,6 @@ mp_enable(u_int boot_addr)
 	/* install an inter-CPU IPI for TLB invalidation */
 	setidt(XINVLTLB_OFFSET, Xinvltlb,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-	setidt(XINVLPG_OFFSET, Xinvlpg,
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-	setidt(XINVLRNG_OFFSET, Xinvlrng,
-	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 
 	/* install an inter-CPU IPI for forwarding hardclock() */
 	setidt(XHARDCLOCK_OFFSET, Xhardclock,
@@ -2201,198 +2186,42 @@ start_ap(int logical_cpu, u_int boot_addr)
 	return 0;		/* return FAILURE */
 }
 
-#if defined(APIC_IO)
-
-#ifdef COUNT_XINVLTLB_HITS
-u_int xhits_gbl[MAXCPU];
-u_int xhits_pg[MAXCPU];
-u_int xhits_rng[MAXCPU];
-SYSCTL_NODE(_debug, OID_AUTO, xhits, CTLFLAG_RW, 0, "");
-SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, global, CTLFLAG_RW, &xhits_gbl,
-    sizeof(xhits_gbl), "IU", "");
-SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, page, CTLFLAG_RW, &xhits_pg,
-    sizeof(xhits_pg), "IU", "");
-SYSCTL_OPAQUE(_debug_xhits, OID_AUTO, range, CTLFLAG_RW, &xhits_rng,
-    sizeof(xhits_rng), "IU", "");
-
-u_int ipi_global;
-u_int ipi_page;
-u_int ipi_range;
-u_int ipi_range_size;
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_global, CTLFLAG_RW, &ipi_global, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_page, CTLFLAG_RW, &ipi_page, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range, CTLFLAG_RW, &ipi_range, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_range_size, CTLFLAG_RW, &ipi_range_size,
-    0, "");
-
-u_int ipi_masked_global;
-u_int ipi_masked_page;
-u_int ipi_masked_range;
-u_int ipi_masked_range_size;
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_global, CTLFLAG_RW,
-    &ipi_masked_global, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_page, CTLFLAG_RW,
-    &ipi_masked_page, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range, CTLFLAG_RW,
-    &ipi_masked_range, 0, "");
-SYSCTL_INT(_debug_xhits, OID_AUTO, ipi_masked_range_size, CTLFLAG_RW,
-    &ipi_masked_range_size, 0, "");
-#endif
-
 /*
  * Flush the TLB on all other CPU's
+ *
+ * XXX: Needs to handshake and wait for completion before proceding.
  */
-static void
-smp_tlb_shootdown(u_int vector, u_int addr1, u_int addr2)
-{
-	u_int ncpu;
-	register_t eflags;
-
-	ncpu = mp_ncpus - 1;	/* does not shootdown self */
-	if (ncpu < 1)
-		return;		/* no other cpus */
-	eflags = read_eflags();
-	if ((eflags & PSL_I) == 0)
-		panic("absolutely cannot call smp_ipi_shootdown with interrupts already disabled");
-	mtx_lock_spin(&smp_tlb_mtx);
-	smp_tlb_addr1 = addr1;
-	smp_tlb_addr2 = addr2;
-	smp_tlb_wait = 0;
-	ipi_all_but_self(vector);
-	while (atomic_load_acq_int(&smp_tlb_wait) < ncpu)
-		/* XXX cpu_pause() */ ;
-	mtx_unlock_spin(&smp_tlb_mtx);
-}
-
-static void
-smp_targeted_tlb_shootdown(u_int mask, u_int vector, u_int addr1, u_int addr2)
-{
-	u_int m;
-	int i, ncpu, othercpus;
-	register_t eflags;
-
-	othercpus = mp_ncpus - 1;
-	if (mask == (u_int)-1) {
-		ncpu = othercpus;
-		if (ncpu < 1)
-			return;
-	} else {
-		/* XXX there should be a pcpu self mask */
-		mask &= ~(1 << PCPU_GET(cpuid));
-		if (mask == 0)
-			return;
-		/* Count the target cpus */
-		ncpu = 0;
-		m = mask;
-		while ((i = ffs(m)) != 0) {
-			m >>= i;
-			ncpu++;
-		}
-		if (ncpu > othercpus) {
-			/* XXX this should be a panic offence */
-			printf("SMP: tlb shootdown to %d other cpus (only have %d)\n",
-			    ncpu, othercpus);
-			ncpu = othercpus;
-		}
-		/* XXX should be a panic, implied by mask == 0 above */
-		if (ncpu < 1)
-			return;
-	}
-	eflags = read_eflags();
-	if ((eflags & PSL_I) == 0)
-		panic("absolutely cannot call smp_targeted_ipi_shootdown with interrupts already disabled");
-	mtx_lock_spin(&smp_tlb_mtx);
-	smp_tlb_addr1 = addr1;
-	smp_tlb_addr2 = addr2;
-	smp_tlb_wait = 0;
-	if (mask == (u_int)-1)
-		ipi_all_but_self(vector);
-	else
-		ipi_selected(mask, vector);
-	while (atomic_load_acq_int(&smp_tlb_wait) < ncpu)
-		/* XXX cpu_pause() */ ;
-	mtx_unlock_spin(&smp_tlb_mtx);
-}
-#endif
-
 void
 smp_invltlb(void)
 {
 #if defined(APIC_IO)
-	if (smp_started) {
-		smp_tlb_shootdown(IPI_INVLTLB, 0, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_global++;
-#endif
-	}
+	if (smp_started)
+		ipi_all_but_self(IPI_INVLTLB);
 #endif  /* APIC_IO */
 }
 
 void
-smp_invlpg(u_int addr)
+invlpg(u_int addr)
 {
-#if defined(APIC_IO)
-	if (smp_started) {
-		smp_tlb_shootdown(IPI_INVLPG, addr, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_page++;
-#endif
-	}
-#endif  /* APIC_IO */
-}
+	__asm   __volatile("invlpg (%0)"::"r"(addr):"memory");
 
-void
-smp_invlpg_range(u_int addr1, u_int addr2)
-{
-#if defined(APIC_IO)
-	if (smp_started) {
-		smp_tlb_shootdown(IPI_INVLRNG, addr1, addr2);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_range++;
-		ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
-#endif
-	}
-#endif  /* APIC_IO */
+	/* send a message to the other CPUs */
+	smp_invltlb();
 }
 
 void
-smp_masked_invltlb(u_int mask)
+invltlb(void)
 {
-#if defined(APIC_IO)
-	if (smp_started) {
-		smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, 0, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_masked_global++;
-#endif
-	}
-#endif  /* APIC_IO */
-}
+	u_long  temp;
 
-void
-smp_masked_invlpg(u_int mask, u_int addr)
-{
-#if defined(APIC_IO)
-	if (smp_started) {
-		smp_targeted_tlb_shootdown(mask, IPI_INVLPG, addr, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_masked_page++;
-#endif
-	}
-#endif  /* APIC_IO */
-}
+	/*
+	 * This should be implemented as load_cr3(rcr3()) when load_cr3() is
+	 * inlined.
+	 */
+	__asm __volatile("movl %%cr3, %0; movl %0, %%cr3":"=r"(temp) :: "memory");
 
-void
-smp_masked_invlpg_range(u_int mask, u_int addr1, u_int addr2)
-{
-#if defined(APIC_IO)
-	if (smp_started) {
-		smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, addr1, addr2);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_masked_range++;
-		ipi_masked_range_size += (addr2 - addr1) / PAGE_SIZE;
-#endif
-	}
-#endif  /* APIC_IO */
+	/* send a message to the other CPUs */
+	smp_invltlb();
 }
 
 
@@ -2451,9 +2280,6 @@ ap_init(void)
 	/* Build our map of 'other' CPUs. */
 	PCPU_SET(other_cpus, all_cpus & ~PCPU_GET(cpumask));
 
-	if (bootverbose)
-		apic_dump("ap_init()");
-
 	printf("SMP: AP CPU #%d Launched!\n", PCPU_GET(cpuid));
 
 	if (smp_cpus == mp_ncpus) {
@@ -2486,8 +2312,7 @@ forwarded_statclock(struct trapframe frame)
 {
 
 	mtx_lock_spin(&sched_lock);
-	statclock_process(curthread->td_kse, TRAPF_PC(&frame),
-	    TRAPF_USERMODE(&frame));
+	statclock_process(curthread->td_kse, TRAPF_PC(&frame), TRAPF_USERMODE(&frame));
 	mtx_unlock_spin(&sched_lock);
 }
 
diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h
index 618bb3f..7358a9e 100644
--- a/sys/i386/include/pmap.h
+++ b/sys/i386/include/pmap.h
@@ -151,7 +151,7 @@ extern pt_entry_t PTmap[], APTmap[];
 extern pd_entry_t PTD[], APTD[];
 extern pd_entry_t PTDpde, APTDpde;
 
-extern pd_entry_t *IdlePTD;	/* physical address of "Idle" state directory */
+extern pd_entry_t IdlePTD;	/* physical address of "Idle" state directory */
 #endif
 
 #ifdef _KERNEL
@@ -267,7 +267,9 @@ void	*pmap_mapdev __P((vm_offset_t, vm_size_t));
 void	pmap_unmapdev __P((vm_offset_t, vm_size_t));
 pt_entry_t *pmap_pte __P((pmap_t, vm_offset_t)) __pure2;
 vm_page_t pmap_use_pt __P((pmap_t, vm_offset_t));
+#ifdef SMP
 void	pmap_set_opt __P((void));
+#endif
 
 #endif /* _KERNEL */
 
diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h
index 4136c20..34228e2 100644
--- a/sys/i386/include/smp.h
+++ b/sys/i386/include/smp.h
@@ -51,8 +51,6 @@ extern int current_postcode;  /** XXX currently in mp_machdep.c */
  * Interprocessor interrupts for SMP.
  */
 #define	IPI_INVLTLB		XINVLTLB_OFFSET
-#define	IPI_INVLPG		XINVLPG_OFFSET
-#define	IPI_INVLRNG		XINVLRNG_OFFSET
 #define	IPI_RENDEZVOUS		XRENDEZVOUS_OFFSET
 #define	IPI_AST			XCPUAST_OFFSET
 #define	IPI_STOP		XCPUSTOP_OFFSET
@@ -109,6 +107,7 @@ void	assign_apic_irq		__P((int apic, int intpin, int irq));
 void	revoke_apic_irq		__P((int irq));
 void	bsp_apic_configure	__P((void));
 void	init_secondary		__P((void));
+void	smp_invltlb		__P((void));
 void	forward_statclock	__P((void));
 void	forwarded_statclock	__P((struct trapframe frame));
 void	forward_hardclock	__P((void));
diff --git a/sys/i386/include/smptests.h b/sys/i386/include/smptests.h
index ea8e84b..d666148 100644
--- a/sys/i386/include/smptests.h
+++ b/sys/i386/include/smptests.h
@@ -90,6 +90,13 @@
  */
 #define APIC_INTR_REORDER
 
+/*
+ * Redirect clock interrupts to a higher priority (fast intr) vector,
+ * while still using the slow interrupt handler. Only effective when 
+ * APIC_INTR_REORDER is defined.
+ */
+#define APIC_INTR_HIGHPRI_CLOCK
+
 #endif /* APIC_IO */
 
 /*