For amd64 non-PCID machines, and for i386 machines with support for

the PG_G global pte flag, pmap_invalidate_all() fails to flush global
TLB entries [*].  This is because TLB shootdown handler for such
configs reloads CR3, and on i386 pmap_invalidate_all() does the same
for the initiating CPU.  Note that current code does not issue total
invalidation requests for the kernel_pmap.

Rename amd64 function invltlb_globpcid() to invltlb_glob(), it is not
specific for PCID for quite some time, and implement the same
functionality for i386.  Use the function instead of invltlb() in
shootdown handlers and in i386 pmap_invalidate_all(), but only for the
kernel pmap (which maps pages with the PG_G attribute set), which
takes care of PG_G TLB entries on flush.

To detect the affected pmap in i386 TLB shootdown handler, pmap should
be passed to the smp_masked_invltlb() function, which makes amd64 and
i386 TLB shootdown code almost identical.  Merge the code under x86/.

Noted by:	jhb [*]
Reviewed by:	cem, jhb, pho
Tested by:	pho
Sponsored by:	The FreeBSD Foundation
Differential revision:	https://reviews.freebsd.org/D4346

4 files changed, 8 insertions, 162 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 3318ddd..d87d38c 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -87,11 +87,6 @@ extern	struct pcpu __pcpu[];
 char *doublefault_stack;
 char *nmi_stack;
 
-/* Variables needed for SMP tlb shootdown. */
-static vm_offset_t smp_tlb_addr1, smp_tlb_addr2;
-static pmap_t smp_tlb_pmap;
-volatile int smp_tlb_wait;
-
 extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
 
 /*
@@ -410,121 +405,6 @@ start_ap(int apic_id)
 	return 0;		/* return FAILURE */
 }
 
-/*
- * Flush the TLB on other CPU's
- */
-
-static void
-smp_targeted_tlb_shootdown(cpuset_t mask, u_int vector, pmap_t pmap,
-    vm_offset_t addr1, vm_offset_t addr2)
-{
-	int cpu, ncpu, othercpus;
-
-	othercpus = mp_ncpus - 1;	/* does not shootdown self */
-
-	/*
-	 * Check for other cpus.  Return if none.
-	 */
-	if (CPU_ISFULLSET(&mask)) {
-		if (othercpus < 1)
-			return;
-	} else {
-		CPU_CLR(PCPU_GET(cpuid), &mask);
-		if (CPU_EMPTY(&mask))
-			return;
-	}
-
-	if (!(read_rflags() & PSL_I))
-		panic("%s: interrupts disabled", __func__);
-	mtx_lock_spin(&smp_ipi_mtx);
-	smp_tlb_addr1 = addr1;
-	smp_tlb_addr2 = addr2;
-	smp_tlb_pmap = pmap;
-	smp_tlb_wait =  0;
-	if (CPU_ISFULLSET(&mask)) {
-		ncpu = othercpus;
-		ipi_all_but_self(vector);
-	} else {
-		ncpu = 0;
-		while ((cpu = CPU_FFS(&mask)) != 0) {
-			cpu--;
-			CPU_CLR(cpu, &mask);
-			CTR3(KTR_SMP, "%s: cpu: %d ipi: %x", __func__,
-			    cpu, vector);
-			ipi_send_cpu(cpu, vector);
-			ncpu++;
-		}
-	}
-	while (smp_tlb_wait < ncpu)
-		ia32_pause();
-	mtx_unlock_spin(&smp_ipi_mtx);
-}
-
-void
-smp_masked_invltlb(cpuset_t mask, pmap_t pmap)
-{
-
-	if (smp_started) {
-		smp_targeted_tlb_shootdown(mask, IPI_INVLTLB, pmap, 0, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_global++;
-#endif
-	}
-}
-
-void
-smp_masked_invlpg(cpuset_t mask, vm_offset_t addr)
-{
-
-	if (smp_started) {
-		smp_targeted_tlb_shootdown(mask, IPI_INVLPG, NULL, addr, 0);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_page++;
-#endif
-	}
-}
-
-void
-smp_masked_invlpg_range(cpuset_t mask, vm_offset_t addr1, vm_offset_t addr2)
-{
-
-	if (smp_started) {
-		smp_targeted_tlb_shootdown(mask, IPI_INVLRNG, NULL,
-		    addr1, addr2);
-#ifdef COUNT_XINVLTLB_HITS
-		ipi_range++;
-		ipi_range_size += (addr2 - addr1) / PAGE_SIZE;
-#endif
-	}
-}
-
-void
-smp_cache_flush(void)
-{
-
-	if (smp_started) {
-		smp_targeted_tlb_shootdown(all_cpus, IPI_INVLCACHE, NULL,
-		    0, 0);
-	}
-}
-
-/*
- * Handlers for TLB related IPIs
- */
-void
-invltlb_handler(void)
-{
-#ifdef COUNT_XINVLTLB_HITS
-	xhits_gbl[PCPU_GET(cpuid)]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
-	(*ipi_invltlb_counts[PCPU_GET(cpuid)])++;
-#endif /* COUNT_IPIS */
-
-	invltlb();
-	atomic_add_int(&smp_tlb_wait, 1);
-}
-
 void
 invltlb_invpcid_handler(void)
 {
@@ -556,7 +436,7 @@ invltlb_pcid_handler(void)
 #endif /* COUNT_IPIS */
 
 	if (smp_tlb_pmap == kernel_pmap) {
-		invltlb_globpcid();
+		invltlb_glob();
 	} else {
 		/*
 		 * The current pmap might not be equal to
@@ -572,38 +452,3 @@ invltlb_pcid_handler(void)
 	}
 	atomic_add_int(&smp_tlb_wait, 1);
 }
-
-void
-invlpg_handler(void)
-{
-#ifdef COUNT_XINVLTLB_HITS
-	xhits_pg[PCPU_GET(cpuid)]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
-	(*ipi_invlpg_counts[PCPU_GET(cpuid)])++;
-#endif /* COUNT_IPIS */
-
-	invlpg(smp_tlb_addr1);
-	atomic_add_int(&smp_tlb_wait, 1);
-}
-
-void
-invlrng_handler(void)
-{
-	vm_offset_t addr;
-
-#ifdef COUNT_XINVLTLB_HITS
-	xhits_rng[PCPU_GET(cpuid)]++;
-#endif /* COUNT_XINVLTLB_HITS */
-#ifdef COUNT_IPIS
-	(*ipi_invlrng_counts[PCPU_GET(cpuid)])++;
-#endif /* COUNT_IPIS */
-
-	addr = smp_tlb_addr1;
-	do {
-		invlpg(addr);
-		addr += PAGE_SIZE;
-	} while (addr < smp_tlb_addr2);
-
-	atomic_add_int(&smp_tlb_wait, 1);
-}
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 883c1c5..9f9654a 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -1321,7 +1321,7 @@ pmap_update_pde_invalidate(pmap_t pmap, vm_offset_t va, pd_entry_t newpde)
 		 * Promotion: flush every 4KB page mapping from the TLB,
 		 * including any global (PG_G) mappings.
 		 */
-		invltlb_globpcid();
+		invltlb_glob();
 	}
 }
 #ifdef SMP
@@ -1482,7 +1482,7 @@ pmap_invalidate_all(pmap_t pmap)
 			bzero(&d, sizeof(d));
 			invpcid(&d, INVPCID_CTXGLOB);
 		} else {
-			invltlb_globpcid();
+			invltlb_glob();
 		}
 		mask = &all_cpus;
 	} else {
@@ -1653,7 +1653,7 @@ pmap_invalidate_all(pmap_t pmap)
 			bzero(&d, sizeof(d));
 			invpcid(&d, INVPCID_CTXGLOB);
 		} else {
-			invltlb_globpcid();
+			invltlb_glob();
 		}
 	} else if (pmap == PCPU_GET(curpmap)) {
 		if (pmap_pcid_enabled) {
diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h
index f2961d6..f234873 100644
--- a/sys/amd64/include/cpufunc.h
+++ b/sys/amd64/include/cpufunc.h
@@ -505,7 +505,7 @@ invltlb(void)
  * Operations that Invalidate TLBs and Paging-Structure Caches.
  */
 static __inline void
-invltlb_globpcid(void)
+invltlb_glob(void)
 {
 	uint64_t cr4;
 
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index 850289a..9c93e5c 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -25,6 +25,8 @@
 #include <x86/apicvar.h>
 #include <machine/pcb.h>
 
+struct pmap;
+
 /* global symbols in mpboot.S */
 extern char			mptramp_start[];
 extern char			mptramp_end[];
@@ -53,6 +55,7 @@ extern u_int ipi_global;
 extern u_int ipi_page;
 extern u_int ipi_range;
 extern u_int ipi_range_size;
+extern struct pmap *smp_tlb_pmap;
 
 extern volatile int smp_tlb_wait;
 
@@ -86,8 +89,6 @@ inthand_t
 	IDTVEC(justreturn),	/* interrupt CPU with minimum overhead */
 	IDTVEC(rendezvous);	/* handle CPU rendezvous */
 
-struct pmap;
-
 /* functions in mp_machdep.c */
 void	assign_cpu_ids(void);
 void	cpu_add(u_int apic_id, char boot_cpu);