Commit Doug White and Alan Cox's fix for the cross-ipi smp deadlock.

We were obtaining different spin mutexes (which disable interrupts after
aquisition) and spin waiting for delivery.  For example, KSE processes
do LDT operations which use smp_rendezvous, while other parts of the
system are doing things like tlb shootdowns with a different mutex.

This patch uses the common smp_rendezvous mutex for all MD home-grown
IPIs that spinwait for delivery.  Having the single mutex means that
the spinloop to aquire it will enable interrupts periodically, thus
avoiding the cross-ipi deadlock.

Obtained from: dwhite, alc
Reviewed by:   jhb

6 files changed, 27 insertions, 24 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 69c5c0f..ff9b08b 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -103,7 +103,6 @@ struct pcb stoppcbs[MAXCPU];
 vm_offset_t smp_tlb_addr1;
 vm_offset_t smp_tlb_addr2;
 volatile int smp_tlb_wait;
-struct mtx smp_tlb_mtx;
 
 extern inthand_t IDTVEC(fast_syscall), IDTVEC(fast_syscall32);
 
@@ -319,8 +318,6 @@ cpu_mp_start(void)
 	/* Install an inter-CPU IPI for CPU stop/restart */
 	setidt(IPI_STOP, IDTVEC(cpustop), SDT_SYSIGT, SEL_KPL, 0);
 
-	mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN);
-
 	/* Set boot_cpu_id if needed. */
 	if (boot_cpu_id == -1) {
 		boot_cpu_id = PCPU_GET(apic_id);
@@ -711,7 +708,7 @@ smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2)
 	ncpu = mp_ncpus - 1;	/* does not shootdown self */
 	if (ncpu < 1)
 		return;		/* no other cpus */
-	mtx_assert(&smp_tlb_mtx, MA_OWNED);
+	mtx_assert(&smp_rv_mtx, MA_OWNED);
 	smp_tlb_addr1 = addr1;
 	smp_tlb_addr2 = addr2;
 	atomic_store_rel_int(&smp_tlb_wait, 0);
@@ -797,7 +794,7 @@ smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offse
 		if (ncpu < 1)
 			return;
 	}
-	mtx_assert(&smp_tlb_mtx, MA_OWNED);
+	mtx_assert(&smp_rv_mtx, MA_OWNED);
 	smp_tlb_addr1 = addr1;
 	smp_tlb_addr2 = addr2;
 	atomic_store_rel_int(&smp_tlb_wait, 0);
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 6421ee8..3f09fdd 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -631,7 +631,7 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 	if (smp_started) {
 		if (!(read_rflags() & PSL_I))
 			panic("%s: interrupts disabled", __func__);
-		mtx_lock_spin(&smp_tlb_mtx);
+		mtx_lock_spin(&smp_rv_mtx);
 	} else
 		critical_enter();
 	/*
@@ -652,7 +652,7 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 			smp_masked_invlpg(pmap->pm_active & other_cpus, va);
 	}
 	if (smp_started)
-		mtx_unlock_spin(&smp_tlb_mtx);
+		mtx_unlock_spin(&smp_rv_mtx);
 	else
 		critical_exit();
 }
@@ -667,7 +667,7 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 	if (smp_started) {
 		if (!(read_rflags() & PSL_I))
 			panic("%s: interrupts disabled", __func__);
-		mtx_lock_spin(&smp_tlb_mtx);
+		mtx_lock_spin(&smp_rv_mtx);
 	} else
 		critical_enter();
 	/*
@@ -691,7 +691,7 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 			    sva, eva);
 	}
 	if (smp_started)
-		mtx_unlock_spin(&smp_tlb_mtx);
+		mtx_unlock_spin(&smp_rv_mtx);
 	else
 		critical_exit();
 }
@@ -705,7 +705,7 @@ pmap_invalidate_all(pmap_t pmap)
 	if (smp_started) {
 		if (!(read_rflags() & PSL_I))
 			panic("%s: interrupts disabled", __func__);
-		mtx_lock_spin(&smp_tlb_mtx);
+		mtx_lock_spin(&smp_rv_mtx);
 	} else
 		critical_enter();
 	/*
@@ -726,7 +726,7 @@ pmap_invalidate_all(pmap_t pmap)
 			smp_masked_invltlb(pmap->pm_active & other_cpus);
 	}
 	if (smp_started)
-		mtx_unlock_spin(&smp_tlb_mtx);
+		mtx_unlock_spin(&smp_rv_mtx);
 	else
 		critical_exit();
 }
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 781f410..e817260 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -177,7 +177,6 @@ struct pcb stoppcbs[MAXCPU];
 vm_offset_t smp_tlb_addr1;
 vm_offset_t smp_tlb_addr2;
 volatile int smp_tlb_wait;
-struct mtx smp_tlb_mtx;
 
 /*
  * Local data and functions.
@@ -387,7 +386,6 @@ cpu_mp_start(void)
 	setidt(IPI_STOP, IDTVEC(cpustop),
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
 
-	mtx_init(&smp_tlb_mtx, "tlb", NULL, MTX_SPIN);
 
 	/* Set boot_cpu_id if needed. */
 	if (boot_cpu_id == -1) {
@@ -912,7 +910,7 @@ smp_tlb_shootdown(u_int vector, vm_offset_t addr1, vm_offset_t addr2)
 	ncpu = mp_ncpus - 1;	/* does not shootdown self */
 	if (ncpu < 1)
 		return;		/* no other cpus */
-	mtx_assert(&smp_tlb_mtx, MA_OWNED);
+	mtx_assert(&smp_rv_mtx, MA_OWNED);
 	smp_tlb_addr1 = addr1;
 	smp_tlb_addr2 = addr2;
 	atomic_store_rel_int(&smp_tlb_wait, 0);
@@ -998,7 +996,7 @@ smp_targeted_tlb_shootdown(u_int mask, u_int vector, vm_offset_t addr1, vm_offse
 		if (ncpu < 1)
 			return;
 	}
-	mtx_assert(&smp_tlb_mtx, MA_OWNED);
+	mtx_assert(&smp_rv_mtx, MA_OWNED);
 	smp_tlb_addr1 = addr1;
 	smp_tlb_addr2 = addr2;
 	atomic_store_rel_int(&smp_tlb_wait, 0);
diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c
index e179457..b5b8da4 100644
--- a/sys/i386/i386/pmap.c
+++ b/sys/i386/i386/pmap.c
@@ -582,7 +582,7 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 	if (smp_started) {
 		if (!(read_eflags() & PSL_I))
 			panic("%s: interrupts disabled", __func__);
-		mtx_lock_spin(&smp_tlb_mtx);
+		mtx_lock_spin(&smp_rv_mtx);
 	} else
 		critical_enter();
 	/*
@@ -603,7 +603,7 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 			smp_masked_invlpg(pmap->pm_active & other_cpus, va);
 	}
 	if (smp_started)
-		mtx_unlock_spin(&smp_tlb_mtx);
+		mtx_unlock_spin(&smp_rv_mtx);
 	else
 		critical_exit();
 }
@@ -618,7 +618,7 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 	if (smp_started) {
 		if (!(read_eflags() & PSL_I))
 			panic("%s: interrupts disabled", __func__);
-		mtx_lock_spin(&smp_tlb_mtx);
+		mtx_lock_spin(&smp_rv_mtx);
 	} else
 		critical_enter();
 	/*
@@ -642,7 +642,7 @@ pmap_invalidate_range(pmap_t pmap, vm_offset_t sva, vm_offset_t eva)
 			    sva, eva);
 	}
 	if (smp_started)
-		mtx_unlock_spin(&smp_tlb_mtx);
+		mtx_unlock_spin(&smp_rv_mtx);
 	else
 		critical_exit();
 }
@@ -656,7 +656,7 @@ pmap_invalidate_all(pmap_t pmap)
 	if (smp_started) {
 		if (!(read_eflags() & PSL_I))
 			panic("%s: interrupts disabled", __func__);
-		mtx_lock_spin(&smp_tlb_mtx);
+		mtx_lock_spin(&smp_rv_mtx);
 	} else
 		critical_enter();
 	/*
@@ -677,7 +677,7 @@ pmap_invalidate_all(pmap_t pmap)
 			smp_masked_invltlb(pmap->pm_active & other_cpus);
 	}
 	if (smp_started)
-		mtx_unlock_spin(&smp_tlb_mtx);
+		mtx_unlock_spin(&smp_rv_mtx);
 	else
 		critical_exit();
 }
@@ -1301,7 +1301,7 @@ pmap_lazyfix(pmap_t pmap)
 	while ((mask = pmap->pm_active) != 0) {
 		spins = 50000000;
 		mask = mask & -mask;	/* Find least significant set bit */
-		mtx_lock_spin(&smp_tlb_mtx);
+		mtx_lock_spin(&smp_rv_mtx);
 #ifdef PAE
 		lazyptd = vtophys(pmap->pm_pdpt);
 #else
@@ -1321,7 +1321,7 @@ pmap_lazyfix(pmap_t pmap)
 					break;
 			}
 		}
-		mtx_unlock_spin(&smp_tlb_mtx);
+		mtx_unlock_spin(&smp_rv_mtx);
 		if (spins == 0)
 			printf("pmap_lazyfix: spun for 50000000\n");
 	}
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index b1be3fa..945ef6f 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -102,7 +102,14 @@ static void (*smp_rv_action_func)(void *arg);
 static void (*smp_rv_teardown_func)(void *arg);
 static void *smp_rv_func_arg;
 static volatile int smp_rv_waiters[2];
-static struct mtx smp_rv_mtx;
+
+/* 
+ * Shared mutex to restrict busywaits between smp_rendezvous() and
+ * smp(_targeted)_tlb_shootdown().  A deadlock occurs if both of these
+ * functions trigger at once and cause multiple CPUs to busywait with
+ * interrupts disabled. 
+ */
+struct mtx smp_rv_mtx;
 
 /*
  * Let the MD SMP code initialize mp_maxid very early if it can.
diff --git a/sys/sys/smp.h b/sys/sys/smp.h
index b9d57e2..5a1ab86 100644
--- a/sys/sys/smp.h
+++ b/sys/sys/smp.h
@@ -95,6 +95,7 @@ void	forward_roundrobin(void);
 int	restart_cpus(cpumask_t);
 int	stop_cpus(cpumask_t);
 void	smp_rendezvous_action(void);
+extern	struct mtx smp_rv_mtx;
 #endif /* SMP */
 void	smp_rendezvous(void (*)(void *), 
 		       void (*)(void *),