Don't wait forever for CPUs to stop or restart. Instead, give up after a

timeout.  If DIAGNOSTIC is turned on, then display a message to the console
with a map of which CPUs failed to stop or restart.  This gives an SMP box
at least a fighting chance of getting into DDB if one of the other CPUs has
interrupts disabled.

7 files changed, 133 insertions, 14 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index 4788ee4..0c1fc03 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index 4788ee4..0c1fc03 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index 4788ee4..0c1fc03 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index 4788ee4..0c1fc03 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index 4788ee4..0c1fc03 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index 4788ee4..0c1fc03 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index 4788ee4..0c1fc03 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -2275,15 +2275,23 @@ invltlb(void)
 int
 stop_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	/* send the Xcpustop IPI to all CPUs in map */
 	selected_apic_ipi(map, XCPUSTOP_OFFSET, APIC_DELMODE_FIXED);
 	
-	while ((stopped_cpus & map) != map)
+	while (count++ < 100000 && (stopped_cpus & map) != map)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != map)
+		printf("Warning: CPUs 0x%x did not stop!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }
 
@@ -2304,14 +2312,23 @@ stop_cpus(u_int map)
 int
 restart_cpus(u_int map)
 {
+	int count = 0;
+
 	if (!smp_started)
 		return 0;
 
 	started_cpus = map;		/* signal other cpus to restart */
 
-	while ((stopped_cpus & map) != 0) /* wait for each to clear its bit */
+	/* wait for each to clear its bit */
+	while (count++ < 100000 && (stopped_cpus & map) != 0)
 		/* spin */ ;
 
+#ifdef DIAGNOSTIC
+	if ((stopped_cpus & map) != 0)
+		printf("Warning: CPUs 0x%x did not restart!\n",
+		    (~(stopped_cpus & map)) & map);
+#endif
+
 	return 1;
 }