A clean fix for the spl "deadlock before smp_active" problem.

Added a new variable, 'bsp_apic_ready', which is set as soon as the bootstrap
CPU has initialized its local APIC.  Conditionalize the GENSPLR functions
to call ss_lock ONLY after bsp_apic_ready is TRUE;  This should prevent
any problems with races between the time the 1st AP becomes ready and the
time smp_active is set.

11 files changed, 49 insertions, 29 deletions

diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c
index b02fde7..7197b67 100644
--- a/sys/amd64/amd64/mp_machdep.c
+++ b/sys/amd64/amd64/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.30 1997/08/24 20:19:47 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -207,6 +207,7 @@ int	current_postcode;
 /** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
+int	bsp_apic_ready = 0;	/* flags useability of BSP apic */
 int	mp_ncpus;		/* # of CPUs, including BSP */
 int	mp_naps;		/* # of Applications processors */
 int	mp_nbusses;		/* # of busses */
@@ -1480,6 +1481,7 @@ start_all_aps(u_int boot_addr)
 
 	/* initialize BSP's local APIC */
 	apic_initialize();
+	bsp_apic_ready = 1;
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c
index b02fde7..7197b67 100644
--- a/sys/amd64/amd64/mptable.c
+++ b/sys/amd64/amd64/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.30 1997/08/24 20:19:47 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -207,6 +207,7 @@ int	current_postcode;
 /** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
+int	bsp_apic_ready = 0;	/* flags useability of BSP apic */
 int	mp_ncpus;		/* # of CPUs, including BSP */
 int	mp_naps;		/* # of Applications processors */
 int	mp_nbusses;		/* # of busses */
@@ -1480,6 +1481,7 @@ start_all_aps(u_int boot_addr)
 
 	/* initialize BSP's local APIC */
 	apic_initialize();
+	bsp_apic_ready = 1;
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h
index b02fde7..7197b67 100644
--- a/sys/amd64/include/mptable.h
+++ b/sys/amd64/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.30 1997/08/24 20:19:47 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -207,6 +207,7 @@ int	current_postcode;
 /** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
+int	bsp_apic_ready = 0;	/* flags useability of BSP apic */
 int	mp_ncpus;		/* # of CPUs, including BSP */
 int	mp_naps;		/* # of Applications processors */
 int	mp_nbusses;		/* # of busses */
@@ -1480,6 +1481,7 @@ start_all_aps(u_int boot_addr)
 
 	/* initialize BSP's local APIC */
 	apic_initialize();
+	bsp_apic_ready = 1;
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h
index 6b195f6f..f433b2f 100644
--- a/sys/amd64/include/smp.h
+++ b/sys/amd64/include/smp.h
@@ -6,7 +6,7 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
- * $Id: smp.h,v 1.23 1997/08/21 04:48:45 smp Exp smp $
+ * $Id: smp.h,v 1.24 1997/08/24 20:17:05 smp Exp smp $
  *
  */
 
@@ -85,6 +85,7 @@ u_int	io_apic_read		__P((int, int));
 void	io_apic_write		__P((int, int, u_int));
 
 /* global data in mp_machdep.c */
+extern int			bsp_apic_ready;
 extern int			mp_ncpus;
 extern int			mp_naps;
 extern int			mp_nbusses;
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index b02fde7..7197b67 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.30 1997/08/24 20:19:47 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -207,6 +207,7 @@ int	current_postcode;
 /** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
+int	bsp_apic_ready = 0;	/* flags useability of BSP apic */
 int	mp_ncpus;		/* # of CPUs, including BSP */
 int	mp_naps;		/* # of Applications processors */
 int	mp_nbusses;		/* # of busses */
@@ -1480,6 +1481,7 @@ start_all_aps(u_int boot_addr)
 
 	/* initialize BSP's local APIC */
 	apic_initialize();
+	bsp_apic_ready = 1;
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index b02fde7..7197b67 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.30 1997/08/24 20:19:47 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -207,6 +207,7 @@ int	current_postcode;
 /** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
+int	bsp_apic_ready = 0;	/* flags useability of BSP apic */
 int	mp_ncpus;		/* # of CPUs, including BSP */
 int	mp_naps;		/* # of Applications processors */
 int	mp_nbusses;		/* # of busses */
@@ -1480,6 +1481,7 @@ start_all_aps(u_int boot_addr)
 
 	/* initialize BSP's local APIC */
 	apic_initialize();
+	bsp_apic_ready = 1;
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index b02fde7..7197b67 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.30 1997/08/24 20:19:47 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -207,6 +207,7 @@ int	current_postcode;
 /** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
+int	bsp_apic_ready = 0;	/* flags useability of BSP apic */
 int	mp_ncpus;		/* # of CPUs, including BSP */
 int	mp_naps;		/* # of Applications processors */
 int	mp_nbusses;		/* # of busses */
@@ -1480,6 +1481,7 @@ start_all_aps(u_int boot_addr)
 
 	/* initialize BSP's local APIC */
 	apic_initialize();
+	bsp_apic_ready = 1;
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
diff --git a/sys/i386/include/smp.h b/sys/i386/include/smp.h
index 6b195f6f..f433b2f 100644
--- a/sys/i386/include/smp.h
+++ b/sys/i386/include/smp.h
@@ -6,7 +6,7 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
- * $Id: smp.h,v 1.23 1997/08/21 04:48:45 smp Exp smp $
+ * $Id: smp.h,v 1.24 1997/08/24 20:17:05 smp Exp smp $
  *
  */
 
@@ -85,6 +85,7 @@ u_int	io_apic_read		__P((int, int));
 void	io_apic_write		__P((int, int, u_int));
 
 /* global data in mp_machdep.c */
+extern int			bsp_apic_ready;
 extern int			mp_ncpus;
 extern int			mp_naps;
 extern int			mp_nbusses;
diff --git a/sys/i386/isa/ipl_funcs.c b/sys/i386/isa/ipl_funcs.c
index 0532249..78977eb 100644
--- a/sys/i386/isa/ipl_funcs.c
+++ b/sys/i386/isa/ipl_funcs.c
@@ -23,7 +23,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: ipl_funcs.c,v 1.3 1997/08/24 00:05:18 fsmp Exp $
+ *	$Id: ipl_funcs.c,v 1.7 1997/08/24 20:18:28 smp Exp smp $
  */
 
 #include <sys/types.h>
@@ -98,6 +98,7 @@ splx(unsigned ipl)
 
 #include <machine/param.h>
 #include <machine/smp.h>
+extern int bspEarly; /* XXX */
 
 #if defined(REAL_IFCPL)
 
@@ -146,6 +147,7 @@ softclockpending(void)
 }
 
 
+#ifdef notneeded
 #define	GENSPL(name, set_cpl)			\
 unsigned name(void)				\
 {						\
@@ -159,38 +161,39 @@ unsigned name(void)				\
 						\
 	return (x);				\
 }
+#endif /* notneeded */
 
 /*
- * This version has to check for smp_active,
+ * This version has to check for bsp_apic_ready,
  * as calling simple_lock() (ie ss_lock) before then deadlocks the system.
+ * A sample count of GENSPLR calls before bsp_apic_ready was set: 2193
  */
-#define	GENSPL2(name, set_cpl)			\
+#define	GENSPLR(name, set_cpl)			\
 unsigned name(void)				\
 {						\
 	unsigned x;				\
 						\
-	if (smp_active)				\
+	if (bsp_apic_ready)			\
 		IFCPL_LOCK();			\
 	x = cpl;				\
 	/* XXX test cil */			\
 	set_cpl;				\
-	if (smp_active)				\
+	if (bsp_apic_ready)			\
 		IFCPL_UNLOCK();			\
 						\
 	return (x);				\
 }
 
-GENSPL2(splbio, cpl |= bio_imask)
-GENSPL2(splclock, cpl = HWI_MASK | SWI_MASK)
-GENSPL2(splimp, cpl |= net_imask)
-GENSPL2(splnet, cpl |= SWI_NET_MASK)
-GENSPL2(splsoftclock, cpl = SWI_CLOCK_MASK)
-GENSPL2(splsofttty, cpl |= SWI_TTY_MASK)
-GENSPL2(splstatclock, cpl |= stat_imask)
-GENSPL2(splvm, cpl |= net_imask | bio_imask)
-
-GENSPL2(splhigh, cpl = HWI_MASK | SWI_MASK)
-GENSPL2(spltty, cpl |= tty_imask)
+GENSPLR(splbio, cpl |= bio_imask)
+GENSPLR(splclock, cpl = HWI_MASK | SWI_MASK)
+GENSPLR(splhigh, cpl = HWI_MASK | SWI_MASK)
+GENSPLR(splimp, cpl |= net_imask)
+GENSPLR(splnet, cpl |= SWI_NET_MASK)
+GENSPLR(splsoftclock, cpl = SWI_CLOCK_MASK)
+GENSPLR(splsofttty, cpl |= SWI_TTY_MASK)
+GENSPLR(splstatclock, cpl |= stat_imask)
+GENSPLR(spltty, cpl |= tty_imask)
+GENSPLR(splvm, cpl |= net_imask | bio_imask)
 
 
 void
@@ -211,18 +214,18 @@ spl0(void)
 void
 splx(unsigned ipl)
 {
-	if (smp_active)
+	if (bsp_apic_ready)
 		IFCPL_LOCK();
 
 	/* XXX test cil */
 	cpl = ipl;
 	if (ipending & ~ipl) {
-		if (smp_active)
+		if (bsp_apic_ready)
 			IFCPL_UNLOCK();
 		splz();
 	}
 	else
-		if (smp_active)
+		if (bsp_apic_ready)
 			IFCPL_UNLOCK();
 }
 
diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c
index b02fde7..7197b67 100644
--- a/sys/kern/subr_smp.c
+++ b/sys/kern/subr_smp.c
@@ -22,7 +22,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: mp_machdep.c,v 1.29 1997/08/21 04:53:27 smp Exp smp $
+ *	$Id: mp_machdep.c,v 1.30 1997/08/24 20:19:47 smp Exp smp $
  */
 
 #include "opt_smp.h"
@@ -207,6 +207,7 @@ int	current_postcode;
 /** XXX FIXME: what system files declare these??? */
 extern struct region_descriptor r_gdt, r_idt;
 
+int	bsp_apic_ready = 0;	/* flags useability of BSP apic */
 int	mp_ncpus;		/* # of CPUs, including BSP */
 int	mp_naps;		/* # of Applications processors */
 int	mp_nbusses;		/* # of busses */
@@ -1480,6 +1481,7 @@ start_all_aps(u_int boot_addr)
 
 	/* initialize BSP's local APIC */
 	apic_initialize();
+	bsp_apic_ready = 1;
 
 	/* install the AP 1st level boot code */
 	install_ap_tramp(boot_addr);
diff --git a/sys/sys/smp.h b/sys/sys/smp.h
index 6b195f6f..f433b2f 100644
--- a/sys/sys/smp.h
+++ b/sys/sys/smp.h
@@ -6,7 +6,7 @@
  * this stuff is worth it, you can buy me a beer in return.   Poul-Henning Kamp
  * ----------------------------------------------------------------------------
  *
- * $Id: smp.h,v 1.23 1997/08/21 04:48:45 smp Exp smp $
+ * $Id: smp.h,v 1.24 1997/08/24 20:17:05 smp Exp smp $
  *
  */
 
@@ -85,6 +85,7 @@ u_int	io_apic_read		__P((int, int));
 void	io_apic_write		__P((int, int, u_int));
 
 /* global data in mp_machdep.c */
+extern int			bsp_apic_ready;
 extern int			mp_ncpus;
 extern int			mp_naps;
 extern int			mp_nbusses;