Compromise for critical*()/cpu_critical*() recommit. Cleanup the interrupt

disablement assumptions in kern_fork.c by adding another API call,
cpu_critical_fork_exit().  Cleanup the td_savecrit field by moving it
from MI to MD.  Temporarily move cpu_critical*() from <arch>/include/cpufunc.h
to <arch>/<arch>/critical.c (stage-2 will clean this up).

Implement interrupt deferral for i386 that allows interrupts to remain
enabled inside critical sections.  This also fixes an IPI interlock bug,
and requires uses of icu_lock to be enclosed in a true interrupt disablement.

This is the stage-1 commit.  Stage-2 will occur after stage-1 has stabilized,
and will move cpu_critical*() into its own header file(s) + other things.
This commit may break non-i386 architectures in trivial ways.  This should
be temporary.

Reviewed by:	core
Approved by:	core

5 files changed, 51 insertions, 10 deletions

diff --git a/sys/pc98/cbus/clock.c b/sys/pc98/cbus/clock.c
index b8b9e3f..b70465a 100644
--- a/sys/pc98/cbus/clock.c
+++ b/sys/pc98/cbus/clock.c
@@ -1331,6 +1331,7 @@ cpu_initclocks()
 	int apic_8254_trial;
 	void *clkdesc;
 #endif /* APIC_IO */
+	register_t crit;
 
 #ifndef PC98
 	if (statclock_disable) {
@@ -1367,9 +1368,11 @@ cpu_initclocks()
 
 	inthand_add("clk", apic_8254_intr, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, &clkdesc);
+	crit = intr_disable();
 	mtx_lock_spin(&icu_lock);
 	INTREN(1 << apic_8254_intr);
 	mtx_unlock_spin(&icu_lock);
+	intr_restore(crit);
 
 #else /* APIC_IO */
 
@@ -1380,9 +1383,11 @@ cpu_initclocks()
 	 */
 	inthand_add("clk", 0, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
+	crit = intr_disable();
 	mtx_lock_spin(&icu_lock);
 	INTREN(IRQ0);
 	mtx_unlock_spin(&icu_lock);
+	intr_restore(crit);
 
 #endif /* APIC_IO */
 
@@ -1408,6 +1413,7 @@ cpu_initclocks()
 	inthand_add("rtc", 8, (driver_intr_t *)rtcintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
 
+	crit = intr_disable();
 	mtx_lock_spin(&icu_lock);
 #ifdef APIC_IO
 	INTREN(APIC_IRQ8);
@@ -1415,6 +1421,7 @@ cpu_initclocks()
 	INTREN(IRQ8);
 #endif /* APIC_IO */
 	mtx_unlock_spin(&icu_lock);
+	intr_restore(crit);
 
 	writertc(RTC_STATUSB, rtc_statusb);
 #endif /* PC98 */
@@ -1432,9 +1439,11 @@ cpu_initclocks()
 			 * on the IO APIC.
 			 * Workaround: Limited variant of mixed mode.
 			 */
+			crit = intr_disable();
 			mtx_lock_spin(&icu_lock);
 			INTRDIS(1 << apic_8254_intr);
 			mtx_unlock_spin(&icu_lock);
+			intr_restore(crit);
 			inthand_remove(clkdesc);
 			printf("APIC_IO: Broken MP table detected: "
 			       "8254 is not connected to "
@@ -1457,9 +1466,11 @@ cpu_initclocks()
 			inthand_add("clk", apic_8254_intr,
 				    (driver_intr_t *)clkintr, NULL,
 				    INTR_TYPE_CLK | INTR_FAST, NULL);
+			crit = intr_disable();
 			mtx_lock_spin(&icu_lock);
 			INTREN(1 << apic_8254_intr);
 			mtx_unlock_spin(&icu_lock);
+			intr_restore(crit);
 		}
 		
 	}
diff --git a/sys/pc98/cbus/pcrtc.c b/sys/pc98/cbus/pcrtc.c
index b8b9e3f..b70465a 100644
--- a/sys/pc98/cbus/pcrtc.c
+++ b/sys/pc98/cbus/pcrtc.c
@@ -1331,6 +1331,7 @@ cpu_initclocks()
 	int apic_8254_trial;
 	void *clkdesc;
 #endif /* APIC_IO */
+	register_t crit;
 
 #ifndef PC98
 	if (statclock_disable) {
@@ -1367,9 +1368,11 @@ cpu_initclocks()
 
 	inthand_add("clk", apic_8254_intr, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, &clkdesc);
+	crit = intr_disable();
 	mtx_lock_spin(&icu_lock);
 	INTREN(1 << apic_8254_intr);
 	mtx_unlock_spin(&icu_lock);
+	intr_restore(crit);
 
 #else /* APIC_IO */
 
@@ -1380,9 +1383,11 @@ cpu_initclocks()
 	 */
 	inthand_add("clk", 0, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
+	crit = intr_disable();
 	mtx_lock_spin(&icu_lock);
 	INTREN(IRQ0);
 	mtx_unlock_spin(&icu_lock);
+	intr_restore(crit);
 
 #endif /* APIC_IO */
 
@@ -1408,6 +1413,7 @@ cpu_initclocks()
 	inthand_add("rtc", 8, (driver_intr_t *)rtcintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
 
+	crit = intr_disable();
 	mtx_lock_spin(&icu_lock);
 #ifdef APIC_IO
 	INTREN(APIC_IRQ8);
@@ -1415,6 +1421,7 @@ cpu_initclocks()
 	INTREN(IRQ8);
 #endif /* APIC_IO */
 	mtx_unlock_spin(&icu_lock);
+	intr_restore(crit);
 
 	writertc(RTC_STATUSB, rtc_statusb);
 #endif /* PC98 */
@@ -1432,9 +1439,11 @@ cpu_initclocks()
 			 * on the IO APIC.
 			 * Workaround: Limited variant of mixed mode.
 			 */
+			crit = intr_disable();
 			mtx_lock_spin(&icu_lock);
 			INTRDIS(1 << apic_8254_intr);
 			mtx_unlock_spin(&icu_lock);
+			intr_restore(crit);
 			inthand_remove(clkdesc);
 			printf("APIC_IO: Broken MP table detected: "
 			       "8254 is not connected to "
@@ -1457,9 +1466,11 @@ cpu_initclocks()
 			inthand_add("clk", apic_8254_intr,
 				    (driver_intr_t *)clkintr, NULL,
 				    INTR_TYPE_CLK | INTR_FAST, NULL);
+			crit = intr_disable();
 			mtx_lock_spin(&icu_lock);
 			INTREN(1 << apic_8254_intr);
 			mtx_unlock_spin(&icu_lock);
+			intr_restore(crit);
 		}
 		
 	}
diff --git a/sys/pc98/i386/machdep.c b/sys/pc98/i386/machdep.c
index 21a538f..e41899f 100644
--- a/sys/pc98/i386/machdep.c
+++ b/sys/pc98/i386/machdep.c
@@ -815,11 +815,10 @@ cpu_idle(void)
 #ifndef SMP
 	if (cpu_idle_hlt) {
 		disable_intr();
-  		if (procrunnable())
+  		if (procrunnable()) {
 			enable_intr();
-		else {
-			enable_intr();
-			__asm __volatile("hlt");
+		} else {
+			__asm __volatile("sti; hlt");
 		}
 	}
 #endif
@@ -1757,12 +1756,17 @@ init386(first)
 
 	/*
 	 * Initialize mutexes.
+	 *
+	 * icu_lock: in order to allow an interrupt to occur in a critical
+	 *	     section, to set pcpu->ipending (etc...) properly, we
+	 *	     must be able to get the icu lock, so it can't be under
+	 *	     witness.
 	 */
 	mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE);
 	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
 	mtx_init(&proc0.p_mtx, "process lock", MTX_DEF);
 	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE);
-	mtx_init(&icu_lock, "icu", MTX_SPIN);
+	mtx_init(&icu_lock, "icu", MTX_SPIN | MTX_NOWITNESS);
 	mtx_lock(&Giant);
 
 	/* make ldt memory segments */
diff --git a/sys/pc98/pc98/clock.c b/sys/pc98/pc98/clock.c
index b8b9e3f..b70465a 100644
--- a/sys/pc98/pc98/clock.c
+++ b/sys/pc98/pc98/clock.c
@@ -1331,6 +1331,7 @@ cpu_initclocks()
 	int apic_8254_trial;
 	void *clkdesc;
 #endif /* APIC_IO */
+	register_t crit;
 
 #ifndef PC98
 	if (statclock_disable) {
@@ -1367,9 +1368,11 @@ cpu_initclocks()
 
 	inthand_add("clk", apic_8254_intr, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, &clkdesc);
+	crit = intr_disable();
 	mtx_lock_spin(&icu_lock);
 	INTREN(1 << apic_8254_intr);
 	mtx_unlock_spin(&icu_lock);
+	intr_restore(crit);
 
 #else /* APIC_IO */
 
@@ -1380,9 +1383,11 @@ cpu_initclocks()
 	 */
 	inthand_add("clk", 0, (driver_intr_t *)clkintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
+	crit = intr_disable();
 	mtx_lock_spin(&icu_lock);
 	INTREN(IRQ0);
 	mtx_unlock_spin(&icu_lock);
+	intr_restore(crit);
 
 #endif /* APIC_IO */
 
@@ -1408,6 +1413,7 @@ cpu_initclocks()
 	inthand_add("rtc", 8, (driver_intr_t *)rtcintr, NULL,
 	    INTR_TYPE_CLK | INTR_FAST, NULL);
 
+	crit = intr_disable();
 	mtx_lock_spin(&icu_lock);
 #ifdef APIC_IO
 	INTREN(APIC_IRQ8);
@@ -1415,6 +1421,7 @@ cpu_initclocks()
 	INTREN(IRQ8);
 #endif /* APIC_IO */
 	mtx_unlock_spin(&icu_lock);
+	intr_restore(crit);
 
 	writertc(RTC_STATUSB, rtc_statusb);
 #endif /* PC98 */
@@ -1432,9 +1439,11 @@ cpu_initclocks()
 			 * on the IO APIC.
 			 * Workaround: Limited variant of mixed mode.
 			 */
+			crit = intr_disable();
 			mtx_lock_spin(&icu_lock);
 			INTRDIS(1 << apic_8254_intr);
 			mtx_unlock_spin(&icu_lock);
+			intr_restore(crit);
 			inthand_remove(clkdesc);
 			printf("APIC_IO: Broken MP table detected: "
 			       "8254 is not connected to "
@@ -1457,9 +1466,11 @@ cpu_initclocks()
 			inthand_add("clk", apic_8254_intr,
 				    (driver_intr_t *)clkintr, NULL,
 				    INTR_TYPE_CLK | INTR_FAST, NULL);
+			crit = intr_disable();
 			mtx_lock_spin(&icu_lock);
 			INTREN(1 << apic_8254_intr);
 			mtx_unlock_spin(&icu_lock);
+			intr_restore(crit);
 		}
 		
 	}
diff --git a/sys/pc98/pc98/machdep.c b/sys/pc98/pc98/machdep.c
index 21a538f..e41899f 100644
--- a/sys/pc98/pc98/machdep.c
+++ b/sys/pc98/pc98/machdep.c
@@ -815,11 +815,10 @@ cpu_idle(void)
 #ifndef SMP
 	if (cpu_idle_hlt) {
 		disable_intr();
-  		if (procrunnable())
+  		if (procrunnable()) {
 			enable_intr();
-		else {
-			enable_intr();
-			__asm __volatile("hlt");
+		} else {
+			__asm __volatile("sti; hlt");
 		}
 	}
 #endif
@@ -1757,12 +1756,17 @@ init386(first)
 
 	/*
 	 * Initialize mutexes.
+	 *
+	 * icu_lock: in order to allow an interrupt to occur in a critical
+	 *	     section, to set pcpu->ipending (etc...) properly, we
+	 *	     must be able to get the icu lock, so it can't be under
+	 *	     witness.
 	 */
 	mtx_init(&Giant, "Giant", MTX_DEF | MTX_RECURSE);
 	mtx_init(&sched_lock, "sched lock", MTX_SPIN | MTX_RECURSE);
 	mtx_init(&proc0.p_mtx, "process lock", MTX_DEF);
 	mtx_init(&clock_lock, "clk", MTX_SPIN | MTX_RECURSE);
-	mtx_init(&icu_lock, "icu", MTX_SPIN);
+	mtx_init(&icu_lock, "icu", MTX_SPIN | MTX_NOWITNESS);
 	mtx_lock(&Giant);
 
 	/* make ldt memory segments */