Implement preemptive scheduling of hardware interrupt threads.

- If possible, context switch to the thread directly in sched_ithd(),
  rather than triggering a delayed ast reschedule.

- Disable interrupts while restoring fpu state in the trap handler,
  in order to ensure that we are not preempted in the middle, which
  could cause migration to another cpu.

Reviewed by:	peter
Tested by:	peter (alpha)

7 files changed, 54 insertions, 12 deletions

diff --git a/sys/alpha/alpha/interrupt.c b/sys/alpha/alpha/interrupt.c
index 8e82b7f..852358b 100644
--- a/sys/alpha/alpha/interrupt.c
+++ b/sys/alpha/alpha/interrupt.c
@@ -553,6 +553,12 @@ alpha_dispatch_intr(void *frame, unsigned long vector)
 	 * is higher priority than their current thread, it gets run now.
 	 */
 	ithd->it_need = 1;
+	if (i->disable) {
+		CTR1(KTR_INTR,
+		    "alpha_dispatch_intr: disabling vector 0x%x", i->vector);
+		i->disable(i->vector);
+	}
+	enable_intr();
 	mtx_enter(&sched_lock, MTX_SPIN);
 	if (ithd->it_proc->p_stat == SWAIT) {
 		/* not on the run queue and not running */
@@ -562,19 +568,17 @@ alpha_dispatch_intr(void *frame, unsigned long vector)
 		alpha_mb();	/* XXX - ??? */
 		ithd->it_proc->p_stat = SRUN;
 		setrunqueue(ithd->it_proc);
-		aston();
+		if (!cold) {
+			if (curproc != PCPU_GET(idleproc))
+				setrunqueue(curproc);
+			mi_switch();
+		}
 	} else {
 		CTR3(KTR_INTR, "alpha_dispatch_intr: %d: it_need %d, state %d",
 		    ithd->it_proc->p_pid, ithd->it_need, ithd->it_proc->p_stat);
-	}
-	if (i->disable) {
-		CTR1(KTR_INTR,
-		    "alpha_dispatch_intr: disabling vector 0x%x", i->vector);
-		i->disable(i->vector);
+		need_resched();
 	}
 	mtx_exit(&sched_lock, MTX_SPIN);
-
-	need_resched();
 }
  
 void
diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c
index bda7bc1..ae225a8 100644
--- a/sys/alpha/alpha/machdep.c
+++ b/sys/alpha/alpha/machdep.c
@@ -2120,6 +2120,10 @@ alpha_fpstate_check(struct proc *p)
 void
 alpha_fpstate_save(struct proc *p, int write)
 {
+	int s;
+
+	s = save_intr();
+	disable_intr();
 	if (p == PCPU_GET(fpcurproc)) {
 		/*
 		 * If curproc != fpcurproc, then we need to enable FEN 
@@ -2154,6 +2158,7 @@ alpha_fpstate_save(struct proc *p, int write)
 				alpha_pal_wrfen(0);
 		}
 	}
+	restore_intr(s);
 }
 
 /*
@@ -2164,6 +2169,10 @@ alpha_fpstate_save(struct proc *p, int write)
 void
 alpha_fpstate_drop(struct proc *p)
 {
+	int s;
+
+	s = save_intr();
+	disable_intr();
 	if (p == PCPU_GET(fpcurproc)) {
 		if (p == curproc) {
 			/*
@@ -2179,6 +2188,7 @@ alpha_fpstate_drop(struct proc *p)
 		}
 		PCPU_SET(fpcurproc, NULL);
 	}
+	restore_intr(s);
 }
 
 /*
@@ -2188,9 +2198,13 @@ alpha_fpstate_drop(struct proc *p)
 void
 alpha_fpstate_switch(struct proc *p)
 {
+	int s;
+
 	/*
 	 * Enable FEN so that we can access the fp registers.
 	 */
+	s = save_intr();
+	disable_intr();
 	alpha_pal_wrfen(1);
 	if (PCPU_GET(fpcurproc)) {
 		/*
@@ -2217,6 +2231,7 @@ alpha_fpstate_switch(struct proc *p)
 	}
 
 	p->p_md.md_flags |= MDP_FPUSED;
+	restore_intr(s);
 }
 
 /*
diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c
index 7c6c4f0..a729e0f 100644
--- a/sys/amd64/amd64/fpu.c
+++ b/sys/amd64/amd64/fpu.c
@@ -796,6 +796,8 @@ npx_intr(dummy)
 int
 npxdna()
 {
+	int s;
+
 	if (!npx_exists)
 		return (0);
 	if (PCPU_GET(npxproc) != NULL) {
@@ -803,6 +805,8 @@ npxdna()
 		       PCPU_GET(npxproc), curproc);
 		panic("npxdna");
 	}
+	s = save_intr();
+	disable_intr();
 	stop_emulating();
 	/*
 	 * Record new context early in case frstor causes an IRQ13.
@@ -822,6 +826,7 @@ npxdna()
 	 * first FPU instruction after a context switch.
 	 */
 	frstor(&PCPU_GET(curpcb)->pcb_savefpu);
+	restore_intr(s);
 
 	return (1);
 }
diff --git a/sys/amd64/isa/ithread.c b/sys/amd64/isa/ithread.c
index 6a59db1..5f64861 100644
--- a/sys/amd64/isa/ithread.c
+++ b/sys/amd64/isa/ithread.c
@@ -121,16 +121,20 @@ sched_ithd(void *cookie)
 /*		membar_lock(); */
 		ir->it_proc->p_stat = SRUN;
 		setrunqueue(ir->it_proc);
-		aston();
+		if (!cold) {
+			if (curproc != PCPU_GET(idleproc))
+				setrunqueue(curproc);
+			mi_switch();
+		}
 	}
 	else {
 		CTR3(KTR_INTR, "sched_ithd %d: it_need %d, state %d",
 			ir->it_proc->p_pid,
 		        ir->it_need,
 		        ir->it_proc->p_stat );
+		need_resched();
 	}
 	mtx_exit(&sched_lock, MTX_SPIN);
-	need_resched();
 }
 
 /*
diff --git a/sys/amd64/isa/npx.c b/sys/amd64/isa/npx.c
index 7c6c4f0..a729e0f 100644
--- a/sys/amd64/isa/npx.c
+++ b/sys/amd64/isa/npx.c
@@ -796,6 +796,8 @@ npx_intr(dummy)
 int
 npxdna()
 {
+	int s;
+
 	if (!npx_exists)
 		return (0);
 	if (PCPU_GET(npxproc) != NULL) {
@@ -803,6 +805,8 @@ npxdna()
 		       PCPU_GET(npxproc), curproc);
 		panic("npxdna");
 	}
+	s = save_intr();
+	disable_intr();
 	stop_emulating();
 	/*
 	 * Record new context early in case frstor causes an IRQ13.
@@ -822,6 +826,7 @@ npxdna()
 	 * first FPU instruction after a context switch.
 	 */
 	frstor(&PCPU_GET(curpcb)->pcb_savefpu);
+	restore_intr(s);
 
 	return (1);
 }
diff --git a/sys/i386/isa/ithread.c b/sys/i386/isa/ithread.c
index 6a59db1..5f64861 100644
--- a/sys/i386/isa/ithread.c
+++ b/sys/i386/isa/ithread.c
@@ -121,16 +121,20 @@ sched_ithd(void *cookie)
 /*		membar_lock(); */
 		ir->it_proc->p_stat = SRUN;
 		setrunqueue(ir->it_proc);
-		aston();
+		if (!cold) {
+			if (curproc != PCPU_GET(idleproc))
+				setrunqueue(curproc);
+			mi_switch();
+		}
 	}
 	else {
 		CTR3(KTR_INTR, "sched_ithd %d: it_need %d, state %d",
 			ir->it_proc->p_pid,
 		        ir->it_need,
 		        ir->it_proc->p_stat );
+		need_resched();
 	}
 	mtx_exit(&sched_lock, MTX_SPIN);
-	need_resched();
 }
 
 /*
diff --git a/sys/i386/isa/npx.c b/sys/i386/isa/npx.c
index 7c6c4f0..a729e0f 100644
--- a/sys/i386/isa/npx.c
+++ b/sys/i386/isa/npx.c
@@ -796,6 +796,8 @@ npx_intr(dummy)
 int
 npxdna()
 {
+	int s;
+
 	if (!npx_exists)
 		return (0);
 	if (PCPU_GET(npxproc) != NULL) {
@@ -803,6 +805,8 @@ npxdna()
 		       PCPU_GET(npxproc), curproc);
 		panic("npxdna");
 	}
+	s = save_intr();
+	disable_intr();
 	stop_emulating();
 	/*
 	 * Record new context early in case frstor causes an IRQ13.
@@ -822,6 +826,7 @@ npxdna()
 	 * first FPU instruction after a context switch.
 	 */
 	frstor(&PCPU_GET(curpcb)->pcb_savefpu);
+	restore_intr(s);
 
 	return (1);
 }