Make intr_nesting_level per-process, rather than per-cpu. Setup

interrupt threads to run with it always >= 1, so that malloc can
detect M_WAITOK from "interrupt" context.  This is also necessary
in order to context switch from sched_ithd() directly.

Reviewed By:	peter

17 files changed, 49 insertions, 34 deletions

diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s
index 4ebf35d..a2cdb47 100644
--- a/sys/i386/i386/apic_vector.s
+++ b/sys/i386/i386/apic_vector.s
@@ -48,7 +48,8 @@ IDTVEC(vec_name) ;							\
 	movl	$KPSEL,%eax ;						\
 	mov	%ax,%fs ;						\
 	FAKE_MCOUNT(13*4(%esp)) ;					\
-	incb	PCPU(INTR_NESTING_LEVEL) ;				\
+	movl	PCPU(CURPROC),%ebx ;					\
+	incl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 	pushl	_intr_unit + (irq_num) * 4 ;				\
 	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
 	addl	$4, %esp ;						\
@@ -58,6 +59,7 @@ IDTVEC(vec_name) ;							\
 	movl	_intr_countp + (irq_num) * 4, %eax ;			\
 	lock ; 								\
 	incl	(%eax) ;						\
+	decl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 	MEXITCOUNT ;							\
 	jmp	_doreti
 
@@ -150,7 +152,8 @@ IDTVEC(vec_name) ;							\
 	MASK_LEVEL_IRQ(irq_num) ;					\
 	EOI_IRQ(irq_num) ;						\
 0: ;									\
-	incb	PCPU(INTR_NESTING_LEVEL) ;				\
+	movl	PCPU(CURPROC),%ebx ;					\
+	incl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 ;	 								\
   /* entry point used by doreti_unpend for HWIs. */			\
 __CONCAT(Xresume,irq_num): ;						\
@@ -160,6 +163,7 @@ __CONCAT(Xresume,irq_num): ;						\
 	call	_sched_ithd ;						\
 	addl	$4, %esp ;		/* discard the parameter */	\
 ;									\
+	decl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 	MEXITCOUNT ;							\
 	jmp	_doreti
 
@@ -301,7 +305,8 @@ _Xcpuast:
 	FAKE_MCOUNT(13*4(%esp))
 
 	orl	$AST_PENDING, PCPU(ASTPENDING)	/* XXX */
-	incb	PCPU(INTR_NESTING_LEVEL)
+	movl	PCPU(CURPROC),%ebx
+	incl	P_INTR_NESTING_LEVEL(%ebx)
 	sti
 	
 	movl	PCPU(CPUID), %eax
@@ -316,6 +321,7 @@ _Xcpuast:
 2:		
 	lock
 	incl	CNAME(cpuast_cnt)
+	decl	P_INTR_NESTING_LEVEL(%ebx)
 	MEXITCOUNT
 	jmp	_doreti
 1:
@@ -323,6 +329,7 @@ _Xcpuast:
 	POP_FRAME
 	iret			
 
+#if 0
 
 /*
  *	 Executed by a CPU when it receives an XFORWARD_IRQ IPI.
@@ -360,7 +367,6 @@ _Xforward_irq:
 	POP_FRAME
 	iret
 
-#if 0
 /*
  * 
  */
diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s
index 603b4be..aafbf0b 100644
--- a/sys/i386/i386/exception.s
+++ b/sys/i386/i386/exception.s
@@ -177,7 +177,6 @@ IDTVEC(fpu)
 	call	_npx_intr
 
 	addl	$4,%esp
-	incb	PCPU(INTR_NESTING_LEVEL)
 	MEXITCOUNT
 	jmp	_doreti
 #else	/* DEV_NPX */
@@ -216,7 +215,6 @@ calltrap:
 	/*
 	 * Return via _doreti to handle ASTs.
 	 */
-	incb	PCPU(INTR_NESTING_LEVEL)
 	MEXITCOUNT
 	jmp	_doreti
 
@@ -257,7 +255,6 @@ IDTVEC(syscall)
 	cli				/* atomic astpending access */
 	cmpl    $0,PCPU(ASTPENDING)	/* AST pending? */
 	je	doreti_syscall_ret	/* no, get out of here */
-	movb	$1,PCPU(INTR_NESTING_LEVEL)
 	jmp	_doreti
 
 /*
@@ -289,7 +286,6 @@ IDTVEC(int0x80_syscall)
 	cli				/* atomic astpending access */
 	cmpl    $0,PCPU(ASTPENDING)	/* AST pending? */
 	je	doreti_syscall_ret	/* no, get out of here */
-	movb	$1,PCPU(INTR_NESTING_LEVEL)
 	jmp	_doreti
 
 ENTRY(fork_trampoline)
@@ -323,7 +319,6 @@ ENTRY(fork_trampoline)
 	/*
 	 * Return via _doreti to handle ASTs.
 	 */
-	movb	$1,PCPU(INTR_NESTING_LEVEL)
 	MEXITCOUNT
 	jmp	_doreti
 
diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c
index 8b9ca33..c307665 100644
--- a/sys/i386/i386/genassym.c
+++ b/sys/i386/i386/genassym.c
@@ -80,6 +80,7 @@ ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace));
 ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap));
 ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active));
 ASSYM(P_ADDR, offsetof(struct proc, p_addr));
+ASSYM(P_INTR_NESTING_LEVEL, offsetof(struct proc, p_intr_nesting_level));
 ASSYM(P_STAT, offsetof(struct proc, p_stat));
 ASSYM(P_WCHAN, offsetof(struct proc, p_wchan));
 
@@ -182,7 +183,6 @@ ASSYM(GD_TSS_GDT, offsetof(struct globaldata, gd_tss_gdt));
 ASSYM(GD_ASTPENDING, offsetof(struct globaldata, gd_astpending));
 ASSYM(AST_PENDING, AST_PENDING);
 ASSYM(AST_RESCHED, AST_RESCHED);
-ASSYM(GD_INTR_NESTING_LEVEL, offsetof(struct globaldata, gd_intr_nesting_level));
 
 #ifdef USER_LDT
 ASSYM(GD_CURRENTLDT, offsetof(struct globaldata, gd_currentldt));
diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c
index bf560cd..966d3f8 100644
--- a/sys/i386/i386/mp_machdep.c
+++ b/sys/i386/i386/mp_machdep.c
@@ -581,10 +581,12 @@ mp_enable(u_int boot_addr)
 	/* install an inter-CPU IPI for forcing an additional software trap */
 	setidt(XCPUAST_OFFSET, Xcpuast,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-	
+
+#if 0	
 	/* install an inter-CPU IPI for interrupt forwarding */
 	setidt(XFORWARD_IRQ_OFFSET, Xforward_irq,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+#endif
 
 	/* install an inter-CPU IPI for CPU stop/restart */
 	setidt(XCPUSTOP_OFFSET, Xcpustop,
diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c
index bf560cd..966d3f8 100644
--- a/sys/i386/i386/mptable.c
+++ b/sys/i386/i386/mptable.c
@@ -581,10 +581,12 @@ mp_enable(u_int boot_addr)
 	/* install an inter-CPU IPI for forcing an additional software trap */
 	setidt(XCPUAST_OFFSET, Xcpuast,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-	
+
+#if 0	
 	/* install an inter-CPU IPI for interrupt forwarding */
 	setidt(XFORWARD_IRQ_OFFSET, Xforward_irq,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+#endif
 
 	/* install an inter-CPU IPI for CPU stop/restart */
 	setidt(XCPUSTOP_OFFSET, Xcpustop,
diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c
index 6ee3e6d..0170dbe 100644
--- a/sys/i386/i386/trap.c
+++ b/sys/i386/i386/trap.c
@@ -480,7 +480,7 @@ restart:
 			if (in_vm86call)
 				break;
 
-			if (PCPU_GET(intr_nesting_level) != 0)
+			if (p->p_intr_nesting_level != 0)
 				break;
 
 			/*
@@ -687,7 +687,7 @@ trap_pfault(frame, usermode, eva)
 
 		if (p == NULL ||
 		    (!usermode && va < VM_MAXUSER_ADDRESS &&
-		     (PCPU_GET(intr_nesting_level) != 0 ||
+		     (p->p_intr_nesting_level != 0 ||
 		      PCPU_GET(curpcb) == NULL ||
 		      PCPU_GET(curpcb)->pcb_onfault == NULL))) {
 			trap_fatal(frame, eva);
@@ -751,7 +751,7 @@ trap_pfault(frame, usermode, eva)
 		return (0);
 nogo:
 	if (!usermode) {
-		if (PCPU_GET(intr_nesting_level) == 0 &&
+		if (p->p_intr_nesting_level == 0 &&
 		    PCPU_GET(curpcb) != NULL &&
 		    PCPU_GET(curpcb)->pcb_onfault != NULL) {
 			frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault;
@@ -858,7 +858,7 @@ trap_pfault(frame, usermode, eva)
 		return (0);
 nogo:
 	if (!usermode) {
-		if (PCPU_GET(intr_nesting_level) == 0 &&
+		if (p->p_intr_nesting_level == 0 &&
 		    PCPU_GET(curpcb) != NULL &&
 		    PCPU_GET(curpcb)->pcb_onfault != NULL) {
 			frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault;
diff --git a/sys/i386/i386/vm86bios.s b/sys/i386/i386/vm86bios.s
index a71dd11..5f3013c 100644
--- a/sys/i386/i386/vm86bios.s
+++ b/sys/i386/i386/vm86bios.s
@@ -131,7 +131,6 @@ ENTRY(vm86_bioscall)
 	/*
 	 * Return via _doreti
 	 */
-	incb	PCPU(INTR_NESTING_LEVEL)
 	MEXITCOUNT
 	jmp	_doreti
 
diff --git a/sys/i386/include/cpu.h b/sys/i386/include/cpu.h
index 6c4162b..87c47c5 100644
--- a/sys/i386/include/cpu.h
+++ b/sys/i386/include/cpu.h
@@ -62,7 +62,7 @@
 #define	CLKF_USERMODE(framep) \
 	((ISPL((framep)->cf_cs) == SEL_UPL) || (framep->cf_eflags & PSL_VM))
 
-#define CLKF_INTR(framep)	(PCPU_GET(intr_nesting_level) >= 2)
+#define CLKF_INTR(framep)	(curproc->p_intr_nesting_level >= 2)
 #define	CLKF_PC(framep)		((framep)->cf_eip)
 
 /*
diff --git a/sys/i386/include/globaldata.h b/sys/i386/include/globaldata.h
index 149026e..7a7437a6 100644
--- a/sys/i386/include/globaldata.h
+++ b/sys/i386/include/globaldata.h
@@ -58,8 +58,6 @@ struct globaldata {
 	struct timeval	gd_switchtime;
 	struct i386tss	gd_common_tss;
 	int		gd_switchticks;
-	u_char		gd_intr_nesting_level;
-	u_char		gd_pad0[3];
 	struct segment_descriptor gd_common_tssd;
 	struct segment_descriptor *gd_tss_gdt;
 	int		gd_currentldt;		/* only used for USER_LDT */
diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h
index bf560cd..966d3f8 100644
--- a/sys/i386/include/mptable.h
+++ b/sys/i386/include/mptable.h
@@ -581,10 +581,12 @@ mp_enable(u_int boot_addr)
 	/* install an inter-CPU IPI for forcing an additional software trap */
 	setidt(XCPUAST_OFFSET, Xcpuast,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
-	
+
+#if 0	
 	/* install an inter-CPU IPI for interrupt forwarding */
 	setidt(XFORWARD_IRQ_OFFSET, Xforward_irq,
 	       SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL));
+#endif
 
 	/* install an inter-CPU IPI for CPU stop/restart */
 	setidt(XCPUSTOP_OFFSET, Xcpustop,
diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h
index 149026e..7a7437a6 100644
--- a/sys/i386/include/pcpu.h
+++ b/sys/i386/include/pcpu.h
@@ -58,8 +58,6 @@ struct globaldata {
 	struct timeval	gd_switchtime;
 	struct i386tss	gd_common_tss;
 	int		gd_switchticks;
-	u_char		gd_intr_nesting_level;
-	u_char		gd_pad0[3];
 	struct segment_descriptor gd_common_tssd;
 	struct segment_descriptor *gd_tss_gdt;
 	int		gd_currentldt;		/* only used for USER_LDT */
diff --git a/sys/i386/isa/apic_vector.s b/sys/i386/isa/apic_vector.s
index 4ebf35d..a2cdb47 100644
--- a/sys/i386/isa/apic_vector.s
+++ b/sys/i386/isa/apic_vector.s
@@ -48,7 +48,8 @@ IDTVEC(vec_name) ;							\
 	movl	$KPSEL,%eax ;						\
 	mov	%ax,%fs ;						\
 	FAKE_MCOUNT(13*4(%esp)) ;					\
-	incb	PCPU(INTR_NESTING_LEVEL) ;				\
+	movl	PCPU(CURPROC),%ebx ;					\
+	incl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 	pushl	_intr_unit + (irq_num) * 4 ;				\
 	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
 	addl	$4, %esp ;						\
@@ -58,6 +59,7 @@ IDTVEC(vec_name) ;							\
 	movl	_intr_countp + (irq_num) * 4, %eax ;			\
 	lock ; 								\
 	incl	(%eax) ;						\
+	decl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 	MEXITCOUNT ;							\
 	jmp	_doreti
 
@@ -150,7 +152,8 @@ IDTVEC(vec_name) ;							\
 	MASK_LEVEL_IRQ(irq_num) ;					\
 	EOI_IRQ(irq_num) ;						\
 0: ;									\
-	incb	PCPU(INTR_NESTING_LEVEL) ;				\
+	movl	PCPU(CURPROC),%ebx ;					\
+	incl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 ;	 								\
   /* entry point used by doreti_unpend for HWIs. */			\
 __CONCAT(Xresume,irq_num): ;						\
@@ -160,6 +163,7 @@ __CONCAT(Xresume,irq_num): ;						\
 	call	_sched_ithd ;						\
 	addl	$4, %esp ;		/* discard the parameter */	\
 ;									\
+	decl	P_INTR_NESTING_LEVEL(%ebx) ;				\
 	MEXITCOUNT ;							\
 	jmp	_doreti
 
@@ -301,7 +305,8 @@ _Xcpuast:
 	FAKE_MCOUNT(13*4(%esp))
 
 	orl	$AST_PENDING, PCPU(ASTPENDING)	/* XXX */
-	incb	PCPU(INTR_NESTING_LEVEL)
+	movl	PCPU(CURPROC),%ebx
+	incl	P_INTR_NESTING_LEVEL(%ebx)
 	sti
 	
 	movl	PCPU(CPUID), %eax
@@ -316,6 +321,7 @@ _Xcpuast:
 2:		
 	lock
 	incl	CNAME(cpuast_cnt)
+	decl	P_INTR_NESTING_LEVEL(%ebx)
 	MEXITCOUNT
 	jmp	_doreti
 1:
@@ -323,6 +329,7 @@ _Xcpuast:
 	POP_FRAME
 	iret			
 
+#if 0
 
 /*
  *	 Executed by a CPU when it receives an XFORWARD_IRQ IPI.
@@ -360,7 +367,6 @@ _Xforward_irq:
 	POP_FRAME
 	iret
 
-#if 0
 /*
  * 
  */
diff --git a/sys/i386/isa/atpic_vector.s b/sys/i386/isa/atpic_vector.s
index 635d2ad..0a3456f 100644
--- a/sys/i386/isa/atpic_vector.s
+++ b/sys/i386/isa/atpic_vector.s
@@ -61,7 +61,8 @@ IDTVEC(vec_name) ; \
 	mov	$KPSEL,%ax ; \
 	mov	%ax,%fs ; \
 	FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \
-	incb	PCPU(INTR_NESTING_LEVEL) ; \
+	movl	PCPU(CURPROC),%ebx ; \
+	incl	P_INTR_NESTING_LEVEL(%ebx) ; \
 	pushl	_intr_unit + (irq_num) * 4 ; \
 	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
 	enable_icus ;		/* (re)enable ASAP (helps edge trigger?) */ \
@@ -69,6 +70,7 @@ IDTVEC(vec_name) ; \
 	incl	_cnt+V_INTR ;	/* book-keeping can wait */ \
 	movl	_intr_countp + (irq_num) * 4,%eax ; \
 	incl	(%eax) ; \
+	decl	P_INTR_NESTING_LEVEL(%ebx) ; \
 	MEXITCOUNT ; \
 	jmp	_doreti
 
@@ -102,13 +104,15 @@ IDTVEC(vec_name) ; \
 	movb	%al,_imen + IRQ_BYTE(irq_num) ; \
 	outb	%al,$icu+ICU_IMR_OFFSET ; \
 	enable_icus ; \
-	incb	PCPU(INTR_NESTING_LEVEL) ; \
+	movl	PCPU(CURPROC),%ebx ; \
+	incl	P_INTR_NESTING_LEVEL(%ebx) ; \
 __CONCAT(Xresume,irq_num): ; \
 	FAKE_MCOUNT(13*4(%esp)) ;	/* XXX late to avoid double count */ \
 	pushl	$irq_num; 	/* pass the IRQ */ \
 	sti ; \
 	call	_sched_ithd ; \
 	addl	$4, %esp ;	/* discard the parameter */ \
+	decl	P_INTR_NESTING_LEVEL(%ebx) ; \
 	MEXITCOUNT ; \
 	/* We could usually avoid the following jmp by inlining some of */ \
 	/* _doreti, but it's probably better to use less cache. */ \
diff --git a/sys/i386/isa/icu_vector.s b/sys/i386/isa/icu_vector.s
index 635d2ad..0a3456f 100644
--- a/sys/i386/isa/icu_vector.s
+++ b/sys/i386/isa/icu_vector.s
@@ -61,7 +61,8 @@ IDTVEC(vec_name) ; \
 	mov	$KPSEL,%ax ; \
 	mov	%ax,%fs ; \
 	FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \
-	incb	PCPU(INTR_NESTING_LEVEL) ; \
+	movl	PCPU(CURPROC),%ebx ; \
+	incl	P_INTR_NESTING_LEVEL(%ebx) ; \
 	pushl	_intr_unit + (irq_num) * 4 ; \
 	call	*_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \
 	enable_icus ;		/* (re)enable ASAP (helps edge trigger?) */ \
@@ -69,6 +70,7 @@ IDTVEC(vec_name) ; \
 	incl	_cnt+V_INTR ;	/* book-keeping can wait */ \
 	movl	_intr_countp + (irq_num) * 4,%eax ; \
 	incl	(%eax) ; \
+	decl	P_INTR_NESTING_LEVEL(%ebx) ; \
 	MEXITCOUNT ; \
 	jmp	_doreti
 
@@ -102,13 +104,15 @@ IDTVEC(vec_name) ; \
 	movb	%al,_imen + IRQ_BYTE(irq_num) ; \
 	outb	%al,$icu+ICU_IMR_OFFSET ; \
 	enable_icus ; \
-	incb	PCPU(INTR_NESTING_LEVEL) ; \
+	movl	PCPU(CURPROC),%ebx ; \
+	incl	P_INTR_NESTING_LEVEL(%ebx) ; \
 __CONCAT(Xresume,irq_num): ; \
 	FAKE_MCOUNT(13*4(%esp)) ;	/* XXX late to avoid double count */ \
 	pushl	$irq_num; 	/* pass the IRQ */ \
 	sti ; \
 	call	_sched_ithd ; \
 	addl	$4, %esp ;	/* discard the parameter */ \
+	decl	P_INTR_NESTING_LEVEL(%ebx) ; \
 	MEXITCOUNT ; \
 	/* We could usually avoid the following jmp by inlining some of */ \
 	/* _doreti, but it's probably better to use less cache. */ \
diff --git a/sys/i386/isa/intr_machdep.c b/sys/i386/isa/intr_machdep.c
index 00f5f23..b9328ae 100644
--- a/sys/i386/isa/intr_machdep.c
+++ b/sys/i386/isa/intr_machdep.c
@@ -577,6 +577,7 @@ inthand_add(const char *name, int irq, driver_intr_t handler, void *arg,
 				if (errcode)
 					panic("inthand_add: Can't create "
 					      "interrupt thread");
+				p->p_intr_nesting_level = 1;
 				p->p_rtprio.type = RTP_PRIO_ITHREAD;
 				p->p_stat = SWAIT; /* we're idle */
 
diff --git a/sys/i386/isa/ipl.s b/sys/i386/isa/ipl.s
index ca874ea..7c41589 100644
--- a/sys/i386/isa/ipl.s
+++ b/sys/i386/isa/ipl.s
@@ -57,8 +57,6 @@
 _doreti:
 	FAKE_MCOUNT(_bintr)		/* init "from" _bintr -> _doreti */
 doreti_next:
-	decb	PCPU(INTR_NESTING_LEVEL)
-
 	/* Check for ASTs that can be handled now. */
 	testl	$AST_PENDING,PCPU(ASTPENDING)
 	je	doreti_exit		/* no AST, exit */
@@ -128,7 +126,6 @@ doreti_ast:
 	sti
 	movl	$T_ASTFLT,TF_TRAPNO(%esp)
 	call	_ast
-	movb	$1,PCPU(INTR_NESTING_LEVEL) /* for doreti_next to decrement */
 	jmp	doreti_next
 
 #ifdef APIC_IO
diff --git a/sys/i386/isa/nmi.c b/sys/i386/isa/nmi.c
index 00f5f23..b9328ae 100644
--- a/sys/i386/isa/nmi.c
+++ b/sys/i386/isa/nmi.c
@@ -577,6 +577,7 @@ inthand_add(const char *name, int irq, driver_intr_t handler, void *arg,
 				if (errcode)
 					panic("inthand_add: Can't create "
 					      "interrupt thread");
+				p->p_intr_nesting_level = 1;
 				p->p_rtprio.type = RTP_PRIO_ITHREAD;
 				p->p_stat = SWAIT; /* we're idle */