From 937122ae6dc02a639d13dd06132201548bd7364f Mon Sep 17 00:00:00 2001 From: jake Date: Sun, 21 Jan 2001 19:25:07 +0000 Subject: Make intr_nesting_level per-process, rather than per-cpu. Setup interrupt threads to run with it always >= 1, so that malloc can detect M_WAITOK from "interrupt" context. This is also necessary in order to context switch from sched_ithd() directly. Reviewed By: peter --- sys/alpha/alpha/interrupt.c | 12 ++++++------ sys/alpha/alpha/mp_machdep.c | 2 +- sys/alpha/include/cpu.h | 4 +--- sys/alpha/include/globaldata.h | 1 - sys/alpha/include/pcpu.h | 1 - sys/amd64/amd64/apic_vector.S | 14 ++++++++++---- sys/amd64/amd64/exception.S | 5 ----- sys/amd64/amd64/exception.s | 5 ----- sys/amd64/amd64/genassym.c | 2 +- sys/amd64/amd64/mp_machdep.c | 4 +++- sys/amd64/amd64/mptable.c | 4 +++- sys/amd64/amd64/trap.c | 8 ++++---- sys/amd64/include/cpu.h | 2 +- sys/amd64/include/mptable.h | 4 +++- sys/amd64/include/pcpu.h | 2 -- sys/amd64/isa/atpic_vector.S | 8 ++++++-- sys/amd64/isa/icu_vector.S | 8 ++++++-- sys/amd64/isa/icu_vector.s | 8 ++++++-- sys/amd64/isa/intr_machdep.c | 1 + sys/amd64/isa/nmi.c | 1 + sys/dev/usb/ohci.c | 2 +- sys/dev/usb/uhci.c | 2 +- sys/dev/usb/usbdi.c | 2 +- sys/dev/vinum/vinumhdr.h | 2 +- sys/dev/vinum/vinummemory.c | 2 +- sys/i386/i386/apic_vector.s | 14 ++++++++++---- sys/i386/i386/exception.s | 5 ----- sys/i386/i386/genassym.c | 2 +- sys/i386/i386/mp_machdep.c | 4 +++- sys/i386/i386/mptable.c | 4 +++- sys/i386/i386/trap.c | 8 ++++---- sys/i386/i386/vm86bios.s | 1 - sys/i386/include/cpu.h | 2 +- sys/i386/include/globaldata.h | 2 -- sys/i386/include/mptable.h | 4 +++- sys/i386/include/pcpu.h | 2 -- sys/i386/isa/apic_vector.s | 14 ++++++++++---- sys/i386/isa/atpic_vector.s | 8 ++++++-- sys/i386/isa/icu_vector.s | 8 ++++++-- sys/i386/isa/intr_machdep.c | 1 + sys/i386/isa/ipl.s | 3 --- sys/i386/isa/nmi.c | 1 + sys/ia64/ia64/interrupt.c | 6 ++++-- sys/ia64/ia64/mp_machdep.c | 2 +- sys/ia64/include/cpu.h | 2 +- sys/ia64/include/globaldata.h | 1 - sys/ia64/include/pcpu.h | 1 - sys/kern/kern_fork.c | 1 + sys/kern/kern_malloc.c | 3 ++- sys/kern/subr_smp.c | 4 +++- sys/kern/subr_trap.c | 8 ++++---- sys/powerpc/include/globaldata.h | 1 - sys/powerpc/include/pcpu.h | 1 - sys/sys/proc.h | 1 + 54 files changed, 124 insertions(+), 96 deletions(-) diff --git a/sys/alpha/alpha/interrupt.c b/sys/alpha/alpha/interrupt.c index 6311937..c11de3a 100644 --- a/sys/alpha/alpha/interrupt.c +++ b/sys/alpha/alpha/interrupt.c @@ -89,15 +89,15 @@ interrupt(a0, a1, a2, framep) unsigned long a0, a1, a2; struct trapframe *framep; { + struct proc *p; + /* * Find our per-cpu globals. */ globalp = (struct globaldata *) alpha_pal_rdval(); - - atomic_add_int(PCPU_PTR(intr_nesting_level), 1); + p = curproc; + atomic_add_int(&p->p_intr_nesting_level, 1); { - struct proc *p = curproc; - if (!p) p = &proc0; if ((caddr_t) framep < (caddr_t) p->p_addr + 1024) { panic("possible stack overflow\n"); } @@ -116,7 +116,7 @@ interrupt(a0, a1, a2, framep) CTR0(KTR_INTR, "clock interrupt"); if (PCPU_GET(cpuno) != hwrpb->rpb_primary_cpu_id) { CTR0(KTR_INTR, "ignoring clock on secondary"); - atomic_subtract_int(PCPU_PTR(intr_nesting_level), 1); + atomic_subtract_int(&p->p_intr_nesting_level, 1); return; } @@ -152,7 +152,7 @@ interrupt(a0, a1, a2, framep) a0, a1, a2); /* NOTREACHED */ } - atomic_subtract_int(PCPU_PTR(intr_nesting_level), 1); + atomic_subtract_int(&p->p_intr_nesting_level, 1); } void diff --git a/sys/alpha/alpha/mp_machdep.c b/sys/alpha/alpha/mp_machdep.c index 2ece633..e2a2be5 100644 --- a/sys/alpha/alpha/mp_machdep.c +++ b/sys/alpha/alpha/mp_machdep.c @@ -1053,7 +1053,7 @@ smp_handle_ipi(struct trapframe *frame) CTR0(KTR_SMP, "IPI_CHECKSTATE"); if (frame->tf_regs[FRAME_PS] & ALPHA_PSL_USERMODE) checkstate_cpustate[cpuno] = CHECKSTATE_USER; - else if (PCPU_GET(intr_nesting_level) == 1) + else if (curproc->p_intr_nesting_level == 1) checkstate_cpustate[cpuno] = CHECKSTATE_SYS; else checkstate_cpustate[cpuno] = CHECKSTATE_INTR; diff --git a/sys/alpha/include/cpu.h b/sys/alpha/include/cpu.h index 3da213f..0edb2fc 100644 --- a/sys/alpha/include/cpu.h +++ b/sys/alpha/include/cpu.h @@ -65,7 +65,7 @@ struct clockframe { #define CLKF_USERMODE(framep) \ (((framep)->cf_tf.tf_regs[FRAME_PS] & ALPHA_PSL_USERMODE) != 0) #define CLKF_PC(framep) ((framep)->cf_tf.tf_regs[FRAME_PC]) -#define CLKF_INTR(framep) (PCPU_GET(intr_nesting_level) >= 2) +#define CLKF_INTR(framep) (curproc->p_intr_nesting_level >= 2) /* * Preempt the current process if in interrupt from user mode, @@ -92,8 +92,6 @@ struct clockframe { #define aston() PCPU_SET(astpending, 1) #ifdef _KERNEL -extern u_int astpending; -extern u_int32_t intr_nesting_level; /* bookeeping only; counts sw intrs */ extern u_int32_t want_resched; /* resched() was called */ #endif diff --git a/sys/alpha/include/globaldata.h b/sys/alpha/include/globaldata.h index 127dd09..d99c5a9 100644 --- a/sys/alpha/include/globaldata.h +++ b/sys/alpha/include/globaldata.h @@ -56,7 +56,6 @@ struct globaldata { u_int64_t gd_pending_ipis; /* pending IPI events */ u_int32_t gd_next_asn; /* next ASN to allocate */ u_int32_t gd_current_asngen; /* ASN rollover check */ - u_int32_t gd_intr_nesting_level; /* interrupt recursion */ u_int gd_astpending; SLIST_ENTRY(globaldata) gd_allcpu; diff --git a/sys/alpha/include/pcpu.h b/sys/alpha/include/pcpu.h index 127dd09..d99c5a9 100644 --- a/sys/alpha/include/pcpu.h +++ b/sys/alpha/include/pcpu.h @@ -56,7 +56,6 @@ struct globaldata { u_int64_t gd_pending_ipis; /* pending IPI events */ u_int32_t gd_next_asn; /* next ASN to allocate */ u_int32_t gd_current_asngen; /* ASN rollover check */ - u_int32_t gd_intr_nesting_level; /* interrupt recursion */ u_int gd_astpending; SLIST_ENTRY(globaldata) gd_allcpu; diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index 4ebf35d..a2cdb47 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -48,7 +48,8 @@ IDTVEC(vec_name) ; \ movl $KPSEL,%eax ; \ mov %ax,%fs ; \ FAKE_MCOUNT(13*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ addl $4, %esp ; \ @@ -58,6 +59,7 @@ IDTVEC(vec_name) ; \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -150,7 +152,8 @@ IDTVEC(vec_name) ; \ MASK_LEVEL_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ 0: ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ @@ -160,6 +163,7 @@ __CONCAT(Xresume,irq_num): ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -301,7 +305,8 @@ _Xcpuast: FAKE_MCOUNT(13*4(%esp)) orl $AST_PENDING, PCPU(ASTPENDING) /* XXX */ - incb PCPU(INTR_NESTING_LEVEL) + movl PCPU(CURPROC),%ebx + incl P_INTR_NESTING_LEVEL(%ebx) sti movl PCPU(CPUID), %eax @@ -316,6 +321,7 @@ _Xcpuast: 2: lock incl CNAME(cpuast_cnt) + decl P_INTR_NESTING_LEVEL(%ebx) MEXITCOUNT jmp _doreti 1: @@ -323,6 +329,7 @@ _Xcpuast: POP_FRAME iret +#if 0 /* * Executed by a CPU when it receives an XFORWARD_IRQ IPI. @@ -360,7 +367,6 @@ _Xforward_irq: POP_FRAME iret -#if 0 /* * */ diff --git a/sys/amd64/amd64/exception.S b/sys/amd64/amd64/exception.S index 603b4be..aafbf0b 100644 --- a/sys/amd64/amd64/exception.S +++ b/sys/amd64/amd64/exception.S @@ -177,7 +177,6 @@ IDTVEC(fpu) call _npx_intr addl $4,%esp - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti #else /* DEV_NPX */ @@ -216,7 +215,6 @@ calltrap: /* * Return via _doreti to handle ASTs. */ - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti @@ -257,7 +255,6 @@ IDTVEC(syscall) cli /* atomic astpending access */ cmpl $0,PCPU(ASTPENDING) /* AST pending? */ je doreti_syscall_ret /* no, get out of here */ - movb $1,PCPU(INTR_NESTING_LEVEL) jmp _doreti /* @@ -289,7 +286,6 @@ IDTVEC(int0x80_syscall) cli /* atomic astpending access */ cmpl $0,PCPU(ASTPENDING) /* AST pending? */ je doreti_syscall_ret /* no, get out of here */ - movb $1,PCPU(INTR_NESTING_LEVEL) jmp _doreti ENTRY(fork_trampoline) @@ -323,7 +319,6 @@ ENTRY(fork_trampoline) /* * Return via _doreti to handle ASTs. */ - movb $1,PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti diff --git a/sys/amd64/amd64/exception.s b/sys/amd64/amd64/exception.s index 603b4be..aafbf0b 100644 --- a/sys/amd64/amd64/exception.s +++ b/sys/amd64/amd64/exception.s @@ -177,7 +177,6 @@ IDTVEC(fpu) call _npx_intr addl $4,%esp - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti #else /* DEV_NPX */ @@ -216,7 +215,6 @@ calltrap: /* * Return via _doreti to handle ASTs. */ - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti @@ -257,7 +255,6 @@ IDTVEC(syscall) cli /* atomic astpending access */ cmpl $0,PCPU(ASTPENDING) /* AST pending? */ je doreti_syscall_ret /* no, get out of here */ - movb $1,PCPU(INTR_NESTING_LEVEL) jmp _doreti /* @@ -289,7 +286,6 @@ IDTVEC(int0x80_syscall) cli /* atomic astpending access */ cmpl $0,PCPU(ASTPENDING) /* AST pending? */ je doreti_syscall_ret /* no, get out of here */ - movb $1,PCPU(INTR_NESTING_LEVEL) jmp _doreti ENTRY(fork_trampoline) @@ -323,7 +319,6 @@ ENTRY(fork_trampoline) /* * Return via _doreti to handle ASTs. */ - movb $1,PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index 8b9ca33..c307665 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -80,6 +80,7 @@ ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace)); ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap)); ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active)); ASSYM(P_ADDR, offsetof(struct proc, p_addr)); +ASSYM(P_INTR_NESTING_LEVEL, offsetof(struct proc, p_intr_nesting_level)); ASSYM(P_STAT, offsetof(struct proc, p_stat)); ASSYM(P_WCHAN, offsetof(struct proc, p_wchan)); @@ -182,7 +183,6 @@ ASSYM(GD_TSS_GDT, offsetof(struct globaldata, gd_tss_gdt)); ASSYM(GD_ASTPENDING, offsetof(struct globaldata, gd_astpending)); ASSYM(AST_PENDING, AST_PENDING); ASSYM(AST_RESCHED, AST_RESCHED); -ASSYM(GD_INTR_NESTING_LEVEL, offsetof(struct globaldata, gd_intr_nesting_level)); #ifdef USER_LDT ASSYM(GD_CURRENTLDT, offsetof(struct globaldata, gd_currentldt)); diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index bf560cd..966d3f8 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c index bf560cd..966d3f8 100644 --- a/sys/amd64/amd64/mptable.c +++ b/sys/amd64/amd64/mptable.c @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index 6ee3e6d..0170dbe 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -480,7 +480,7 @@ restart: if (in_vm86call) break; - if (PCPU_GET(intr_nesting_level) != 0) + if (p->p_intr_nesting_level != 0) break; /* @@ -687,7 +687,7 @@ trap_pfault(frame, usermode, eva) if (p == NULL || (!usermode && va < VM_MAXUSER_ADDRESS && - (PCPU_GET(intr_nesting_level) != 0 || + (p->p_intr_nesting_level != 0 || PCPU_GET(curpcb) == NULL || PCPU_GET(curpcb)->pcb_onfault == NULL))) { trap_fatal(frame, eva); @@ -751,7 +751,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (PCPU_GET(intr_nesting_level) == 0 && + if (p->p_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; @@ -858,7 +858,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (PCPU_GET(intr_nesting_level) == 0 && + if (p->p_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h index 6c4162b..87c47c5 100644 --- a/sys/amd64/include/cpu.h +++ b/sys/amd64/include/cpu.h @@ -62,7 +62,7 @@ #define CLKF_USERMODE(framep) \ ((ISPL((framep)->cf_cs) == SEL_UPL) || (framep->cf_eflags & PSL_VM)) -#define CLKF_INTR(framep) (PCPU_GET(intr_nesting_level) >= 2) +#define CLKF_INTR(framep) (curproc->p_intr_nesting_level >= 2) #define CLKF_PC(framep) ((framep)->cf_eip) /* diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h index bf560cd..966d3f8 100644 --- a/sys/amd64/include/mptable.h +++ b/sys/amd64/include/mptable.h @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h index 149026e..7a7437a6 100644 --- a/sys/amd64/include/pcpu.h +++ b/sys/amd64/include/pcpu.h @@ -58,8 +58,6 @@ struct globaldata { struct timeval gd_switchtime; struct i386tss gd_common_tss; int gd_switchticks; - u_char gd_intr_nesting_level; - u_char gd_pad0[3]; struct segment_descriptor gd_common_tssd; struct segment_descriptor *gd_tss_gdt; int gd_currentldt; /* only used for USER_LDT */ diff --git a/sys/amd64/isa/atpic_vector.S b/sys/amd64/isa/atpic_vector.S index 635d2ad..0a3456f 100644 --- a/sys/amd64/isa/atpic_vector.S +++ b/sys/amd64/isa/atpic_vector.S @@ -61,7 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -69,6 +70,7 @@ IDTVEC(vec_name) ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -102,13 +104,15 @@ IDTVEC(vec_name) ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ sti ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* _doreti, but it's probably better to use less cache. */ \ diff --git a/sys/amd64/isa/icu_vector.S b/sys/amd64/isa/icu_vector.S index 635d2ad..0a3456f 100644 --- a/sys/amd64/isa/icu_vector.S +++ b/sys/amd64/isa/icu_vector.S @@ -61,7 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -69,6 +70,7 @@ IDTVEC(vec_name) ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -102,13 +104,15 @@ IDTVEC(vec_name) ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ sti ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* _doreti, but it's probably better to use less cache. */ \ diff --git a/sys/amd64/isa/icu_vector.s b/sys/amd64/isa/icu_vector.s index 635d2ad..0a3456f 100644 --- a/sys/amd64/isa/icu_vector.s +++ b/sys/amd64/isa/icu_vector.s @@ -61,7 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -69,6 +70,7 @@ IDTVEC(vec_name) ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -102,13 +104,15 @@ IDTVEC(vec_name) ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ sti ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* _doreti, but it's probably better to use less cache. */ \ diff --git a/sys/amd64/isa/intr_machdep.c b/sys/amd64/isa/intr_machdep.c index 00f5f23..b9328ae 100644 --- a/sys/amd64/isa/intr_machdep.c +++ b/sys/amd64/isa/intr_machdep.c @@ -577,6 +577,7 @@ inthand_add(const char *name, int irq, driver_intr_t handler, void *arg, if (errcode) panic("inthand_add: Can't create " "interrupt thread"); + p->p_intr_nesting_level = 1; p->p_rtprio.type = RTP_PRIO_ITHREAD; p->p_stat = SWAIT; /* we're idle */ diff --git a/sys/amd64/isa/nmi.c b/sys/amd64/isa/nmi.c index 00f5f23..b9328ae 100644 --- a/sys/amd64/isa/nmi.c +++ b/sys/amd64/isa/nmi.c @@ -577,6 +577,7 @@ inthand_add(const char *name, int irq, driver_intr_t handler, void *arg, if (errcode) panic("inthand_add: Can't create " "interrupt thread"); + p->p_intr_nesting_level = 1; p->p_rtprio.type = RTP_PRIO_ITHREAD; p->p_stat = SWAIT; /* we're idle */ diff --git a/sys/dev/usb/ohci.c b/sys/dev/usb/ohci.c index c9062a4..c2abcd8 100644 --- a/sys/dev/usb/ohci.c +++ b/sys/dev/usb/ohci.c @@ -1889,7 +1889,7 @@ ohci_abort_xfer(xfer, status) timeout(ohci_abort_xfer_end, xfer, hz / USB_FRAMES_PER_SECOND); } else { #if defined(DIAGNOSTIC) && defined(__i386__) && defined(__FreeBSD__) - KASSERT(PCPU_GET(intr_nesting_level) == 0, + KASSERT(curproc->p_intr_nesting_level == 0, ("ohci_abort_req in interrupt context")); #endif usb_delay_ms(opipe->pipe.device->bus, 1); diff --git a/sys/dev/usb/uhci.c b/sys/dev/usb/uhci.c index 7098db1..1628268 100644 --- a/sys/dev/usb/uhci.c +++ b/sys/dev/usb/uhci.c @@ -1610,7 +1610,7 @@ uhci_abort_xfer(usbd_xfer_handle xfer, usbd_status status) timeout(uhci_abort_xfer_end, xfer, hz / USB_FRAMES_PER_SECOND); } else { #if defined(DIAGNOSTIC) && defined(__i386__) && defined(__FreeBSD__) - KASSERT(PCPU_GET(intr_nesting_level) == 0, + KASSERT(curproc->p_intr_nesting_level == 0, ("ohci_abort_req in interrupt context")); #endif usb_delay_ms(xfer->pipe->device->bus, 1); diff --git a/sys/dev/usb/usbdi.c b/sys/dev/usb/usbdi.c index c39a856..6203e8a 100644 --- a/sys/dev/usb/usbdi.c +++ b/sys/dev/usb/usbdi.c @@ -931,7 +931,7 @@ usbd_do_request_flags(dev, req, data, flags, actlen) #ifdef DIAGNOSTIC #if defined(__i386__) && defined(__FreeBSD__) - KASSERT(PCPU_GET(intr_nesting_level) == 0, + KASSERT(curproc->p_intr_nesting_level == 0, ("usbd_do_request: in interrupt context")); #endif if (dev->bus->intr_context) { diff --git a/sys/dev/vinum/vinumhdr.h b/sys/dev/vinum/vinumhdr.h index 8a5f70e..2b7d290 100644 --- a/sys/dev/vinum/vinumhdr.h +++ b/sys/dev/vinum/vinumhdr.h @@ -87,7 +87,7 @@ void FFree (void *mem, char *, int); #define LOCKDRIVE(d) lockdrive (d, __FILE__, __LINE__) #else #define Malloc(x) malloc((x), M_DEVBUF, \ - PCPU_GET(intr_nesting_level) == 0? M_WAITOK: M_NOWAIT) + curproc->p_intr_nesting_level == 0? M_WAITOK: M_NOWAIT) #define Free(x) free((x), M_DEVBUF) #define LOCKDRIVE(d) lockdrive (d) #endif diff --git a/sys/dev/vinum/vinummemory.c b/sys/dev/vinum/vinummemory.c index e28f7ae..58eea67 100644 --- a/sys/dev/vinum/vinummemory.c +++ b/sys/dev/vinum/vinummemory.c @@ -151,7 +151,7 @@ MMalloc(int size, char *file, int line) return 0; /* can't continue */ } /* Wait for malloc if we can */ - result = malloc(size, M_DEVBUF, PCPU_GET(intr_nesting_level) == 0 ? M_WAITOK : M_NOWAIT); + result = malloc(size, M_DEVBUF, curproc->p_intr_nesting_level == 0 ? M_WAITOK : M_NOWAIT); if (result == NULL) log(LOG_ERR, "vinum: can't allocate %d bytes from %s:%d\n", size, file, line); else { diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s index 4ebf35d..a2cdb47 100644 --- a/sys/i386/i386/apic_vector.s +++ b/sys/i386/i386/apic_vector.s @@ -48,7 +48,8 @@ IDTVEC(vec_name) ; \ movl $KPSEL,%eax ; \ mov %ax,%fs ; \ FAKE_MCOUNT(13*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ addl $4, %esp ; \ @@ -58,6 +59,7 @@ IDTVEC(vec_name) ; \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -150,7 +152,8 @@ IDTVEC(vec_name) ; \ MASK_LEVEL_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ 0: ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ @@ -160,6 +163,7 @@ __CONCAT(Xresume,irq_num): ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -301,7 +305,8 @@ _Xcpuast: FAKE_MCOUNT(13*4(%esp)) orl $AST_PENDING, PCPU(ASTPENDING) /* XXX */ - incb PCPU(INTR_NESTING_LEVEL) + movl PCPU(CURPROC),%ebx + incl P_INTR_NESTING_LEVEL(%ebx) sti movl PCPU(CPUID), %eax @@ -316,6 +321,7 @@ _Xcpuast: 2: lock incl CNAME(cpuast_cnt) + decl P_INTR_NESTING_LEVEL(%ebx) MEXITCOUNT jmp _doreti 1: @@ -323,6 +329,7 @@ _Xcpuast: POP_FRAME iret +#if 0 /* * Executed by a CPU when it receives an XFORWARD_IRQ IPI. @@ -360,7 +367,6 @@ _Xforward_irq: POP_FRAME iret -#if 0 /* * */ diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s index 603b4be..aafbf0b 100644 --- a/sys/i386/i386/exception.s +++ b/sys/i386/i386/exception.s @@ -177,7 +177,6 @@ IDTVEC(fpu) call _npx_intr addl $4,%esp - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti #else /* DEV_NPX */ @@ -216,7 +215,6 @@ calltrap: /* * Return via _doreti to handle ASTs. */ - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti @@ -257,7 +255,6 @@ IDTVEC(syscall) cli /* atomic astpending access */ cmpl $0,PCPU(ASTPENDING) /* AST pending? */ je doreti_syscall_ret /* no, get out of here */ - movb $1,PCPU(INTR_NESTING_LEVEL) jmp _doreti /* @@ -289,7 +286,6 @@ IDTVEC(int0x80_syscall) cli /* atomic astpending access */ cmpl $0,PCPU(ASTPENDING) /* AST pending? */ je doreti_syscall_ret /* no, get out of here */ - movb $1,PCPU(INTR_NESTING_LEVEL) jmp _doreti ENTRY(fork_trampoline) @@ -323,7 +319,6 @@ ENTRY(fork_trampoline) /* * Return via _doreti to handle ASTs. */ - movb $1,PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c index 8b9ca33..c307665 100644 --- a/sys/i386/i386/genassym.c +++ b/sys/i386/i386/genassym.c @@ -80,6 +80,7 @@ ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace)); ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap)); ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active)); ASSYM(P_ADDR, offsetof(struct proc, p_addr)); +ASSYM(P_INTR_NESTING_LEVEL, offsetof(struct proc, p_intr_nesting_level)); ASSYM(P_STAT, offsetof(struct proc, p_stat)); ASSYM(P_WCHAN, offsetof(struct proc, p_wchan)); @@ -182,7 +183,6 @@ ASSYM(GD_TSS_GDT, offsetof(struct globaldata, gd_tss_gdt)); ASSYM(GD_ASTPENDING, offsetof(struct globaldata, gd_astpending)); ASSYM(AST_PENDING, AST_PENDING); ASSYM(AST_RESCHED, AST_RESCHED); -ASSYM(GD_INTR_NESTING_LEVEL, offsetof(struct globaldata, gd_intr_nesting_level)); #ifdef USER_LDT ASSYM(GD_CURRENTLDT, offsetof(struct globaldata, gd_currentldt)); diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index bf560cd..966d3f8 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c index bf560cd..966d3f8 100644 --- a/sys/i386/i386/mptable.c +++ b/sys/i386/i386/mptable.c @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 6ee3e6d..0170dbe 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -480,7 +480,7 @@ restart: if (in_vm86call) break; - if (PCPU_GET(intr_nesting_level) != 0) + if (p->p_intr_nesting_level != 0) break; /* @@ -687,7 +687,7 @@ trap_pfault(frame, usermode, eva) if (p == NULL || (!usermode && va < VM_MAXUSER_ADDRESS && - (PCPU_GET(intr_nesting_level) != 0 || + (p->p_intr_nesting_level != 0 || PCPU_GET(curpcb) == NULL || PCPU_GET(curpcb)->pcb_onfault == NULL))) { trap_fatal(frame, eva); @@ -751,7 +751,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (PCPU_GET(intr_nesting_level) == 0 && + if (p->p_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; @@ -858,7 +858,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (PCPU_GET(intr_nesting_level) == 0 && + if (p->p_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; diff --git a/sys/i386/i386/vm86bios.s b/sys/i386/i386/vm86bios.s index a71dd11..5f3013c 100644 --- a/sys/i386/i386/vm86bios.s +++ b/sys/i386/i386/vm86bios.s @@ -131,7 +131,6 @@ ENTRY(vm86_bioscall) /* * Return via _doreti */ - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti diff --git a/sys/i386/include/cpu.h b/sys/i386/include/cpu.h index 6c4162b..87c47c5 100644 --- a/sys/i386/include/cpu.h +++ b/sys/i386/include/cpu.h @@ -62,7 +62,7 @@ #define CLKF_USERMODE(framep) \ ((ISPL((framep)->cf_cs) == SEL_UPL) || (framep->cf_eflags & PSL_VM)) -#define CLKF_INTR(framep) (PCPU_GET(intr_nesting_level) >= 2) +#define CLKF_INTR(framep) (curproc->p_intr_nesting_level >= 2) #define CLKF_PC(framep) ((framep)->cf_eip) /* diff --git a/sys/i386/include/globaldata.h b/sys/i386/include/globaldata.h index 149026e..7a7437a6 100644 --- a/sys/i386/include/globaldata.h +++ b/sys/i386/include/globaldata.h @@ -58,8 +58,6 @@ struct globaldata { struct timeval gd_switchtime; struct i386tss gd_common_tss; int gd_switchticks; - u_char gd_intr_nesting_level; - u_char gd_pad0[3]; struct segment_descriptor gd_common_tssd; struct segment_descriptor *gd_tss_gdt; int gd_currentldt; /* only used for USER_LDT */ diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h index bf560cd..966d3f8 100644 --- a/sys/i386/include/mptable.h +++ b/sys/i386/include/mptable.h @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h index 149026e..7a7437a6 100644 --- a/sys/i386/include/pcpu.h +++ b/sys/i386/include/pcpu.h @@ -58,8 +58,6 @@ struct globaldata { struct timeval gd_switchtime; struct i386tss gd_common_tss; int gd_switchticks; - u_char gd_intr_nesting_level; - u_char gd_pad0[3]; struct segment_descriptor gd_common_tssd; struct segment_descriptor *gd_tss_gdt; int gd_currentldt; /* only used for USER_LDT */ diff --git a/sys/i386/isa/apic_vector.s b/sys/i386/isa/apic_vector.s index 4ebf35d..a2cdb47 100644 --- a/sys/i386/isa/apic_vector.s +++ b/sys/i386/isa/apic_vector.s @@ -48,7 +48,8 @@ IDTVEC(vec_name) ; \ movl $KPSEL,%eax ; \ mov %ax,%fs ; \ FAKE_MCOUNT(13*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ addl $4, %esp ; \ @@ -58,6 +59,7 @@ IDTVEC(vec_name) ; \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -150,7 +152,8 @@ IDTVEC(vec_name) ; \ MASK_LEVEL_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ 0: ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ @@ -160,6 +163,7 @@ __CONCAT(Xresume,irq_num): ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -301,7 +305,8 @@ _Xcpuast: FAKE_MCOUNT(13*4(%esp)) orl $AST_PENDING, PCPU(ASTPENDING) /* XXX */ - incb PCPU(INTR_NESTING_LEVEL) + movl PCPU(CURPROC),%ebx + incl P_INTR_NESTING_LEVEL(%ebx) sti movl PCPU(CPUID), %eax @@ -316,6 +321,7 @@ _Xcpuast: 2: lock incl CNAME(cpuast_cnt) + decl P_INTR_NESTING_LEVEL(%ebx) MEXITCOUNT jmp _doreti 1: @@ -323,6 +329,7 @@ _Xcpuast: POP_FRAME iret +#if 0 /* * Executed by a CPU when it receives an XFORWARD_IRQ IPI. @@ -360,7 +367,6 @@ _Xforward_irq: POP_FRAME iret -#if 0 /* * */ diff --git a/sys/i386/isa/atpic_vector.s b/sys/i386/isa/atpic_vector.s index 635d2ad..0a3456f 100644 --- a/sys/i386/isa/atpic_vector.s +++ b/sys/i386/isa/atpic_vector.s @@ -61,7 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -69,6 +70,7 @@ IDTVEC(vec_name) ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -102,13 +104,15 @@ IDTVEC(vec_name) ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ sti ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* _doreti, but it's probably better to use less cache. */ \ diff --git a/sys/i386/isa/icu_vector.s b/sys/i386/isa/icu_vector.s index 635d2ad..0a3456f 100644 --- a/sys/i386/isa/icu_vector.s +++ b/sys/i386/isa/icu_vector.s @@ -61,7 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -69,6 +70,7 @@ IDTVEC(vec_name) ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -102,13 +104,15 @@ IDTVEC(vec_name) ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ sti ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* _doreti, but it's probably better to use less cache. */ \ diff --git a/sys/i386/isa/intr_machdep.c b/sys/i386/isa/intr_machdep.c index 00f5f23..b9328ae 100644 --- a/sys/i386/isa/intr_machdep.c +++ b/sys/i386/isa/intr_machdep.c @@ -577,6 +577,7 @@ inthand_add(const char *name, int irq, driver_intr_t handler, void *arg, if (errcode) panic("inthand_add: Can't create " "interrupt thread"); + p->p_intr_nesting_level = 1; p->p_rtprio.type = RTP_PRIO_ITHREAD; p->p_stat = SWAIT; /* we're idle */ diff --git a/sys/i386/isa/ipl.s b/sys/i386/isa/ipl.s index ca874ea..7c41589 100644 --- a/sys/i386/isa/ipl.s +++ b/sys/i386/isa/ipl.s @@ -57,8 +57,6 @@ _doreti: FAKE_MCOUNT(_bintr) /* init "from" _bintr -> _doreti */ doreti_next: - decb PCPU(INTR_NESTING_LEVEL) - /* Check for ASTs that can be handled now. */ testl $AST_PENDING,PCPU(ASTPENDING) je doreti_exit /* no AST, exit */ @@ -128,7 +126,6 @@ doreti_ast: sti movl $T_ASTFLT,TF_TRAPNO(%esp) call _ast - movb $1,PCPU(INTR_NESTING_LEVEL) /* for doreti_next to decrement */ jmp doreti_next #ifdef APIC_IO diff --git a/sys/i386/isa/nmi.c b/sys/i386/isa/nmi.c index 00f5f23..b9328ae 100644 --- a/sys/i386/isa/nmi.c +++ b/sys/i386/isa/nmi.c @@ -577,6 +577,7 @@ inthand_add(const char *name, int irq, driver_intr_t handler, void *arg, if (errcode) panic("inthand_add: Can't create " "interrupt thread"); + p->p_intr_nesting_level = 1; p->p_rtprio.type = RTP_PRIO_ITHREAD; p->p_stat = SWAIT; /* we're idle */ diff --git a/sys/ia64/ia64/interrupt.c b/sys/ia64/ia64/interrupt.c index 698b46c..fc2f84a 100644 --- a/sys/ia64/ia64/interrupt.c +++ b/sys/ia64/ia64/interrupt.c @@ -78,8 +78,10 @@ static u_int schedclk2; void interrupt(u_int64_t vector, struct trapframe *framep) { - atomic_add_int(PCPU_PTR(intr_nesting_level), 1); + struct proc *p; + p = curproc; + atomic_add_int(&p->p_intr_nesting_level, 1); switch (vector) { case 240: /* clock interrupt */ CTR0(KTR_INTR, "clock interrupt"); @@ -104,7 +106,7 @@ interrupt(u_int64_t vector, struct trapframe *framep) panic("unexpected interrupt: vec %ld\n", vector); /* NOTREACHED */ } - atomic_subtract_int(PCPU_PTR(intr_nesting_level), 1); + atomic_subtract_int(&p->p_intr_nesting_level, 1); } diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c index 486d0aa..aa42156 100644 --- a/sys/ia64/ia64/mp_machdep.c +++ b/sys/ia64/ia64/mp_machdep.c @@ -772,7 +772,7 @@ smp_handle_ipi(struct trapframe *frame) if ((frame->tf_cr_ipsr & IA64_PSR_CPL) == IA64_PSR_CPL_USER) checkstate_cpustate[cpuno] = CHECKSTATE_USER; - else if (PCPU_GET(intr_nesting_level) == 1) + else if (curproc->p_intr_nesting_level == 1) checkstate_cpustate[cpuno] = CHECKSTATE_SYS; else checkstate_cpustate[cpuno] = CHECKSTATE_INTR; diff --git a/sys/ia64/include/cpu.h b/sys/ia64/include/cpu.h index d581447..ff36f05 100644 --- a/sys/ia64/include/cpu.h +++ b/sys/ia64/include/cpu.h @@ -67,7 +67,7 @@ struct clockframe { #define CLKF_BASEPRI(framep) \ (((framep)->cf_tf.tf_cr_ipsr & IA64_PSR_I) == 0) #define CLKF_PC(framep) ((framep)->cf_tf.tf_cr_iip) -#define CLKF_INTR(framep) (PCPU_GET(intr_nesting_level) >= 2) +#define CLKF_INTR(framep) (curproc->p_intr_nesting_level >= 2) /* * Preempt the current process if in interrupt from user mode, diff --git a/sys/ia64/include/globaldata.h b/sys/ia64/include/globaldata.h index 6068bd1..91c7fff 100644 --- a/sys/ia64/include/globaldata.h +++ b/sys/ia64/include/globaldata.h @@ -56,7 +56,6 @@ struct globaldata { struct pmap *gd_current_pmap; /* which pmap is active */ u_int32_t gd_next_asn; /* next ASN to allocate */ u_int32_t gd_current_asngen; /* ASN rollover check */ - u_int32_t gd_intr_nesting_level; /* interrupt recursion */ u_int gd_astpending; SLIST_ENTRY(globaldata) gd_allcpu; diff --git a/sys/ia64/include/pcpu.h b/sys/ia64/include/pcpu.h index 6068bd1..91c7fff 100644 --- a/sys/ia64/include/pcpu.h +++ b/sys/ia64/include/pcpu.h @@ -56,7 +56,6 @@ struct globaldata { struct pmap *gd_current_pmap; /* which pmap is active */ u_int32_t gd_next_asn; /* next ASN to allocate */ u_int32_t gd_current_asngen; /* ASN rollover check */ - u_int32_t gd_intr_nesting_level; /* interrupt recursion */ u_int gd_astpending; SLIST_ENTRY(globaldata) gd_allcpu; diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index 1c8cb38..9dc8ec3 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -351,6 +351,7 @@ again: nextpid = trypid; p2 = newproc; + p2->p_intr_nesting_level = 0; p2->p_stat = SIDL; /* protect against others */ p2->p_pid = trypid; LIST_INSERT_HEAD(&allproc, p2, p_list); diff --git a/sys/kern/kern_malloc.c b/sys/kern/kern_malloc.c index 396f02b..ec33f61 100644 --- a/sys/kern/kern_malloc.c +++ b/sys/kern/kern_malloc.c @@ -44,6 +44,7 @@ #include #include #include +#include #include #include @@ -147,7 +148,7 @@ malloc(size, type, flags) #if defined(INVARIANTS) && defined(__i386__) if (flags == M_WAITOK) - KASSERT(PCPU_GET(intr_nesting_level) == 0, + KASSERT(curproc->p_intr_nesting_level == 0, ("malloc(M_WAITOK) in interrupt context")); #endif indx = BUCKETINDX(size); diff --git a/sys/kern/subr_smp.c b/sys/kern/subr_smp.c index bf560cd..966d3f8 100644 --- a/sys/kern/subr_smp.c +++ b/sys/kern/subr_smp.c @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index 6ee3e6d..0170dbe 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -480,7 +480,7 @@ restart: if (in_vm86call) break; - if (PCPU_GET(intr_nesting_level) != 0) + if (p->p_intr_nesting_level != 0) break; /* @@ -687,7 +687,7 @@ trap_pfault(frame, usermode, eva) if (p == NULL || (!usermode && va < VM_MAXUSER_ADDRESS && - (PCPU_GET(intr_nesting_level) != 0 || + (p->p_intr_nesting_level != 0 || PCPU_GET(curpcb) == NULL || PCPU_GET(curpcb)->pcb_onfault == NULL))) { trap_fatal(frame, eva); @@ -751,7 +751,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (PCPU_GET(intr_nesting_level) == 0 && + if (p->p_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; @@ -858,7 +858,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (PCPU_GET(intr_nesting_level) == 0 && + if (p->p_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; diff --git a/sys/powerpc/include/globaldata.h b/sys/powerpc/include/globaldata.h index 127dd09..d99c5a9 100644 --- a/sys/powerpc/include/globaldata.h +++ b/sys/powerpc/include/globaldata.h @@ -56,7 +56,6 @@ struct globaldata { u_int64_t gd_pending_ipis; /* pending IPI events */ u_int32_t gd_next_asn; /* next ASN to allocate */ u_int32_t gd_current_asngen; /* ASN rollover check */ - u_int32_t gd_intr_nesting_level; /* interrupt recursion */ u_int gd_astpending; SLIST_ENTRY(globaldata) gd_allcpu; diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h index 127dd09..d99c5a9 100644 --- a/sys/powerpc/include/pcpu.h +++ b/sys/powerpc/include/pcpu.h @@ -56,7 +56,6 @@ struct globaldata { u_int64_t gd_pending_ipis; /* pending IPI events */ u_int32_t gd_next_asn; /* next ASN to allocate */ u_int32_t gd_current_asngen; /* ASN rollover check */ - u_int32_t gd_intr_nesting_level; /* interrupt recursion */ u_int gd_astpending; SLIST_ENTRY(globaldata) gd_allcpu; diff --git a/sys/sys/proc.h b/sys/sys/proc.h index 41f8f6a..ee093ba 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -159,6 +159,7 @@ struct proc { #define p_rlimit p_limit->pl_rlimit int p_flag; /* (c/j) P_* flags. */ + int p_intr_nesting_level; /* (n) Interrupt recursion. */ char p_stat; /* (j) S* process status. */ char p_pad1[3]; -- cgit v1.1