From 937122ae6dc02a639d13dd06132201548bd7364f Mon Sep 17 00:00:00 2001 From: jake Date: Sun, 21 Jan 2001 19:25:07 +0000 Subject: Make intr_nesting_level per-process, rather than per-cpu. Setup interrupt threads to run with it always >= 1, so that malloc can detect M_WAITOK from "interrupt" context. This is also necessary in order to context switch from sched_ithd() directly. Reviewed By: peter --- sys/i386/i386/apic_vector.s | 14 ++++++++++---- sys/i386/i386/exception.s | 5 ----- sys/i386/i386/genassym.c | 2 +- sys/i386/i386/mp_machdep.c | 4 +++- sys/i386/i386/mptable.c | 4 +++- sys/i386/i386/trap.c | 8 ++++---- sys/i386/i386/vm86bios.s | 1 - sys/i386/include/cpu.h | 2 +- sys/i386/include/globaldata.h | 2 -- sys/i386/include/mptable.h | 4 +++- sys/i386/include/pcpu.h | 2 -- sys/i386/isa/apic_vector.s | 14 ++++++++++---- sys/i386/isa/atpic_vector.s | 8 ++++++-- sys/i386/isa/icu_vector.s | 8 ++++++-- sys/i386/isa/intr_machdep.c | 1 + sys/i386/isa/ipl.s | 3 --- sys/i386/isa/nmi.c | 1 + 17 files changed, 49 insertions(+), 34 deletions(-) (limited to 'sys/i386') diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s index 4ebf35d..a2cdb47 100644 --- a/sys/i386/i386/apic_vector.s +++ b/sys/i386/i386/apic_vector.s @@ -48,7 +48,8 @@ IDTVEC(vec_name) ; \ movl $KPSEL,%eax ; \ mov %ax,%fs ; \ FAKE_MCOUNT(13*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ addl $4, %esp ; \ @@ -58,6 +59,7 @@ IDTVEC(vec_name) ; \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -150,7 +152,8 @@ IDTVEC(vec_name) ; \ MASK_LEVEL_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ 0: ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ @@ -160,6 +163,7 @@ __CONCAT(Xresume,irq_num): ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -301,7 +305,8 @@ _Xcpuast: FAKE_MCOUNT(13*4(%esp)) orl $AST_PENDING, PCPU(ASTPENDING) /* XXX */ - incb PCPU(INTR_NESTING_LEVEL) + movl PCPU(CURPROC),%ebx + incl P_INTR_NESTING_LEVEL(%ebx) sti movl PCPU(CPUID), %eax @@ -316,6 +321,7 @@ _Xcpuast: 2: lock incl CNAME(cpuast_cnt) + decl P_INTR_NESTING_LEVEL(%ebx) MEXITCOUNT jmp _doreti 1: @@ -323,6 +329,7 @@ _Xcpuast: POP_FRAME iret +#if 0 /* * Executed by a CPU when it receives an XFORWARD_IRQ IPI. @@ -360,7 +367,6 @@ _Xforward_irq: POP_FRAME iret -#if 0 /* * */ diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s index 603b4be..aafbf0b 100644 --- a/sys/i386/i386/exception.s +++ b/sys/i386/i386/exception.s @@ -177,7 +177,6 @@ IDTVEC(fpu) call _npx_intr addl $4,%esp - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti #else /* DEV_NPX */ @@ -216,7 +215,6 @@ calltrap: /* * Return via _doreti to handle ASTs. */ - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti @@ -257,7 +255,6 @@ IDTVEC(syscall) cli /* atomic astpending access */ cmpl $0,PCPU(ASTPENDING) /* AST pending? */ je doreti_syscall_ret /* no, get out of here */ - movb $1,PCPU(INTR_NESTING_LEVEL) jmp _doreti /* @@ -289,7 +286,6 @@ IDTVEC(int0x80_syscall) cli /* atomic astpending access */ cmpl $0,PCPU(ASTPENDING) /* AST pending? */ je doreti_syscall_ret /* no, get out of here */ - movb $1,PCPU(INTR_NESTING_LEVEL) jmp _doreti ENTRY(fork_trampoline) @@ -323,7 +319,6 @@ ENTRY(fork_trampoline) /* * Return via _doreti to handle ASTs. */ - movb $1,PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c index 8b9ca33..c307665 100644 --- a/sys/i386/i386/genassym.c +++ b/sys/i386/i386/genassym.c @@ -80,6 +80,7 @@ ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace)); ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap)); ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active)); ASSYM(P_ADDR, offsetof(struct proc, p_addr)); +ASSYM(P_INTR_NESTING_LEVEL, offsetof(struct proc, p_intr_nesting_level)); ASSYM(P_STAT, offsetof(struct proc, p_stat)); ASSYM(P_WCHAN, offsetof(struct proc, p_wchan)); @@ -182,7 +183,6 @@ ASSYM(GD_TSS_GDT, offsetof(struct globaldata, gd_tss_gdt)); ASSYM(GD_ASTPENDING, offsetof(struct globaldata, gd_astpending)); ASSYM(AST_PENDING, AST_PENDING); ASSYM(AST_RESCHED, AST_RESCHED); -ASSYM(GD_INTR_NESTING_LEVEL, offsetof(struct globaldata, gd_intr_nesting_level)); #ifdef USER_LDT ASSYM(GD_CURRENTLDT, offsetof(struct globaldata, gd_currentldt)); diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index bf560cd..966d3f8 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c index bf560cd..966d3f8 100644 --- a/sys/i386/i386/mptable.c +++ b/sys/i386/i386/mptable.c @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 6ee3e6d..0170dbe 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -480,7 +480,7 @@ restart: if (in_vm86call) break; - if (PCPU_GET(intr_nesting_level) != 0) + if (p->p_intr_nesting_level != 0) break; /* @@ -687,7 +687,7 @@ trap_pfault(frame, usermode, eva) if (p == NULL || (!usermode && va < VM_MAXUSER_ADDRESS && - (PCPU_GET(intr_nesting_level) != 0 || + (p->p_intr_nesting_level != 0 || PCPU_GET(curpcb) == NULL || PCPU_GET(curpcb)->pcb_onfault == NULL))) { trap_fatal(frame, eva); @@ -751,7 +751,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (PCPU_GET(intr_nesting_level) == 0 && + if (p->p_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; @@ -858,7 +858,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (PCPU_GET(intr_nesting_level) == 0 && + if (p->p_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; diff --git a/sys/i386/i386/vm86bios.s b/sys/i386/i386/vm86bios.s index a71dd11..5f3013c 100644 --- a/sys/i386/i386/vm86bios.s +++ b/sys/i386/i386/vm86bios.s @@ -131,7 +131,6 @@ ENTRY(vm86_bioscall) /* * Return via _doreti */ - incb PCPU(INTR_NESTING_LEVEL) MEXITCOUNT jmp _doreti diff --git a/sys/i386/include/cpu.h b/sys/i386/include/cpu.h index 6c4162b..87c47c5 100644 --- a/sys/i386/include/cpu.h +++ b/sys/i386/include/cpu.h @@ -62,7 +62,7 @@ #define CLKF_USERMODE(framep) \ ((ISPL((framep)->cf_cs) == SEL_UPL) || (framep->cf_eflags & PSL_VM)) -#define CLKF_INTR(framep) (PCPU_GET(intr_nesting_level) >= 2) +#define CLKF_INTR(framep) (curproc->p_intr_nesting_level >= 2) #define CLKF_PC(framep) ((framep)->cf_eip) /* diff --git a/sys/i386/include/globaldata.h b/sys/i386/include/globaldata.h index 149026e..7a7437a6 100644 --- a/sys/i386/include/globaldata.h +++ b/sys/i386/include/globaldata.h @@ -58,8 +58,6 @@ struct globaldata { struct timeval gd_switchtime; struct i386tss gd_common_tss; int gd_switchticks; - u_char gd_intr_nesting_level; - u_char gd_pad0[3]; struct segment_descriptor gd_common_tssd; struct segment_descriptor *gd_tss_gdt; int gd_currentldt; /* only used for USER_LDT */ diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h index bf560cd..966d3f8 100644 --- a/sys/i386/include/mptable.h +++ b/sys/i386/include/mptable.h @@ -581,10 +581,12 @@ mp_enable(u_int boot_addr) /* install an inter-CPU IPI for forcing an additional software trap */ setidt(XCPUAST_OFFSET, Xcpuast, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); - + +#if 0 /* install an inter-CPU IPI for interrupt forwarding */ setidt(XFORWARD_IRQ_OFFSET, Xforward_irq, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif /* install an inter-CPU IPI for CPU stop/restart */ setidt(XCPUSTOP_OFFSET, Xcpustop, diff --git a/sys/i386/include/pcpu.h b/sys/i386/include/pcpu.h index 149026e..7a7437a6 100644 --- a/sys/i386/include/pcpu.h +++ b/sys/i386/include/pcpu.h @@ -58,8 +58,6 @@ struct globaldata { struct timeval gd_switchtime; struct i386tss gd_common_tss; int gd_switchticks; - u_char gd_intr_nesting_level; - u_char gd_pad0[3]; struct segment_descriptor gd_common_tssd; struct segment_descriptor *gd_tss_gdt; int gd_currentldt; /* only used for USER_LDT */ diff --git a/sys/i386/isa/apic_vector.s b/sys/i386/isa/apic_vector.s index 4ebf35d..a2cdb47 100644 --- a/sys/i386/isa/apic_vector.s +++ b/sys/i386/isa/apic_vector.s @@ -48,7 +48,8 @@ IDTVEC(vec_name) ; \ movl $KPSEL,%eax ; \ mov %ax,%fs ; \ FAKE_MCOUNT(13*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ addl $4, %esp ; \ @@ -58,6 +59,7 @@ IDTVEC(vec_name) ; \ movl _intr_countp + (irq_num) * 4, %eax ; \ lock ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -150,7 +152,8 @@ IDTVEC(vec_name) ; \ MASK_LEVEL_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ 0: ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ @@ -160,6 +163,7 @@ __CONCAT(Xresume,irq_num): ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -301,7 +305,8 @@ _Xcpuast: FAKE_MCOUNT(13*4(%esp)) orl $AST_PENDING, PCPU(ASTPENDING) /* XXX */ - incb PCPU(INTR_NESTING_LEVEL) + movl PCPU(CURPROC),%ebx + incl P_INTR_NESTING_LEVEL(%ebx) sti movl PCPU(CPUID), %eax @@ -316,6 +321,7 @@ _Xcpuast: 2: lock incl CNAME(cpuast_cnt) + decl P_INTR_NESTING_LEVEL(%ebx) MEXITCOUNT jmp _doreti 1: @@ -323,6 +329,7 @@ _Xcpuast: POP_FRAME iret +#if 0 /* * Executed by a CPU when it receives an XFORWARD_IRQ IPI. @@ -360,7 +367,6 @@ _Xforward_irq: POP_FRAME iret -#if 0 /* * */ diff --git a/sys/i386/isa/atpic_vector.s b/sys/i386/isa/atpic_vector.s index 635d2ad..0a3456f 100644 --- a/sys/i386/isa/atpic_vector.s +++ b/sys/i386/isa/atpic_vector.s @@ -61,7 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -69,6 +70,7 @@ IDTVEC(vec_name) ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -102,13 +104,15 @@ IDTVEC(vec_name) ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ sti ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* _doreti, but it's probably better to use less cache. */ \ diff --git a/sys/i386/isa/icu_vector.s b/sys/i386/isa/icu_vector.s index 635d2ad..0a3456f 100644 --- a/sys/i386/isa/icu_vector.s +++ b/sys/i386/isa/icu_vector.s @@ -61,7 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -69,6 +70,7 @@ IDTVEC(vec_name) ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp _doreti @@ -102,13 +104,15 @@ IDTVEC(vec_name) ; \ movb %al,_imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - incb PCPU(INTR_NESTING_LEVEL) ; \ + movl PCPU(CURPROC),%ebx ; \ + incl P_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ sti ; \ call _sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ + decl P_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* _doreti, but it's probably better to use less cache. */ \ diff --git a/sys/i386/isa/intr_machdep.c b/sys/i386/isa/intr_machdep.c index 00f5f23..b9328ae 100644 --- a/sys/i386/isa/intr_machdep.c +++ b/sys/i386/isa/intr_machdep.c @@ -577,6 +577,7 @@ inthand_add(const char *name, int irq, driver_intr_t handler, void *arg, if (errcode) panic("inthand_add: Can't create " "interrupt thread"); + p->p_intr_nesting_level = 1; p->p_rtprio.type = RTP_PRIO_ITHREAD; p->p_stat = SWAIT; /* we're idle */ diff --git a/sys/i386/isa/ipl.s b/sys/i386/isa/ipl.s index ca874ea..7c41589 100644 --- a/sys/i386/isa/ipl.s +++ b/sys/i386/isa/ipl.s @@ -57,8 +57,6 @@ _doreti: FAKE_MCOUNT(_bintr) /* init "from" _bintr -> _doreti */ doreti_next: - decb PCPU(INTR_NESTING_LEVEL) - /* Check for ASTs that can be handled now. */ testl $AST_PENDING,PCPU(ASTPENDING) je doreti_exit /* no AST, exit */ @@ -128,7 +126,6 @@ doreti_ast: sti movl $T_ASTFLT,TF_TRAPNO(%esp) call _ast - movb $1,PCPU(INTR_NESTING_LEVEL) /* for doreti_next to decrement */ jmp doreti_next #ifdef APIC_IO diff --git a/sys/i386/isa/nmi.c b/sys/i386/isa/nmi.c index 00f5f23..b9328ae 100644 --- a/sys/i386/isa/nmi.c +++ b/sys/i386/isa/nmi.c @@ -577,6 +577,7 @@ inthand_add(const char *name, int irq, driver_intr_t handler, void *arg, if (errcode) panic("inthand_add: Can't create " "interrupt thread"); + p->p_intr_nesting_level = 1; p->p_rtprio.type = RTP_PRIO_ITHREAD; p->p_stat = SWAIT; /* we're idle */ -- cgit v1.1