diff options
author | fsmp <fsmp@FreeBSD.org> | 1997-12-08 22:59:39 +0000 |
---|---|---|
committer | fsmp <fsmp@FreeBSD.org> | 1997-12-08 22:59:39 +0000 |
commit | c29fe145047eadea9dcee01d907b3a990906a6af (patch) | |
tree | c4ae29ea386990372267c53b25ce4c0aba947b0e | |
parent | f950ecfd861dc1e3b8cf62d9aadd25436df2186d (diff) | |
download | FreeBSD-src-c29fe145047eadea9dcee01d907b3a990906a6af.zip FreeBSD-src-c29fe145047eadea9dcee01d907b3a990906a6af.tar.gz |
The improvements to clock statistics by Tor Egge
Wrappered and enabled by the define BETTER_CLOCK (on by default in smpyests.h)
apic_vector.s also contains a small change I (smp) made to eliminate
the double level INT problem. It seems stable, but I haven't the tools
in place to prove it fixes the problem.
Reviewed by: smp@csn.net
Submitted by: Tor Egge <Tor.Egge@idi.ntnu.no>
-rw-r--r-- | sys/amd64/amd64/apic_vector.S | 150 | ||||
-rw-r--r-- | sys/amd64/isa/intr_machdep.h | 12 | ||||
-rw-r--r-- | sys/i386/i386/apic_vector.s | 150 | ||||
-rw-r--r-- | sys/i386/isa/apic_vector.s | 150 | ||||
-rw-r--r-- | sys/i386/isa/intr_machdep.h | 12 |
5 files changed, 460 insertions, 14 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index 1968142..97832f2 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.22 1997/09/21 21:40:53 gibbs Exp $ + * $Id: apic_vector.s,v 1.38 1997/12/04 19:46:26 smp Exp smp $ */ @@ -190,7 +190,12 @@ IDTVEC(vec_name) ; \ #define UNMASK_IRQ(irq_num) \ IMASK_LOCK ; /* into critical reg */ \ testl $IRQ_BIT(irq_num), _apic_imen ; \ - je 9f ; \ + jne 7f ; /* bit set, masked */ \ + testl $IRQ_BIT(irq_num), _apic_pin_trigger ; \ + jz 9f ; /* edge, don't EOI */ \ + movl $0, lapic_eoi ; /* should be safe */ \ + jmp 9f ; /* skip unmasking */ \ +7: \ andl $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */ \ movl _ioapic,%ecx ; /* ioapic[0]addr */ \ movl $REDTBL_IDX(irq_num),(%ecx) ; /* write the index */ \ @@ -239,7 +244,7 @@ IDTVEC(vec_name) ; \ orl $IRQ_BIT(irq_num), _cil ; \ AVCPL_UNLOCK ; \ ; \ - movl $0, lapic_eoi ; /* XXX too soon? */ \ +;;; movl $0, lapic_eoi ; /* XXX too soon? */ \ incb _intr_nesting_level ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ @@ -314,7 +319,7 @@ IDTVEC(vec_name) ; \ jne 2f ; /* this INT masked */ \ AVCPL_UNLOCK ; \ ; \ - movl $0, lapic_eoi ; /* XXX too soon? */ \ +;;; movl $0, lapic_eoi ; /* XXX too soon? */ \ incb _intr_nesting_level ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ @@ -406,6 +411,132 @@ _Xinvltlb: iret +#ifdef BETTER_CLOCK + +/* + * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU, + * + * - Stores current cpu state in checkstate_cpustate[cpuid] + * 0 == user, 1 == sys, 2 == intr + * - Stores current process in checkstate_curproc[cpuid] + * + * - Signals its receipt by setting bit cpuid in checkstate_probed_cpus. + * + * stack: 0 -> ds, 4 -> ebx, 8 -> eax, 12 -> eip, 16 -> cs, 20 -> eflags + */ + + .text + SUPERALIGN_TEXT + .globl _Xcpucheckstate + .globl _checkstate_cpustate + .globl _checkstate_curproc + .globl _checkstate_pc +_Xcpucheckstate: + pushl %eax + pushl %ebx + pushl %ds /* save current data segment */ + + movl $KDSEL, %eax + movl %ax, %ds /* use KERNEL data segment */ + + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + + movl $0, %ebx + movl 16(%esp), %eax + andl $3, %eax + cmpl $3, %eax + je 1f +#ifdef VM86 + testl $PSL_VM, 20(%esp) + jne 1f +#endif + incl %ebx /* system or interrupt */ +#ifdef CPL_AND_CML + cmpl $0, _inside_intr + je 1f + incl %ebx /* interrupt */ +#endif +1: + movl _cpuid, %eax + movl %ebx, _checkstate_cpustate(,%eax,4) + movl _curproc, %ebx + movl %ebx, _checkstate_curproc(,%eax,4) + movl 12(%esp), %ebx + movl %ebx, _checkstate_pc(,%eax,4) + + lock /* checkstate_probed_cpus |= (1<<id) */ + btsl %eax, _checkstate_probed_cpus + + popl %ds /* restore previous data segment */ + popl %ebx + popl %eax + iret + +/* + * Executed by a CPU when it receives an Xcpuast IPI from another CPU, + * + * - Signals its receipt by clearing bit cpuid in checkstate_need_ast. + * + * - We need a better method of triggering asts on other cpus. + */ + + .text + SUPERALIGN_TEXT + .globl _Xcpuast +_Xcpuast: + PUSH_FRAME + movl $KDSEL, %eax + movl %ax, %ds /* use KERNEL data segment */ + movl %ax, %es + + movl _cpuid, %eax + lock /* checkstate_need_ast &= ~(1<<id) */ + btrl %eax, _checkstate_need_ast + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + + lock + btsl %eax, _checkstate_pending_ast + jc 1f + + FAKE_MCOUNT(12*4(%esp)) + + /* + * Giant locks do not come cheap. + * A lot of cycles are going to be wasted here. + */ + call _get_isrlock + + AVCPL_LOCK +#ifdef CPL_AND_CML + movl _cml, %eax +#else + movl _cpl, %eax +#endif + pushl %eax + AVCPL_UNLOCK + lock + incb _intr_nesting_level + sti + + pushl $0 + + lock + orl $SWI_AST_PENDING, _ipending + + movl _cpuid, %eax + lock + btrl %eax, _checkstate_pending_ast + + MEXITCOUNT + jmp _doreti +1: + /* We are already in the process of delivering an ast for this CPU */ + POP_FRAME + iret + +#endif /* BETTER_CLOCK */ + + /* * Executed by a CPU when it receives an Xcpustop IPI from another CPU, * @@ -561,6 +692,17 @@ _stopped_cpus: _started_cpus: .long 0 +#ifdef BETTER_CLOCK + .globl _checkstate_probed_cpus + .globl _checkstate_need_ast +_checkstate_probed_cpus: + .long 0 +_checkstate_need_ast: + .long 0 +_checkstate_pending_ast: + .long 0 +#endif + .globl _apic_pin_trigger _apic_pin_trigger: .space (NAPIC * 4), 0 diff --git a/sys/amd64/isa/intr_machdep.h b/sys/amd64/isa/intr_machdep.h index 6df8f3a..81a875f 100644 --- a/sys/amd64/isa/intr_machdep.h +++ b/sys/amd64/isa/intr_machdep.h @@ -31,7 +31,7 @@ * SUCH DAMAGE. * * from: @(#)isa_device.h 7.1 (Berkeley) 5/9/91 - * $Id: intr_machdep.h,v 1.10 1997/08/29 18:37:23 smp Exp smp $ + * $Id: intr_machdep.h,v 1.7 1997/08/29 18:45:21 fsmp Exp $ */ #ifndef _I386_ISA_INTR_MACHDEP_H_ @@ -107,6 +107,12 @@ /* TLB shootdowns */ #define XINVLTLB_OFFSET (ICU_OFFSET + 112) +#ifdef BETTER_CLOCK +/* inter-cpu clock handling */ +#define XCPUCHECKSTATE_OFFSET (ICU_OFFSET + 113) +#define XCPUAST_OFFSET (ICU_OFFSET + 114) +#endif + /* IPI to signal CPUs to stop and wait for another CPU to restart them */ #define XCPUSTOP_OFFSET (ICU_OFFSET + 128) @@ -160,6 +166,10 @@ inthand_t inthand_t Xinvltlb, /* TLB shootdowns */ +#ifdef BETTER_CLOCK + Xcpucheckstate, /* Check cpu state */ + Xcpuast, /* Additional software trap on other cpu */ +#endif Xcpustop, /* CPU stops & waits for another CPU to restart it */ Xspuriousint; /* handle APIC "spurious INTs" */ diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s index 1968142..97832f2 100644 --- a/sys/i386/i386/apic_vector.s +++ b/sys/i386/i386/apic_vector.s @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.22 1997/09/21 21:40:53 gibbs Exp $ + * $Id: apic_vector.s,v 1.38 1997/12/04 19:46:26 smp Exp smp $ */ @@ -190,7 +190,12 @@ IDTVEC(vec_name) ; \ #define UNMASK_IRQ(irq_num) \ IMASK_LOCK ; /* into critical reg */ \ testl $IRQ_BIT(irq_num), _apic_imen ; \ - je 9f ; \ + jne 7f ; /* bit set, masked */ \ + testl $IRQ_BIT(irq_num), _apic_pin_trigger ; \ + jz 9f ; /* edge, don't EOI */ \ + movl $0, lapic_eoi ; /* should be safe */ \ + jmp 9f ; /* skip unmasking */ \ +7: \ andl $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */ \ movl _ioapic,%ecx ; /* ioapic[0]addr */ \ movl $REDTBL_IDX(irq_num),(%ecx) ; /* write the index */ \ @@ -239,7 +244,7 @@ IDTVEC(vec_name) ; \ orl $IRQ_BIT(irq_num), _cil ; \ AVCPL_UNLOCK ; \ ; \ - movl $0, lapic_eoi ; /* XXX too soon? */ \ +;;; movl $0, lapic_eoi ; /* XXX too soon? */ \ incb _intr_nesting_level ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ @@ -314,7 +319,7 @@ IDTVEC(vec_name) ; \ jne 2f ; /* this INT masked */ \ AVCPL_UNLOCK ; \ ; \ - movl $0, lapic_eoi ; /* XXX too soon? */ \ +;;; movl $0, lapic_eoi ; /* XXX too soon? */ \ incb _intr_nesting_level ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ @@ -406,6 +411,132 @@ _Xinvltlb: iret +#ifdef BETTER_CLOCK + +/* + * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU, + * + * - Stores current cpu state in checkstate_cpustate[cpuid] + * 0 == user, 1 == sys, 2 == intr + * - Stores current process in checkstate_curproc[cpuid] + * + * - Signals its receipt by setting bit cpuid in checkstate_probed_cpus. + * + * stack: 0 -> ds, 4 -> ebx, 8 -> eax, 12 -> eip, 16 -> cs, 20 -> eflags + */ + + .text + SUPERALIGN_TEXT + .globl _Xcpucheckstate + .globl _checkstate_cpustate + .globl _checkstate_curproc + .globl _checkstate_pc +_Xcpucheckstate: + pushl %eax + pushl %ebx + pushl %ds /* save current data segment */ + + movl $KDSEL, %eax + movl %ax, %ds /* use KERNEL data segment */ + + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + + movl $0, %ebx + movl 16(%esp), %eax + andl $3, %eax + cmpl $3, %eax + je 1f +#ifdef VM86 + testl $PSL_VM, 20(%esp) + jne 1f +#endif + incl %ebx /* system or interrupt */ +#ifdef CPL_AND_CML + cmpl $0, _inside_intr + je 1f + incl %ebx /* interrupt */ +#endif +1: + movl _cpuid, %eax + movl %ebx, _checkstate_cpustate(,%eax,4) + movl _curproc, %ebx + movl %ebx, _checkstate_curproc(,%eax,4) + movl 12(%esp), %ebx + movl %ebx, _checkstate_pc(,%eax,4) + + lock /* checkstate_probed_cpus |= (1<<id) */ + btsl %eax, _checkstate_probed_cpus + + popl %ds /* restore previous data segment */ + popl %ebx + popl %eax + iret + +/* + * Executed by a CPU when it receives an Xcpuast IPI from another CPU, + * + * - Signals its receipt by clearing bit cpuid in checkstate_need_ast. + * + * - We need a better method of triggering asts on other cpus. + */ + + .text + SUPERALIGN_TEXT + .globl _Xcpuast +_Xcpuast: + PUSH_FRAME + movl $KDSEL, %eax + movl %ax, %ds /* use KERNEL data segment */ + movl %ax, %es + + movl _cpuid, %eax + lock /* checkstate_need_ast &= ~(1<<id) */ + btrl %eax, _checkstate_need_ast + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + + lock + btsl %eax, _checkstate_pending_ast + jc 1f + + FAKE_MCOUNT(12*4(%esp)) + + /* + * Giant locks do not come cheap. + * A lot of cycles are going to be wasted here. + */ + call _get_isrlock + + AVCPL_LOCK +#ifdef CPL_AND_CML + movl _cml, %eax +#else + movl _cpl, %eax +#endif + pushl %eax + AVCPL_UNLOCK + lock + incb _intr_nesting_level + sti + + pushl $0 + + lock + orl $SWI_AST_PENDING, _ipending + + movl _cpuid, %eax + lock + btrl %eax, _checkstate_pending_ast + + MEXITCOUNT + jmp _doreti +1: + /* We are already in the process of delivering an ast for this CPU */ + POP_FRAME + iret + +#endif /* BETTER_CLOCK */ + + /* * Executed by a CPU when it receives an Xcpustop IPI from another CPU, * @@ -561,6 +692,17 @@ _stopped_cpus: _started_cpus: .long 0 +#ifdef BETTER_CLOCK + .globl _checkstate_probed_cpus + .globl _checkstate_need_ast +_checkstate_probed_cpus: + .long 0 +_checkstate_need_ast: + .long 0 +_checkstate_pending_ast: + .long 0 +#endif + .globl _apic_pin_trigger _apic_pin_trigger: .space (NAPIC * 4), 0 diff --git a/sys/i386/isa/apic_vector.s b/sys/i386/isa/apic_vector.s index 1968142..97832f2 100644 --- a/sys/i386/isa/apic_vector.s +++ b/sys/i386/isa/apic_vector.s @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.22 1997/09/21 21:40:53 gibbs Exp $ + * $Id: apic_vector.s,v 1.38 1997/12/04 19:46:26 smp Exp smp $ */ @@ -190,7 +190,12 @@ IDTVEC(vec_name) ; \ #define UNMASK_IRQ(irq_num) \ IMASK_LOCK ; /* into critical reg */ \ testl $IRQ_BIT(irq_num), _apic_imen ; \ - je 9f ; \ + jne 7f ; /* bit set, masked */ \ + testl $IRQ_BIT(irq_num), _apic_pin_trigger ; \ + jz 9f ; /* edge, don't EOI */ \ + movl $0, lapic_eoi ; /* should be safe */ \ + jmp 9f ; /* skip unmasking */ \ +7: \ andl $~IRQ_BIT(irq_num), _apic_imen ;/* clear mask bit */ \ movl _ioapic,%ecx ; /* ioapic[0]addr */ \ movl $REDTBL_IDX(irq_num),(%ecx) ; /* write the index */ \ @@ -239,7 +244,7 @@ IDTVEC(vec_name) ; \ orl $IRQ_BIT(irq_num), _cil ; \ AVCPL_UNLOCK ; \ ; \ - movl $0, lapic_eoi ; /* XXX too soon? */ \ +;;; movl $0, lapic_eoi ; /* XXX too soon? */ \ incb _intr_nesting_level ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ @@ -314,7 +319,7 @@ IDTVEC(vec_name) ; \ jne 2f ; /* this INT masked */ \ AVCPL_UNLOCK ; \ ; \ - movl $0, lapic_eoi ; /* XXX too soon? */ \ +;;; movl $0, lapic_eoi ; /* XXX too soon? */ \ incb _intr_nesting_level ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ @@ -406,6 +411,132 @@ _Xinvltlb: iret +#ifdef BETTER_CLOCK + +/* + * Executed by a CPU when it receives an Xcpucheckstate IPI from another CPU, + * + * - Stores current cpu state in checkstate_cpustate[cpuid] + * 0 == user, 1 == sys, 2 == intr + * - Stores current process in checkstate_curproc[cpuid] + * + * - Signals its receipt by setting bit cpuid in checkstate_probed_cpus. + * + * stack: 0 -> ds, 4 -> ebx, 8 -> eax, 12 -> eip, 16 -> cs, 20 -> eflags + */ + + .text + SUPERALIGN_TEXT + .globl _Xcpucheckstate + .globl _checkstate_cpustate + .globl _checkstate_curproc + .globl _checkstate_pc +_Xcpucheckstate: + pushl %eax + pushl %ebx + pushl %ds /* save current data segment */ + + movl $KDSEL, %eax + movl %ax, %ds /* use KERNEL data segment */ + + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + + movl $0, %ebx + movl 16(%esp), %eax + andl $3, %eax + cmpl $3, %eax + je 1f +#ifdef VM86 + testl $PSL_VM, 20(%esp) + jne 1f +#endif + incl %ebx /* system or interrupt */ +#ifdef CPL_AND_CML + cmpl $0, _inside_intr + je 1f + incl %ebx /* interrupt */ +#endif +1: + movl _cpuid, %eax + movl %ebx, _checkstate_cpustate(,%eax,4) + movl _curproc, %ebx + movl %ebx, _checkstate_curproc(,%eax,4) + movl 12(%esp), %ebx + movl %ebx, _checkstate_pc(,%eax,4) + + lock /* checkstate_probed_cpus |= (1<<id) */ + btsl %eax, _checkstate_probed_cpus + + popl %ds /* restore previous data segment */ + popl %ebx + popl %eax + iret + +/* + * Executed by a CPU when it receives an Xcpuast IPI from another CPU, + * + * - Signals its receipt by clearing bit cpuid in checkstate_need_ast. + * + * - We need a better method of triggering asts on other cpus. + */ + + .text + SUPERALIGN_TEXT + .globl _Xcpuast +_Xcpuast: + PUSH_FRAME + movl $KDSEL, %eax + movl %ax, %ds /* use KERNEL data segment */ + movl %ax, %es + + movl _cpuid, %eax + lock /* checkstate_need_ast &= ~(1<<id) */ + btrl %eax, _checkstate_need_ast + movl $0, lapic_eoi /* End Of Interrupt to APIC */ + + lock + btsl %eax, _checkstate_pending_ast + jc 1f + + FAKE_MCOUNT(12*4(%esp)) + + /* + * Giant locks do not come cheap. + * A lot of cycles are going to be wasted here. + */ + call _get_isrlock + + AVCPL_LOCK +#ifdef CPL_AND_CML + movl _cml, %eax +#else + movl _cpl, %eax +#endif + pushl %eax + AVCPL_UNLOCK + lock + incb _intr_nesting_level + sti + + pushl $0 + + lock + orl $SWI_AST_PENDING, _ipending + + movl _cpuid, %eax + lock + btrl %eax, _checkstate_pending_ast + + MEXITCOUNT + jmp _doreti +1: + /* We are already in the process of delivering an ast for this CPU */ + POP_FRAME + iret + +#endif /* BETTER_CLOCK */ + + /* * Executed by a CPU when it receives an Xcpustop IPI from another CPU, * @@ -561,6 +692,17 @@ _stopped_cpus: _started_cpus: .long 0 +#ifdef BETTER_CLOCK + .globl _checkstate_probed_cpus + .globl _checkstate_need_ast +_checkstate_probed_cpus: + .long 0 +_checkstate_need_ast: + .long 0 +_checkstate_pending_ast: + .long 0 +#endif + .globl _apic_pin_trigger _apic_pin_trigger: .space (NAPIC * 4), 0 diff --git a/sys/i386/isa/intr_machdep.h b/sys/i386/isa/intr_machdep.h index 6df8f3a..81a875f 100644 --- a/sys/i386/isa/intr_machdep.h +++ b/sys/i386/isa/intr_machdep.h @@ -31,7 +31,7 @@ * SUCH DAMAGE. * * from: @(#)isa_device.h 7.1 (Berkeley) 5/9/91 - * $Id: intr_machdep.h,v 1.10 1997/08/29 18:37:23 smp Exp smp $ + * $Id: intr_machdep.h,v 1.7 1997/08/29 18:45:21 fsmp Exp $ */ #ifndef _I386_ISA_INTR_MACHDEP_H_ @@ -107,6 +107,12 @@ /* TLB shootdowns */ #define XINVLTLB_OFFSET (ICU_OFFSET + 112) +#ifdef BETTER_CLOCK +/* inter-cpu clock handling */ +#define XCPUCHECKSTATE_OFFSET (ICU_OFFSET + 113) +#define XCPUAST_OFFSET (ICU_OFFSET + 114) +#endif + /* IPI to signal CPUs to stop and wait for another CPU to restart them */ #define XCPUSTOP_OFFSET (ICU_OFFSET + 128) @@ -160,6 +166,10 @@ inthand_t inthand_t Xinvltlb, /* TLB shootdowns */ +#ifdef BETTER_CLOCK + Xcpucheckstate, /* Check cpu state */ + Xcpuast, /* Additional software trap on other cpu */ +#endif Xcpustop, /* CPU stops & waits for another CPU to restart it */ Xspuriousint; /* handle APIC "spurious INTs" */ |