diff options
author | fsmp <fsmp@FreeBSD.org> | 1997-08-20 05:25:48 +0000 |
---|---|---|
committer | fsmp <fsmp@FreeBSD.org> | 1997-08-20 05:25:48 +0000 |
commit | 2c414e3eff799c18e5d3e8edfe37d7dc71f35c7a (patch) | |
tree | 4060f694285dcb8b0a5d572747820da61322b0df /sys/i386 | |
parent | ea72b8e976791b005527662091bc146ad0bfd3b6 (diff) | |
download | FreeBSD-src-2c414e3eff799c18e5d3e8edfe37d7dc71f35c7a.zip FreeBSD-src-2c414e3eff799c18e5d3e8edfe37d7dc71f35c7a.tar.gz |
Preperation for moving cpl into critical region access.
Several new fine-grained locks.
New FAST_INTR() methods:
- separate simplelock for FAST_INTR, no more giant lock.
- FAST_INTR()s no longer checks ipending on way out of ISR.
sio made MP-safe (I hope).
Diffstat (limited to 'sys/i386')
-rw-r--r-- | sys/i386/i386/apic_vector.s | 80 | ||||
-rw-r--r-- | sys/i386/i386/exception.s | 56 | ||||
-rw-r--r-- | sys/i386/i386/mp_machdep.c | 21 | ||||
-rw-r--r-- | sys/i386/i386/mplock.s | 118 | ||||
-rw-r--r-- | sys/i386/i386/mptable.c | 21 | ||||
-rw-r--r-- | sys/i386/i386/trap.c | 10 | ||||
-rw-r--r-- | sys/i386/include/mptable.h | 21 | ||||
-rw-r--r-- | sys/i386/isa/apic_ipl.s | 20 | ||||
-rw-r--r-- | sys/i386/isa/apic_vector.s | 80 | ||||
-rw-r--r-- | sys/i386/isa/ipl.s | 11 | ||||
-rw-r--r-- | sys/i386/isa/ipl_funcs.c | 12 | ||||
-rw-r--r-- | sys/i386/isa/sio.c | 49 |
12 files changed, 342 insertions, 157 deletions
diff --git a/sys/i386/i386/apic_vector.s b/sys/i386/i386/apic_vector.s index f73ddc5..f2a8bce 100644 --- a/sys/i386/i386/apic_vector.s +++ b/sys/i386/i386/apic_vector.s @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.21 1997/08/10 20:47:53 smp Exp smp $ + * $Id: apic_vector.s,v 1.15 1997/08/10 20:58:57 fsmp Exp $ */ @@ -11,12 +11,41 @@ #include "i386/isa/intr_machdep.h" +#ifdef FAST_SIMPLELOCK + +#define GET_FAST_INTR_LOCK \ + pushl $_fast_intr_lock ; /* address of lock */ \ + call _s_lock ; /* MP-safe */ \ + addl $4,%esp + +#define REL_FAST_INTR_LOCK \ + pushl $_fast_intr_lock ; /* address of lock */ \ + call _s_unlock ; /* MP-safe */ \ + addl $4,%esp + +#else /* FAST_SIMPLELOCK */ + +#define GET_FAST_INTR_LOCK \ + call _get_isrlock +#define REL_FAST_INTR_LOCK \ + pushl $_mp_lock ; /* GIANT_LOCK */ \ + call _MPrellock ; \ + add $4, %esp + +#endif /* FAST_SIMPLELOCK */ + +#define REL_ISR_LOCK \ + pushl $_mp_lock ; /* GIANT_LOCK */ \ + call _MPrellock ; \ + add $4, %esp + /* convert an absolute IRQ# into a bitmask */ #define IRQ_BIT(irq_num) (1 << (irq_num)) /* make an index into the IO APIC from the IRQ# */ #define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) + /* * 'lazy masking' code suggested by Bruce Evans <bde@zeta.org.au> */ @@ -115,6 +144,8 @@ * Macros for interrupt interrupt entry, call to handler, and exit. */ +#ifdef FAST_WITHOUTCPL + #define FAST_INTR(irq_num, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ @@ -128,11 +159,45 @@ IDTVEC(vec_name) ; \ movl %ax,%ds ; \ MAYBE_MOVW_AX_ES ; \ FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \ - call _get_isrlock ; \ + GET_FAST_INTR_LOCK ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ + addl $4, %esp ; \ movl $0, lapic_eoi ; \ + lock ; \ + incl _cnt+V_INTR ; /* book-keeping can wait */ \ + movl _intr_countp + (irq_num) * 4, %eax ; \ + lock ; \ + incl (%eax) ; \ + MEXITCOUNT ; \ + REL_FAST_INTR_LOCK ; \ + MAYBE_POPL_ES ; \ + popl %ds ; \ + popl %edx ; \ + popl %ecx ; \ + popl %eax ; \ + iret + +#else + +#define FAST_INTR(irq_num, vec_name) \ + .text ; \ + SUPERALIGN_TEXT ; \ +IDTVEC(vec_name) ; \ + pushl %eax ; /* save only call-used registers */ \ + pushl %ecx ; \ + pushl %edx ; \ + pushl %ds ; \ + MAYBE_PUSHL_ES ; \ + movl $KDSEL,%eax ; \ + movl %ax,%ds ; \ + MAYBE_MOVW_AX_ES ; \ + FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \ + GET_FAST_INTR_LOCK ; \ + pushl _intr_unit + (irq_num) * 4 ; \ + call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ addl $4,%esp ; \ + movl $0, lapic_eoi ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ @@ -142,9 +207,7 @@ IDTVEC(vec_name) ; \ jne 2f ; /* yes, maybe handle them */ \ 1: ; \ MEXITCOUNT ; \ - pushl $_mp_lock ; /* GIANT_LOCK */ \ - call _MPrellock ; \ - add $4, %esp ; \ + REL_FAST_INTR_LOCK ; \ MAYBE_POPL_ES ; \ popl %ds ; \ popl %edx ; \ @@ -178,6 +241,9 @@ IDTVEC(vec_name) ; \ MEXITCOUNT ; \ jmp _doreti +#endif /** FAST_WITHOUTCPL */ + + #define INTR(irq_num, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ @@ -217,9 +283,7 @@ __CONCAT(Xresume,irq_num): ; \ /* XXX skip mcounting here to avoid double count */ \ lock ; /* MP-safe */ \ orl $IRQ_BIT(irq_num), _ipending ; \ - pushl $_mp_lock ; /* GIANT_LOCK */ \ - call _MPrellock ; \ - add $4, %esp ; \ + REL_ISR_LOCK ; \ popl %es ; \ popl %ds ; \ popal ; \ diff --git a/sys/i386/i386/exception.s b/sys/i386/i386/exception.s index 6305724..09635fa 100644 --- a/sys/i386/i386/exception.s +++ b/sys/i386/i386/exception.s @@ -30,7 +30,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: exception.s,v 1.37 1997/08/10 20:59:06 fsmp Exp $ + * $Id: exception.s,v 1.38 1997/08/10 21:18:01 fsmp Exp $ */ #include "npx.h" /* NNPX */ @@ -42,8 +42,12 @@ #ifdef SMP -#define MP_INSTR_LOCK \ - lock /* MP-safe */ +#define MPLOCKED lock ; + +#define FPU_LOCK call _get_fpu_lock +#define ALIGN_LOCK call _get_align_lock +#define SYSCALL_LOCK call _get_syscall_lock +#define ALTSYSCALL_LOCK call _get_altsyscall_lock /* protects the IO APIC and apic_imen as a critical region */ #define IMASK_LOCK \ @@ -56,11 +60,25 @@ call _s_unlock ; /* MP-safe */ \ addl $4,%esp -#else +/* protects cpl updates as a critical region */ +#define CPL_LOCK \ + pushl $_cpl_lock ; /* address of lock */ \ + call _s_lock ; /* MP-safe */ \ + addl $4,%esp + +#define CPL_UNLOCK \ + pushl $_cpl_lock ; /* address of lock */ \ + call _s_unlock ; /* MP-safe */ \ + addl $4,%esp -#define MP_INSTR_LOCK /* NOP */ -#define IMASK_LOCK /* NOP */ -#define IMASK_UNLOCK /* NOP */ +#else /* SMP */ + +#define MPLOCKED /* NOP */ + +#define FPU_LOCK /* NOP */ +#define ALIGN_LOCK /* NOP */ +#define SYSCALL_LOCK /* NOP */ +#define ALTSYSCALL_LOCK /* NOP */ #endif /* SMP */ @@ -160,10 +178,8 @@ IDTVEC(fpu) movl _cpl,%eax pushl %eax pushl $0 /* dummy unit to finish intr frame */ -#ifdef SMP - call _get_fpu_lock -#endif /* SMP */ - incl _cnt+V_TRAP + MPLOCKED incl _cnt+V_TRAP + FPU_LOCK orl $SWI_AST_MASK,%eax movl %eax,_cpl call _npxintr @@ -188,11 +204,9 @@ alltraps_with_regs_pushed: movl %ax,%es FAKE_MCOUNT(12*4(%esp)) calltrap: -#ifdef SMP - call _get_align_lock -#endif /* SMP */ + ALIGN_LOCK FAKE_MCOUNT(_btrap) /* init "from" _btrap -> calltrap */ - incl _cnt+V_TRAP + MPLOCKED incl _cnt+V_TRAP orl $SWI_AST_MASK,_cpl call _trap @@ -244,10 +258,8 @@ IDTVEC(syscall) movl %eax,TF_EFLAGS(%esp) movl $7,TF_ERR(%esp) /* sizeof "lcall 7,0" */ FAKE_MCOUNT(12*4(%esp)) -#ifdef SMP - call _get_syscall_lock -#endif /* SMP */ - incl _cnt+V_SYSCALL + SYSCALL_LOCK + MPLOCKED incl _cnt+V_SYSCALL movl $SWI_AST_MASK,_cpl call _syscall /* @@ -273,10 +285,8 @@ IDTVEC(int0x80_syscall) movl %ax,%es movl $2,TF_ERR(%esp) /* sizeof "int 0x80" */ FAKE_MCOUNT(12*4(%esp)) -#ifdef SMP - call _get_int0x80_syscall_lock -#endif /* SMP */ - incl _cnt+V_SYSCALL + ALTSYSCALL_LOCK + MPLOCKED incl _cnt+V_SYSCALL movl $SWI_AST_MASK,_cpl call _syscall /* diff --git a/sys/i386/i386/mp_machdep.c b/sys/i386/i386/mp_machdep.c index ce107d9..e5fb310 100644 --- a/sys/i386/i386/mp_machdep.c +++ b/sys/i386/i386/mp_machdep.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.40 1997/08/09 23:01:03 fsmp Exp $ + * $Id: mp_machdep.c,v 1.41 1997/08/10 19:32:38 fsmp Exp $ */ #include "opt_smp.h" @@ -1418,6 +1418,10 @@ default_mp_table(int type) /* * initialize all the SMP locks */ + +/* lock the com (tty) data structures */ +struct simplelock com_lock; + static void init_locks(void) { @@ -1427,8 +1431,23 @@ init_locks(void) */ mp_lock = 0x00000001; + /* ISR uses its own "giant lock" */ + isr_lock = 0x00000000; + + /* serializes FAST_INTR() accesses */ + s_lock_init((struct simplelock*)&fast_intr_lock); + + /* serializes INTR() accesses */ + s_lock_init((struct simplelock*)&intr_lock); + /* locks the IO APIC and apic_imen accesses */ s_lock_init((struct simplelock*)&imen_lock); + + /* locks cpl accesses */ + s_lock_init((struct simplelock*)&cpl_lock); + + /* locks com (tty) data/hardware accesses: a FASTINTR() */ + s_lock_init((struct simplelock*)&com_lock); } diff --git a/sys/i386/i386/mplock.s b/sys/i386/i386/mplock.s index 3a73ee5..7eb2113 100644 --- a/sys/i386/i386/mplock.s +++ b/sys/i386/i386/mplock.s @@ -6,7 +6,7 @@ * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp * ---------------------------------------------------------------------------- * - * $Id: mplock.s,v 1.15 1997/08/10 20:51:52 smp Exp smp $ + * $Id: mplock.s,v 1.17 1997/08/10 20:59:07 fsmp Exp $ * * Functions for locking between CPUs in a SMP system. * @@ -27,17 +27,7 @@ #include <i386/isa/intr_machdep.h> -/* - * GLPROFILE showed that the lock was found to be free approx. 16 times - * more often that it was found to be already owned. FREE_FIRST forces - * the code to look for a free lock before looking for an owned lock. - - owned: free: fail: -_gethits: 118de 1c7dd3 3f106 -_tryhits: 9938 2196d 44cc - */ -#define FREE_FIRST #define GLPROFILE #ifdef CHEAP_TPR @@ -54,7 +44,7 @@ _tryhits: 9938 2196d 44cc * silicon seems happy with just writing. If the behaviour of the * silicon changes, all code that access the lapic_tpr must be modified. * The last version to contain such code was: - * $Id: mplock.s,v 1.15 1997/08/10 20:51:52 smp Exp smp $ + * Id: mplock.s,v 1.17 1997/08/10 20:59:07 fsmp Exp */ #endif /* CHEAP_TPR */ @@ -65,7 +55,7 @@ _tryhits: 9938 2196d 44cc */ /* location of saved TPR on stack */ -#define TPR_TARGET 12(%esp) +#define TPR_TARGET 12(%esp) /* after 1st acquire of lock we attempt to grab all hardware INTs */ #define GRAB_HWI movl $ALLHWI_LEVEL, TPR_TARGET @@ -92,9 +82,6 @@ _tryhits: 9938 2196d 44cc NON_GPROF_ENTRY(MPgetlock) movl 4(%esp), %edx /* Get the address of the lock */ - -#ifdef FREE_FIRST - 1: movl $FREE_LOCK, %eax /* Assume it's free */ movl _cpu_lockid, %ecx /* - get pre-shifted logical cpu id */ @@ -130,48 +117,7 @@ NON_GPROF_ENTRY(MPgetlock) 3: cmpl $FREE_LOCK, (%edx) /* Wait for it to become free */ jne 3b - jmp 1b /* XXX 1b ? */ - -#else /* FREE_FIRST */ - -1: - movl (%edx), %eax /* Try to see if we have it already */ - andl $COUNT_FIELD, %eax /* - get count */ - movl _cpu_lockid, %ecx /* - get pre-shifted logical cpu id */ - orl %ecx, %eax /* - combine them */ - movl %eax, %ecx - incl %ecx /* - new count is one more */ - lock - cmpxchg %ecx, (%edx) /* - try it atomically */ - jne 2f /* - miss */ -#ifdef GLPROFILE - incl _gethits -#endif /* GLPROFILE */ - ret -2: - movl $FREE_LOCK, %eax /* Assume it's free */ - movl _cpu_lockid, %ecx /* - get pre-shifted logical cpu id */ - incl %ecx /* - new count is one */ - lock - cmpxchg %ecx, (%edx) /* - try it atomically */ -#ifdef GLPROFILE - jne 4f /* ...do not collect $200 */ - incl _gethits2 -#else - jne 3f /* ...do not collect $200 */ -#endif /* GLPROFILE */ - GRAB_HWI /* 1st acquire, grab hw INTs */ - ret -#ifdef GLPROFILE -4: - incl _gethits3 -#endif /* GLPROFILE */ -3: - cmpl $FREE_LOCK, (%edx) /* Wait for it to become free */ - jne 3b - jmp 2b /* XXX 1b ? */ - -#endif /* FREE_FIRST */ + jmp 1b /*********************************************************************** @@ -184,8 +130,6 @@ NON_GPROF_ENTRY(MPgetlock) NON_GPROF_ENTRY(MPtrylock) movl 4(%esp), %edx /* Get the address of the lock */ -#ifdef FREE_FIRST - movl $FREE_LOCK, %eax /* Assume it's free */ movl _cpu_lockid, %ecx /* - get pre-shifted logical cpu id */ incl %ecx /* - new count is one */ @@ -220,44 +164,6 @@ NON_GPROF_ENTRY(MPtrylock) movl $0, %eax ret -#else /* FREE_FIRST */ - - movl (%edx), %eax /* Try to see if we have it already */ - andl $COUNT_FIELD, %eax /* - get count */ - movl _cpu_lockid, %ecx /* - get pre-shifted logical cpu id */ - orl %ecx, %eax /* - combine them */ - movl %eax, %ecx - incl %ecx /* - new count is one more */ - lock - cmpxchg %ecx, (%edx) /* - try it atomically */ - jne 1f /* - miss */ -#ifdef GLPROFILE - incl _tryhits -#endif /* GLPROFILE */ - movl $1, %eax - ret -1: - movl $FREE_LOCK, %eax /* Assume it's free */ - movl _cpu_lockid, %ecx /* - get pre-shifted logical cpu id */ - incl %ecx /* - new count is one */ - lock - cmpxchg %ecx, (%edx) /* - try it atomically */ - jne 2f /* ...do not collect $200 */ -#ifdef GLPROFILE - incl _tryhits2 -#endif /* GLPROFILE */ - GRAB_HWI_2 /* 1st acquire, grab hw INTs */ - movl $1, %eax - ret -2: -#ifdef GLPROFILE - incl _tryhits3 -#endif /* GLPROFILE */ - movl $0, %eax - ret - -#endif /* FREE_FIRST */ - /*********************************************************************** * void MPrellock(unsigned int *lock) @@ -505,10 +411,10 @@ NON_GPROF_ENTRY(rel_syscall_lock) /*********************************************************************** - * int0x80_syscall locks + * altsyscall locks */ -NON_GPROF_ENTRY(get_int0x80_syscall_lock) +NON_GPROF_ENTRY(get_altsyscall_lock) pushl lapic_tpr pushfl movl $TPR_BLOCK_HWI, lapic_tpr /* CHEAP_TPR */ @@ -521,13 +427,13 @@ NON_GPROF_ENTRY(get_int0x80_syscall_lock) ret #ifdef notneeded -NON_GPROF_ENTRY(try_int0x80_syscall_lock) +NON_GPROF_ENTRY(try_altsyscall_lock) pushl $_mp_lock call _MPtrylock add $4, %esp ret -NON_GPROF_ENTRY(rel_int0x80_syscall_lock) +NON_GPROF_ENTRY(rel_altsyscall_lock) pushl $_mp_lock call _MPrellock add $4, %esp @@ -538,12 +444,16 @@ NON_GPROF_ENTRY(rel_int0x80_syscall_lock) /*********************************************************************** * */ - .data + .align 2 /* xx_lock aligned on int boundary */ + .globl _mp_lock - .align 2 /* mp_lock aligned on int boundary */ _mp_lock: .long 0 + .globl _isr_lock +_isr_lock: .long 0 + + #ifdef GLPROFILE .globl _gethits _gethits: diff --git a/sys/i386/i386/mptable.c b/sys/i386/i386/mptable.c index ce107d9..e5fb310 100644 --- a/sys/i386/i386/mptable.c +++ b/sys/i386/i386/mptable.c @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.40 1997/08/09 23:01:03 fsmp Exp $ + * $Id: mp_machdep.c,v 1.41 1997/08/10 19:32:38 fsmp Exp $ */ #include "opt_smp.h" @@ -1418,6 +1418,10 @@ default_mp_table(int type) /* * initialize all the SMP locks */ + +/* lock the com (tty) data structures */ +struct simplelock com_lock; + static void init_locks(void) { @@ -1427,8 +1431,23 @@ init_locks(void) */ mp_lock = 0x00000001; + /* ISR uses its own "giant lock" */ + isr_lock = 0x00000000; + + /* serializes FAST_INTR() accesses */ + s_lock_init((struct simplelock*)&fast_intr_lock); + + /* serializes INTR() accesses */ + s_lock_init((struct simplelock*)&intr_lock); + /* locks the IO APIC and apic_imen accesses */ s_lock_init((struct simplelock*)&imen_lock); + + /* locks cpl accesses */ + s_lock_init((struct simplelock*)&cpl_lock); + + /* locks com (tty) data/hardware accesses: a FASTINTR() */ + s_lock_init((struct simplelock*)&com_lock); } diff --git a/sys/i386/i386/trap.c b/sys/i386/i386/trap.c index 72df964..de901c0 100644 --- a/sys/i386/i386/trap.c +++ b/sys/i386/i386/trap.c @@ -35,7 +35,7 @@ * SUCH DAMAGE. * * from: @(#)trap.c 7.4 (Berkeley) 5/13/91 - * $Id: trap.c,v 1.104 1997/08/12 19:07:42 dyson Exp $ + * $Id: trap.c,v 1.1 1997/08/19 20:26:01 smp Exp smp $ */ /* @@ -778,6 +778,14 @@ trap_fatal(frame) printf("bio "); if (cpl == 0) printf("none"); +#ifdef SMP +/** + * XXX FIXME: + * we probably SHOULD have stopped the other CPUs before now! + * another CPU COULD have been touching cpl at this moment... + */ + printf(" <- SMP: XXX"); +#endif printf("\n"); #ifdef KDB diff --git a/sys/i386/include/mptable.h b/sys/i386/include/mptable.h index ce107d9..e5fb310 100644 --- a/sys/i386/include/mptable.h +++ b/sys/i386/include/mptable.h @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: mp_machdep.c,v 1.40 1997/08/09 23:01:03 fsmp Exp $ + * $Id: mp_machdep.c,v 1.41 1997/08/10 19:32:38 fsmp Exp $ */ #include "opt_smp.h" @@ -1418,6 +1418,10 @@ default_mp_table(int type) /* * initialize all the SMP locks */ + +/* lock the com (tty) data structures */ +struct simplelock com_lock; + static void init_locks(void) { @@ -1427,8 +1431,23 @@ init_locks(void) */ mp_lock = 0x00000001; + /* ISR uses its own "giant lock" */ + isr_lock = 0x00000000; + + /* serializes FAST_INTR() accesses */ + s_lock_init((struct simplelock*)&fast_intr_lock); + + /* serializes INTR() accesses */ + s_lock_init((struct simplelock*)&intr_lock); + /* locks the IO APIC and apic_imen accesses */ s_lock_init((struct simplelock*)&imen_lock); + + /* locks cpl accesses */ + s_lock_init((struct simplelock*)&cpl_lock); + + /* locks com (tty) data/hardware accesses: a FASTINTR() */ + s_lock_init((struct simplelock*)&com_lock); } diff --git a/sys/i386/isa/apic_ipl.s b/sys/i386/isa/apic_ipl.s index b6f34e1..1412367 100644 --- a/sys/i386/isa/apic_ipl.s +++ b/sys/i386/isa/apic_ipl.s @@ -22,7 +22,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: apic_ipl.s,v 1.10 1997/07/24 23:49:44 fsmp Exp $ + * $Id: apic_ipl.s,v 1.11 1997/07/31 17:28:56 fsmp Exp $ */ @@ -66,11 +66,29 @@ _vec: .long vec8, vec9, vec10, vec11, vec12, vec13, vec14, vec15 .long vec16, vec17, vec18, vec19, vec20, vec21, vec22, vec23 +/* various simple locks */ .align 2 /* MUST be 32bit aligned */ + +/* critical region around IO APIC */ .globl _imen_lock _imen_lock: .long 0 +/* critical region around spl & cpl */ + .globl _cpl_lock +_cpl_lock: + .long 0 + +/* critical region around FAST_INTR() routines */ + .globl _fast_intr_lock +_fast_intr_lock: + .long 0 + +/* critical region around INTR() routines */ + .globl _intr_lock +_intr_lock: + .long 0 + /* * Note: * This is the UP equivilant of _imen. diff --git a/sys/i386/isa/apic_vector.s b/sys/i386/isa/apic_vector.s index f73ddc5..f2a8bce 100644 --- a/sys/i386/isa/apic_vector.s +++ b/sys/i386/isa/apic_vector.s @@ -1,6 +1,6 @@ /* * from: vector.s, 386BSD 0.1 unknown origin - * $Id: apic_vector.s,v 1.21 1997/08/10 20:47:53 smp Exp smp $ + * $Id: apic_vector.s,v 1.15 1997/08/10 20:58:57 fsmp Exp $ */ @@ -11,12 +11,41 @@ #include "i386/isa/intr_machdep.h" +#ifdef FAST_SIMPLELOCK + +#define GET_FAST_INTR_LOCK \ + pushl $_fast_intr_lock ; /* address of lock */ \ + call _s_lock ; /* MP-safe */ \ + addl $4,%esp + +#define REL_FAST_INTR_LOCK \ + pushl $_fast_intr_lock ; /* address of lock */ \ + call _s_unlock ; /* MP-safe */ \ + addl $4,%esp + +#else /* FAST_SIMPLELOCK */ + +#define GET_FAST_INTR_LOCK \ + call _get_isrlock +#define REL_FAST_INTR_LOCK \ + pushl $_mp_lock ; /* GIANT_LOCK */ \ + call _MPrellock ; \ + add $4, %esp + +#endif /* FAST_SIMPLELOCK */ + +#define REL_ISR_LOCK \ + pushl $_mp_lock ; /* GIANT_LOCK */ \ + call _MPrellock ; \ + add $4, %esp + /* convert an absolute IRQ# into a bitmask */ #define IRQ_BIT(irq_num) (1 << (irq_num)) /* make an index into the IO APIC from the IRQ# */ #define REDTBL_IDX(irq_num) (0x10 + ((irq_num) * 2)) + /* * 'lazy masking' code suggested by Bruce Evans <bde@zeta.org.au> */ @@ -115,6 +144,8 @@ * Macros for interrupt interrupt entry, call to handler, and exit. */ +#ifdef FAST_WITHOUTCPL + #define FAST_INTR(irq_num, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ @@ -128,11 +159,45 @@ IDTVEC(vec_name) ; \ movl %ax,%ds ; \ MAYBE_MOVW_AX_ES ; \ FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \ - call _get_isrlock ; \ + GET_FAST_INTR_LOCK ; \ pushl _intr_unit + (irq_num) * 4 ; \ call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ + addl $4, %esp ; \ movl $0, lapic_eoi ; \ + lock ; \ + incl _cnt+V_INTR ; /* book-keeping can wait */ \ + movl _intr_countp + (irq_num) * 4, %eax ; \ + lock ; \ + incl (%eax) ; \ + MEXITCOUNT ; \ + REL_FAST_INTR_LOCK ; \ + MAYBE_POPL_ES ; \ + popl %ds ; \ + popl %edx ; \ + popl %ecx ; \ + popl %eax ; \ + iret + +#else + +#define FAST_INTR(irq_num, vec_name) \ + .text ; \ + SUPERALIGN_TEXT ; \ +IDTVEC(vec_name) ; \ + pushl %eax ; /* save only call-used registers */ \ + pushl %ecx ; \ + pushl %edx ; \ + pushl %ds ; \ + MAYBE_PUSHL_ES ; \ + movl $KDSEL,%eax ; \ + movl %ax,%ds ; \ + MAYBE_MOVW_AX_ES ; \ + FAKE_MCOUNT((4+ACTUALLY_PUSHED)*4(%esp)) ; \ + GET_FAST_INTR_LOCK ; \ + pushl _intr_unit + (irq_num) * 4 ; \ + call *_intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ addl $4,%esp ; \ + movl $0, lapic_eoi ; \ incl _cnt+V_INTR ; /* book-keeping can wait */ \ movl _intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ @@ -142,9 +207,7 @@ IDTVEC(vec_name) ; \ jne 2f ; /* yes, maybe handle them */ \ 1: ; \ MEXITCOUNT ; \ - pushl $_mp_lock ; /* GIANT_LOCK */ \ - call _MPrellock ; \ - add $4, %esp ; \ + REL_FAST_INTR_LOCK ; \ MAYBE_POPL_ES ; \ popl %ds ; \ popl %edx ; \ @@ -178,6 +241,9 @@ IDTVEC(vec_name) ; \ MEXITCOUNT ; \ jmp _doreti +#endif /** FAST_WITHOUTCPL */ + + #define INTR(irq_num, vec_name) \ .text ; \ SUPERALIGN_TEXT ; \ @@ -217,9 +283,7 @@ __CONCAT(Xresume,irq_num): ; \ /* XXX skip mcounting here to avoid double count */ \ lock ; /* MP-safe */ \ orl $IRQ_BIT(irq_num), _ipending ; \ - pushl $_mp_lock ; /* GIANT_LOCK */ \ - call _MPrellock ; \ - add $4, %esp ; \ + REL_ISR_LOCK ; \ popl %es ; \ popl %ds ; \ popal ; \ diff --git a/sys/i386/isa/ipl.s b/sys/i386/isa/ipl.s index 002e924..3b2bf73 100644 --- a/sys/i386/isa/ipl.s +++ b/sys/i386/isa/ipl.s @@ -36,7 +36,7 @@ * * @(#)ipl.s * - * $Id: ipl.s,v 1.7 1997/08/10 20:58:58 fsmp Exp $ + * $Id: ipl.s,v 1.8 1997/08/10 21:17:55 fsmp Exp $ */ @@ -182,8 +182,7 @@ doreti_unpend: */ sti bsfl %ecx,%ecx /* slow, but not worth optimizing */ - MP_INSTR_LOCK - btrl %ecx,_ipending + MPLOCKED btrl %ecx, _ipending jnc doreti_next /* some intr cleared memory copy */ movl ihandlers(,%ecx,4),%edx testl %edx,%edx @@ -244,8 +243,7 @@ swi_ast_phantom: * using by using cli, but they are unavoidable for lcall entries. */ cli - MP_INSTR_LOCK - orl $SWI_AST_PENDING,_ipending + MPLOCKED orl $SWI_AST_PENDING, _ipending subl %eax,%eax jmp doreti_exit /* SWI_AST is highest so we must be done */ @@ -283,8 +281,7 @@ splz_next: ALIGN_TEXT splz_unpend: bsfl %ecx,%ecx - MP_INSTR_LOCK - btrl %ecx,_ipending + MPLOCKED btrl %ecx, _ipending jnc splz_next movl ihandlers(,%ecx,4),%edx testl %edx,%edx diff --git a/sys/i386/isa/ipl_funcs.c b/sys/i386/isa/ipl_funcs.c index fa3136a..c0849cd 100644 --- a/sys/i386/isa/ipl_funcs.c +++ b/sys/i386/isa/ipl_funcs.c @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id$ + * $Id: ipl_funcs.c,v 1.1 1997/05/31 08:57:05 peter Exp $ */ #include <sys/types.h> @@ -92,3 +92,13 @@ splx(unsigned ipl) if (ipending & ~ipl) splz(); } + +#ifdef SMP +intrmask_t +splq(intrmask_t mask) +{ + intrmask_t tmp = cpl; + cpl |= mask; + return (tmp); +} +#endif diff --git a/sys/i386/isa/sio.c b/sys/i386/isa/sio.c index 55944b2..5249698 100644 --- a/sys/i386/isa/sio.c +++ b/sys/i386/isa/sio.c @@ -31,7 +31,7 @@ * SUCH DAMAGE. * * from: @(#)com.c 7.5 (Berkeley) 5/16/91 - * $Id: sio.c,v 1.1 1997/08/04 17:58:18 smp Exp smp $ + * $Id: sio.c,v 1.174 1997/08/04 19:12:54 fsmp Exp $ */ #include "opt_comconsole.h" @@ -82,6 +82,15 @@ #include <pccard/slot.h> #endif +#ifdef SMP +#include <machine/smp.h> +#define COM_LOCK() s_lock(&com_lock) +#define COM_UNLOCK() s_unlock(&com_lock) +#else +#define COM_LOCK() +#define COM_UNLOCK() +#endif /* SMP */ + #ifdef APIC_IO /* * INTs are masked in the (global) IO APIC, @@ -559,9 +568,11 @@ static int card_intr(struct pccard_dev *dp) { struct com_s *com; + COM_LOCK(); com = com_addr(dp->isahd.id_unit); if (com && !com_addr(dp->isahd.id_unit)->gone) siointr1(com_addr(dp->isahd.id_unit)); + COM_UNLOCK(); return(1); } #endif /* NCRD > 0 */ @@ -1183,12 +1194,14 @@ open_top: } disable_intr(); + COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); com->prev_modem_status = com->last_modem_status = inb(com->modem_status_port); outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); + COM_UNLOCK(); enable_intr(); /* * Handle initial DCD. Callout devices get a fake initial @@ -1417,7 +1430,9 @@ siointr(unit) int unit; { #ifndef COM_MULTIPORT + COM_LOCK(); siointr1(com_addr(unit)); + COM_UNLOCK(); #else /* COM_MULTIPORT */ struct com_s *com; bool_t possibly_more_intrs; @@ -1429,6 +1444,7 @@ siointr(unit) * devices, then the edge from one may be lost because another is * on. */ + COM_LOCK(); do { possibly_more_intrs = FALSE; for (unit = 0; unit < NSIO; ++unit) { @@ -1442,6 +1458,7 @@ siointr(unit) } } } while (possibly_more_intrs); + COM_UNLOCK(); #endif /* COM_MULTIPORT */ } @@ -1821,6 +1838,7 @@ repeat: * loop. */ disable_intr(); + COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; if (com->state & CS_CHECKMSR) { @@ -1828,6 +1846,7 @@ repeat: com->state &= ~CS_CHECKMSR; } com_events -= incc; + COM_UNLOCK(); enable_intr(); if (incc != 0) log(LOG_DEBUG, @@ -1843,6 +1862,7 @@ repeat: } else { buf = ibuf; disable_intr(); + COM_LOCK(); incc = com->iptr - buf; com_events -= incc; if (ibuf == com->ibuf1) @@ -1863,6 +1883,7 @@ repeat: && !(tp->t_state & TS_TBLOCK)) outb(com->modem_ctl_port, com->mcr_image |= MCR_RTS); + COM_UNLOCK(); enable_intr(); com->ibuf = ibuf; } @@ -1871,11 +1892,13 @@ repeat: u_char delta_modem_status; disable_intr(); + COM_LOCK(); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; + COM_UNLOCK(); enable_intr(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) @@ -1883,8 +1906,10 @@ repeat: } if (com->state & CS_ODONE) { disable_intr(); + COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; + COM_UNLOCK(); enable_intr(); if (!(com->state & CS_BUSY) && !(com->extra_state & CSE_BUSYCHECK)) { @@ -2040,9 +2065,11 @@ comparam(tp, t) * error bits, but that is acceptable here. */ disable_intr(); + COM_LOCK(); retry: com->state &= ~CS_TTGO; txtimeout = tp->t_timeout; + COM_UNLOCK(); enable_intr(); while ((inb(com->line_status_port) & (LSR_TSRE | LSR_TXRDY)) != (LSR_TSRE | LSR_TXRDY)) { @@ -2059,7 +2086,9 @@ retry: if (error != 0 && error != EAGAIN) { if (!(tp->t_state & TS_TTSTOP)) { disable_intr(); + COM_LOCK(); com->state |= CS_TTGO; + COM_UNLOCK(); enable_intr(); } splx(s); @@ -2068,6 +2097,7 @@ retry: } disable_intr(); /* very important while com_data is hidden */ + COM_LOCK(); /* * XXX - clearing CS_TTGO is not sufficient to stop further output, @@ -2163,6 +2193,7 @@ retry: if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); + COM_UNLOCK(); enable_intr(); splx(s); comstart(tp); @@ -2181,6 +2212,7 @@ comstart(tp) com = com_addr(unit); s = spltty(); disable_intr(); + COM_LOCK(); if (tp->t_state & TS_TTSTOP) com->state &= ~CS_TTGO; else @@ -2193,6 +2225,7 @@ comstart(tp) && com->state & CS_RTS_IFLOW) outb(com->modem_ctl_port, com->mcr_image |= MCR_RTS); } + COM_UNLOCK(); enable_intr(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { splx(s); @@ -2209,6 +2242,7 @@ comstart(tp) com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; disable_intr(); + COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; while ((next = qp->l_next) != NULL) @@ -2220,6 +2254,7 @@ comstart(tp) com->obufq.l_next = &com->obufs[0]; com->state |= CS_BUSY; } + COM_UNLOCK(); enable_intr(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { @@ -2229,6 +2264,7 @@ comstart(tp) com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; disable_intr(); + COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; while ((next = qp->l_next) != NULL) @@ -2240,13 +2276,16 @@ comstart(tp) com->obufq.l_next = &com->obufs[1]; com->state |= CS_BUSY; } + COM_UNLOCK(); enable_intr(); } tp->t_state |= TS_BUSY; } disable_intr(); + COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ + COM_UNLOCK(); enable_intr(); ttwwakeup(tp); splx(s); @@ -2263,6 +2302,7 @@ siostop(tp, rw) if (com->gone) return; disable_intr(); + COM_LOCK(); if (rw & FWRITE) { if (com->hasfifo) #ifdef COM_ESP @@ -2291,6 +2331,7 @@ siostop(tp, rw) com_events -= (com->iptr - com->ibuf); com->iptr = com->ibuf; } + COM_UNLOCK(); enable_intr(); comstart(tp); } @@ -2351,6 +2392,7 @@ commctl(com, bits, how) if (com->gone) return(0); disable_intr(); + COM_LOCK(); switch (how) { case DMSET: outb(com->modem_ctl_port, @@ -2363,6 +2405,7 @@ commctl(com, bits, how) outb(com->modem_ctl_port, com->mcr_image &= ~mcr); break; } + COM_UNLOCK(); enable_intr(); return (0); } @@ -2422,7 +2465,9 @@ comwakeup(chan) if (com != NULL && !com->gone && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { disable_intr(); + COM_LOCK(); siointr1(com); + COM_UNLOCK(); enable_intr(); } } @@ -2446,8 +2491,10 @@ comwakeup(chan) u_long total; disable_intr(); + COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; + COM_UNLOCK(); enable_intr(); if (delta == 0) continue; |