diff options
| author | jhb <jhb@FreeBSD.org> | 2001-12-18 00:27:18 +0000 |
|---|---|---|
| committer | jhb <jhb@FreeBSD.org> | 2001-12-18 00:27:18 +0000 |
| commit | a3b98398cbfb4b809f8577b6a95aabb2c30a1aeb (patch) | |
| tree | bd1f842c61588e8478e798dece6dff8b2be41310 /sys | |
| parent | 090c933e94e7345e9c9e9a9fbe29ea6c8397a662 (diff) | |
| download | FreeBSD-src-a3b98398cbfb4b809f8577b6a95aabb2c30a1aeb.zip FreeBSD-src-a3b98398cbfb4b809f8577b6a95aabb2c30a1aeb.tar.gz | |
Modify the critical section API as follows:
- The MD functions critical_enter/exit are renamed to start with a cpu_
prefix.
- MI wrapper functions critical_enter/exit maintain a per-thread nesting
count and a per-thread critical section saved state set when entering
a critical section while at nesting level 0 and restored when exiting
to nesting level 0. This moves the saved state out of spin mutexes so
that interlocking spin mutexes works properly.
- Most low-level MD code that used critical_enter/exit now use
cpu_critical_enter/exit. MI code such as device drivers and spin
mutexes use the MI wrappers. Note that since the MI wrappers store
the state in the current thread, they do not have any return values or
arguments.
- mtx_intr_enable() is replaced with a constant CRITICAL_FORK which is
assigned to curthread->td_savecrit during fork_exit().
Tested on: i386, alpha
Diffstat (limited to 'sys')
61 files changed, 376 insertions, 502 deletions
diff --git a/sys/alpha/alpha/db_interface.c b/sys/alpha/alpha/db_interface.c index fea763d..041b0b9 100644 --- a/sys/alpha/alpha/db_interface.c +++ b/sys/alpha/alpha/db_interface.c @@ -192,7 +192,7 @@ kdb_trap(a0, a1, a2, entry, regs) ddb_regs = *regs; - s = critical_enter(); + s = cpu_critical_enter(); #ifdef SMP #ifdef DIAGNOSTIC @@ -219,7 +219,7 @@ kdb_trap(a0, a1, a2, entry, regs) restart_cpus(stopped_cpus); #endif - critical_exit(s); + cpu_critical_exit(s); *regs = ddb_regs; diff --git a/sys/alpha/alpha/genassym.c b/sys/alpha/alpha/genassym.c index 8d68d73..e989675 100644 --- a/sys/alpha/alpha/genassym.c +++ b/sys/alpha/alpha/genassym.c @@ -76,7 +76,6 @@ ASSYM(PC_IDLEPCBPHYS, offsetof(struct pcpu, pc_idlepcbphys)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSE, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); ASSYM(MTX_UNOWNED, MTX_UNOWNED); ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); diff --git a/sys/alpha/alpha/interrupt.c b/sys/alpha/alpha/interrupt.c index 29bb74f..3143856 100644 --- a/sys/alpha/alpha/interrupt.c +++ b/sys/alpha/alpha/interrupt.c @@ -99,13 +99,13 @@ interrupt(a0, a1, a2, framep) * Find our per-cpu globals. */ #ifdef SMP - s = critical_enter(); + s = cpu_critical_enter(); #endif pcpup = (struct pcpu *) alpha_pal_rdval(); td = curthread; #ifdef SMP td->td_md.md_kernnest++; - critical_exit(s); + cpu_critical_exit(s); #endif atomic_add_int(&td->td_intr_nesting_level, 1); #ifndef KSTACK_GUARD diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c index ef763a9..a6f181c 100644 --- a/sys/alpha/alpha/machdep.c +++ b/sys/alpha/alpha/machdep.c @@ -2007,11 +2007,11 @@ alpha_fpstate_check(struct thread *td) #ifndef SMP critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (td->td_pcb->pcb_hw.apcb_flags & ALPHA_PCB_FLAGS_FEN) if (td != PCPU_GET(fpcurthread)) panic("alpha_check_fpcurthread: bogus"); - critical_exit(s); + cpu_critical_exit(s); #endif } @@ -2033,7 +2033,7 @@ alpha_fpstate_save(struct thread *td, int write) { critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (td != NULL && td == PCPU_GET(fpcurthread)) { /* * If curthread != fpcurthread, then we need to enable FEN @@ -2068,7 +2068,7 @@ alpha_fpstate_save(struct thread *td, int write) alpha_pal_wrfen(0); } } - critical_exit(s); + cpu_critical_exit(s); } /* @@ -2081,7 +2081,7 @@ alpha_fpstate_drop(struct thread *td) { critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) { if (td == curthread) { /* @@ -2097,7 +2097,7 @@ alpha_fpstate_drop(struct thread *td) } PCPU_SET(fpcurthread, NULL); } - critical_exit(s); + cpu_critical_exit(s); } /* @@ -2112,7 +2112,7 @@ alpha_fpstate_switch(struct thread *td) /* * Enable FEN so that we can access the fp registers. */ - s = critical_enter(); + s = cpu_critical_enter(); alpha_pal_wrfen(1); if (PCPU_GET(fpcurthread)) { /* @@ -2139,7 +2139,7 @@ alpha_fpstate_switch(struct thread *td) } td->td_md.md_flags |= MDP_FPUSED; - critical_exit(s); + cpu_critical_exit(s); } /* diff --git a/sys/alpha/alpha/prom.c b/sys/alpha/alpha/prom.c index 54d913a..d4dd267 100644 --- a/sys/alpha/alpha/prom.c +++ b/sys/alpha/alpha/prom.c @@ -193,7 +193,7 @@ enter_prom() pt_entry_t *lev1map; critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (!prom_mapped) { #ifdef SIMOS @@ -232,7 +232,7 @@ leave_prom(s) lev1map[0] = saved_pte[0]; /* XXX */ prom_cache_sync(); /* XXX */ } - critical_exit(s); + cpu_critical_exit(s); } static void @@ -272,7 +272,7 @@ prom_halt(halt) /* * Turn off interrupts, for sanity. */ - critical_enter(); + cpu_critical_enter(); /* * Set "boot request" part of the CPU state depending on what diff --git a/sys/alpha/alpha/trap.c b/sys/alpha/alpha/trap.c index 4c7694d..99278fa 100644 --- a/sys/alpha/alpha/trap.c +++ b/sys/alpha/alpha/trap.c @@ -270,13 +270,13 @@ trap(a0, a1, a2, entry, framep) * Find our per-cpu globals. */ #ifdef SMP - s = critical_enter(); + s = cpu_critical_enter(); #endif pcpup = (struct pcpu *) alpha_pal_rdval(); td = curthread; #ifdef SMP td->td_md.md_kernnest++; - critical_exit(s); + cpu_critical_exit(s); #endif p = td->td_proc; @@ -672,13 +672,13 @@ syscall(code, framep) * Find our per-cpu globals. */ #ifdef SMP - s = critical_enter(); + s = cpu_critical_enter(); #endif pcpup = (struct pcpu *) alpha_pal_rdval(); td = curthread; #ifdef SMP td->td_md.md_kernnest++; - critical_exit(s); + cpu_critical_exit(s); #endif p = td->td_proc; diff --git a/sys/alpha/include/cpufunc.h b/sys/alpha/include/cpufunc.h index 51a1fdc..dbee911 100644 --- a/sys/alpha/include/cpufunc.h +++ b/sys/alpha/include/cpufunc.h @@ -35,6 +35,8 @@ #include <machine/chipset.h> #include <machine/alpha_cpu.h> +#define CRITICAL_FORK (ALPHA_PSL_IPL_0) + #ifdef __GNUC__ static __inline void @@ -46,13 +48,13 @@ breakpoint(void) #endif static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { return (alpha_pal_swpipl(ALPHA_PSL_IPL_MCES)); } static __inline void -critical_exit(critical_t ipl) +cpu_critical_exit(critical_t ipl) { alpha_pal_swpipl(ipl); } diff --git a/sys/alpha/include/mutex.h b/sys/alpha/include/mutex.h index 970049b..7300c08 100644 --- a/sys/alpha/include/mutex.h +++ b/sys/alpha/include/mutex.h @@ -39,8 +39,6 @@ /* Global locks */ extern struct mtx clock_lock; -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit = ALPHA_PSL_IPL_0; while (0) - #endif /* _KERNEL */ #else /* !LOCORE */ diff --git a/sys/alpha/pci/cia.c b/sys/alpha/pci/cia.c index c2bcafc..09436e3 100644 --- a/sys/alpha/pci/cia.c +++ b/sys/alpha/pci/cia.c @@ -156,12 +156,12 @@ cia_swiz_set_hae_mem(void *arg, u_int32_t pa) u_int32_t msb = pa & REG1; critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); cia_hae_mem = (cia_hae_mem & ~REG1) | msb; REGVAL(CIA_CSR_HAE_MEM) = cia_hae_mem; alpha_mb(); cia_hae_mem = REGVAL(CIA_CSR_HAE_MEM); - critical_exit(s); + cpu_critical_exit(s); } return pa & ~REG1; } @@ -226,7 +226,7 @@ cia_sgmap_invalidate_pyxis(void) int i; critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); /* * Put the Pyxis into PCI loopback mode. @@ -257,7 +257,7 @@ cia_sgmap_invalidate_pyxis(void) REGVAL(CIA_CSR_CTRL) = ctrl; alpha_mb(); - critical_exit(s); + cpu_critical_exit(s); } static void diff --git a/sys/alpha/pci/t2.c b/sys/alpha/pci/t2.c index c8ade0c..d9d0b9d 100644 --- a/sys/alpha/pci/t2.c +++ b/sys/alpha/pci/t2.c @@ -118,14 +118,14 @@ t2_set_hae_mem(void *arg, u_int32_t pa) msb = pa & 0xf8000000; pa -= msb; msb >>= 27; /* t2 puts high bits in the bottom of the register */ - s = critical_enter(); + s = cpu_critical_enter(); if (msb != t2_hae_mem[hose]) { t2_hae_mem[hose] = msb; t2_csr[hose]->hae0_1 = t2_hae_mem[hose]; alpha_mb(); t2_hae_mem[hose] = t2_csr[hose]->hae0_1; } - critical_exit(s); + cpu_critical_exit(s); } return pa; } diff --git a/sys/alpha/pci/t2_pci.c b/sys/alpha/pci/t2_pci.c index 87a3dca..b50555f 100644 --- a/sys/alpha/pci/t2_pci.c +++ b/sys/alpha/pci/t2_pci.c @@ -92,7 +92,7 @@ t2_pcib_maxslots(device_t dev) #define T2_TYPE1_SETUP(b,s,old_hae3) if((b)) { \ do { \ - (s) = critical_enter(); \ + (s) = cpu_critical_enter(); \ (old_hae3) = REGVAL(T2_HAE0_3); \ alpha_mb(); \ REGVAL(T2_HAE0_3) = (old_hae3) | (1<<30); \ @@ -105,7 +105,7 @@ t2_pcib_maxslots(device_t dev) alpha_mb(); \ REGVAL(T2_HAE0_3) = (old_hae3); \ alpha_mb(); \ - critical_exit((s)); \ + cpu_critical_exit((s)); \ } while(0); \ } diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c index 9708cad..000085b 100644 --- a/sys/amd64/amd64/db_interface.c +++ b/sys/amd64/amd64/db_interface.c @@ -324,10 +324,10 @@ Debugger(msg) return; if (atomic_cmpset_acq_int(&in_Debugger, 0, 1)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); db_printf("Debugger(\"%s\")\n", msg); breakpoint(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); atomic_store_rel_int(&in_Debugger, 0); } } diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c index ff8c4cf..34a822a 100644 --- a/sys/amd64/amd64/fpu.c +++ b/sys/amd64/amd64/fpu.c @@ -515,7 +515,7 @@ npxinit(control) * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -527,7 +527,7 @@ npxinit(control) if (PCPU_GET(curpcb) != NULL) fpusave(&PCPU_GET(curpcb)->pcb_save); start_emulating(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } /* @@ -539,10 +539,10 @@ npxexit(td) { critical_t savecrit; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #ifdef NPX_DEBUG if (npx_exists) { u_int masked_exceptions; @@ -762,7 +762,7 @@ npxtrap() PCPU_GET(fpcurthread), curthread, npx_exists); panic("npxtrap from nowhere"); } - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -783,7 +783,7 @@ npxtrap() GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -807,7 +807,7 @@ npxdna() PCPU_GET(fpcurthread), curthread); panic("npxdna"); } - s = critical_enter(); + s = cpu_critical_enter(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -829,7 +829,7 @@ npxdna() * first FPU instruction after a context switch. */ fpurstor(&PCPU_GET(curpcb)->pcb_save); - critical_exit(s); + cpu_critical_exit(s); return (1); } diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index 59f3147..99fa502 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -205,7 +205,6 @@ ASSYM(VM86_FRAMESIZE, sizeof(struct vm86frame)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSECNT, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); #ifdef PC98 #include <machine/bus.h> diff --git a/sys/amd64/amd64/initcpu.c b/sys/amd64/amd64/initcpu.c index b32c786..8a9a2a5 100644 --- a/sys/amd64/amd64/initcpu.c +++ b/sys/amd64/amd64/initcpu.c @@ -646,7 +646,7 @@ enable_K5_wt_alloc(void) * a stepping of 4 or greater. */ if (((cpu_id & 0xf0) > 0) && ((cpu_id & 0x0f) > 3)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); msr = rdmsr(0x83); /* HWCR */ wrmsr(0x83, msr & !(0x10)); @@ -678,7 +678,7 @@ enable_K5_wt_alloc(void) msr=rdmsr(0x83); wrmsr(0x83, msr|0x10); /* enable write allocate */ - critical_exit(savecrit); + cpu_critical_exit(savecrit); } } diff --git a/sys/amd64/amd64/sys_machdep.c b/sys/amd64/amd64/sys_machdep.c index a88acc9..b7beea9 100644 --- a/sys/amd64/amd64/sys_machdep.c +++ b/sys/amd64/amd64/sys_machdep.c @@ -532,13 +532,13 @@ i386_set_ldt(td, args) } /* Fill in range */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); error = copyin(uap->descs, &((union descriptor *)(pldt->ldt_base))[uap->start], uap->num * sizeof(union descriptor)); if (!error) td->td_retval[0] = uap->start; - critical_exit(savecrit); + cpu_critical_exit(savecrit); return(error); } diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c index 322b5e7..4f1dab2 100644 --- a/sys/amd64/amd64/vm_machdep.c +++ b/sys/amd64/amd64/vm_machdep.c @@ -153,10 +153,10 @@ cpu_fork(td1, p2, flags) #ifdef DEV_NPX if (td1 == curthread) td1->td_pcb->pcb_gs = rgs(); - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (PCPU_GET(fpcurthread) == td1) npxsave(&td1->td_pcb->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #endif /* Point the pcb to the top of the stack */ diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h index 4a9300a..a4f57cb 100644 --- a/sys/amd64/include/cpufunc.h +++ b/sys/amd64/include/cpufunc.h @@ -41,6 +41,7 @@ #define _MACHINE_CPUFUNC_H_ #include <sys/cdefs.h> +#include <machine/psl.h> __BEGIN_DECLS #define readb(va) (*(volatile u_int8_t *) (va)) @@ -51,6 +52,8 @@ __BEGIN_DECLS #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) +#define CRITICAL_FORK (read_eflags() | PSL_I) + #ifdef __GNUC__ #ifdef SWTCH_OPTIM_STATS @@ -548,7 +551,7 @@ load_dr7(u_int sel) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t eflags; @@ -558,7 +561,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t eflags) +cpu_critical_exit(critical_t eflags) { write_eflags(eflags); } @@ -597,8 +600,8 @@ u_int rfs __P((void)); u_int rgs __P((void)); void load_fs __P((u_int sel)); void load_gs __P((u_int sel)); -critical_t critical_enter __P((void)); -void critical_exit __P((critical_t eflags)); +critical_t cpu_critical_enter __P((void)); +void cpu_critical_exit __P((critical_t eflags)); #endif /* __GNUC__ */ diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h index ae37b23..e68bce7 100644 --- a/sys/amd64/include/mutex.h +++ b/sys/amd64/include/mutex.h @@ -35,13 +35,10 @@ #ifndef LOCORE #ifdef _KERNEL -#include <machine/psl.h> /* Global locks */ extern struct mtx clock_lock; -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= PSL_I; while (0) - /* * Assembly macros (for internal use only) *------------------------------------------------------------------------------ @@ -246,51 +243,20 @@ extern struct mtx clock_lock; * locks) in the near future, however. */ #define MTX_LOCK_SPIN(lck, flags) \ - pushl %eax ; \ - pushl %ecx ; \ - pushl %ebx ; \ - movl $(MTX_UNOWNED) , %eax ; \ - movl PCPU(CURTHREAD), %ebx ; \ - pushfl ; \ - popl %ecx ; \ - cli ; \ - MPLOCKED cmpxchgl %ebx, lck+MTX_LOCK ; \ - jz 2f ; \ - cmpl lck+MTX_LOCK, %ebx ; \ - je 3f ; \ pushl $0 ; \ pushl $0 ; \ - pushl %ecx ; \ pushl $flags ; \ pushl $lck ; \ - call _mtx_lock_spin ; \ - addl $0x14, %esp ; \ - jmp 1f ; \ -3: movl lck+MTX_RECURSECNT, %ebx ; \ - incl %ebx ; \ - movl %ebx, lck+MTX_RECURSECNT ; \ - jmp 1f ; \ -2: movl %ecx, lck+MTX_SAVECRIT ; \ -1: popl %ebx ; \ - popl %ecx ; \ - popl %eax + call _mtx_lock_spin_flags ; \ + addl $0x10, %esp ; \ #define MTX_UNLOCK_SPIN(lck) \ - pushl %edx ; \ - pushl %eax ; \ - movl lck+MTX_SAVECRIT, %edx ; \ - movl lck+MTX_RECURSECNT, %eax ; \ - testl %eax, %eax ; \ - jne 2f ; \ - movl $(MTX_UNOWNED), %eax ; \ - xchgl %eax, lck+MTX_LOCK ; \ - pushl %edx ; \ - popfl ; \ - jmp 1f ; \ -2: decl %eax ; \ - movl %eax, lck+MTX_RECURSECNT ; \ -1: popl %eax ; \ - popl %edx + pushl $0 ; \ + pushl $0 ; \ + pushl $0 ; \ + pushl $lck ; \ + call _mtx_unlock_spin_flags ; \ + addl $0x10, %esp ; \ /* * XXX: These two are broken right now and need to be made to work for diff --git a/sys/amd64/isa/npx.c b/sys/amd64/isa/npx.c index ff8c4cf..34a822a 100644 --- a/sys/amd64/isa/npx.c +++ b/sys/amd64/isa/npx.c @@ -515,7 +515,7 @@ npxinit(control) * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -527,7 +527,7 @@ npxinit(control) if (PCPU_GET(curpcb) != NULL) fpusave(&PCPU_GET(curpcb)->pcb_save); start_emulating(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } /* @@ -539,10 +539,10 @@ npxexit(td) { critical_t savecrit; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #ifdef NPX_DEBUG if (npx_exists) { u_int masked_exceptions; @@ -762,7 +762,7 @@ npxtrap() PCPU_GET(fpcurthread), curthread, npx_exists); panic("npxtrap from nowhere"); } - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -783,7 +783,7 @@ npxtrap() GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -807,7 +807,7 @@ npxdna() PCPU_GET(fpcurthread), curthread); panic("npxdna"); } - s = critical_enter(); + s = cpu_critical_enter(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -829,7 +829,7 @@ npxdna() * first FPU instruction after a context switch. */ fpurstor(&PCPU_GET(curpcb)->pcb_save); - critical_exit(s); + cpu_critical_exit(s); return (1); } diff --git a/sys/dev/bktr/bktr_os.h b/sys/dev/bktr/bktr_os.h index 21d3f60..1740723 100644 --- a/sys/dev/bktr/bktr_os.h +++ b/sys/dev/bktr/bktr_os.h @@ -61,9 +61,9 @@ void free_bktr_mem(bktr_ptr_t, bus_dmamap_t, vm_offset_t); /************************************/ #if defined(__FreeBSD__) #if (__FreeBSD_version >=500000) -#define DECLARE_INTR_MASK(s) critical_t s -#define DISABLE_INTR(s) s = critical_enter() -#define ENABLE_INTR(s) critical_exit(s) +#define DECLARE_INTR_MASK(s) /* no need to declare 's' */ +#define DISABLE_INTR(s) critical_enter() +#define ENABLE_INTR(s) critical_exit() #else #define DECLARE_INTR_MASK(s) intrmask_t s #define DISABLE_INTR(s) s=spltty() diff --git a/sys/dev/cy/cy.c b/sys/dev/cy/cy.c index 20316ad..0510e72 100644 --- a/sys/dev/cy/cy.c +++ b/sys/dev/cy/cy.c @@ -356,7 +356,7 @@ static int cd_getreg __P((struct com_s *com, int reg)); static void cd_setreg __P((struct com_s *com, int reg, int val)); static timeout_t siodtrwakeup; static void comhardclose __P((struct com_s *com)); -static void sioinput __P((struct com_s *com, critical_t *savecrit)); +static void sioinput __P((struct com_s *com)); #if 0 static void siointr1 __P((struct com_s *com)); #endif @@ -673,7 +673,6 @@ sioopen(dev, flag, mode, td) int s; struct tty *tp; int unit; - critical_t savecrit; mynor = minor(dev); unit = MINOR_TO_UNIT(mynor); @@ -775,7 +774,7 @@ open_top: } } - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); @@ -784,7 +783,7 @@ open_top: outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #else /* !0 */ /* * Flush fifos. This requires a full channel reset which @@ -795,7 +794,7 @@ open_top: CD1400_CCR_CMDRESET | CD1400_CCR_CHANRESET); cd1400_channel_cmd(com, com->channel_control); - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->prev_modem_status = com->last_modem_status = cd_getreg(com, CD1400_MSVR2); @@ -803,7 +802,7 @@ open_top: com->intr_enable = CD1400_SRER_MDMCH | CD1400_SRER_RXDATA); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif /* 0 */ /* * Handle initial DCD. Callout devices get a fake initial @@ -886,7 +885,6 @@ comhardclose(com) int s; struct tty *tp; int unit; - critical_t savecrit; unit = com->unit; iobase = com->iobase; @@ -900,12 +898,12 @@ comhardclose(com) outb(iobase + com_cfcr, com->cfcr_image &= ~CFCR_SBREAK); #else /* XXX */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->etc = ETC_NONE; cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); cd1400_channel_cmd(com, CD1400_CCR_CMDRESET | CD1400_CCR_FTF); #endif @@ -913,11 +911,11 @@ comhardclose(com) #if 0 outb(iobase + com_ier, 0); #else - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); cd_setreg(com, CD1400_SRER, com->intr_enable = 0); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif tp = com->tp; if ((tp->t_cflag & HUPCL) @@ -1013,9 +1011,8 @@ siodtrwakeup(chan) * b) needs to return with COM_LOCK() held. */ static void -sioinput(com, savecrit) +sioinput(com) struct com_s *com; - critical_t *savecrit; { u_char *buf; int incc; @@ -1046,7 +1043,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); incc = com->iptr - buf; if (tp->t_rawq.c_cc + incc > tp->t_ihiwat && (com->state & CS_RTS_IFLOW @@ -1067,7 +1064,7 @@ sioinput(com, savecrit) tp->t_lflag &= ~FLUSHO; comstart(tp); } - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } else { @@ -1078,7 +1075,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); line_status = buf[com->ierroff]; recv_data = *buf++; if (line_status @@ -1093,7 +1090,7 @@ sioinput(com, savecrit) recv_data |= TTY_PE; } (*linesw[tp->t_line].l_rint)(recv_data, tp); - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } @@ -1767,7 +1764,6 @@ static void siopoll(void *arg) { int unit; - critical_t savecrit; #ifdef CyDebug ++cy_timeouts; @@ -1790,7 +1786,7 @@ repeat: * (actually never opened devices) so that we don't * loop. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; @@ -1800,7 +1796,7 @@ repeat: } com_events -= incc; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (incc != 0) log(LOG_DEBUG, "sio%d: %d events for device with no tp\n", @@ -1808,36 +1804,36 @@ repeat: continue; } if (com->iptr != com->ibuf) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (com->state & CS_CHECKMSR) { u_char delta_modem_status; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) (tp, com->prev_modem_status & MSR_DCD); } if (com->extra_state & CSE_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->extra_state &= ~CSE_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (!(com->state & CS_BUSY)) { tp->t_state &= ~TS_BUSY; ttwwakeup(com->tp); @@ -1849,12 +1845,12 @@ repeat: } } if (com->state & CS_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); (*linesw[tp->t_line].l_start)(tp); } if (com_events == 0) @@ -1883,7 +1879,6 @@ comparam(tp, t) u_char opt; int s; int unit; - critical_t savecrit; /* do historical conversions */ if (t->c_ispeed == 0) @@ -2031,14 +2026,14 @@ comparam(tp, t) if (cflag & CCTS_OFLOW) opt |= CD1400_COR2_CCTS_OFLOW; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (opt != com->cor[1]) { cor_change |= CD1400_CCR_COR2; cd_setreg(com, CD1400_COR2, com->cor[1] = opt); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); /* * set channel option register 3 - @@ -2159,7 +2154,7 @@ comparam(tp, t) * XXX should have done this long ago, but there is too much state * to change all atomically. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->state &= ~CS_TTGO; @@ -2227,7 +2222,7 @@ comparam(tp, t) } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); splx(s); comstart(tp); if (com->ibufold != NULL) { @@ -2246,7 +2241,6 @@ siosetwater(com, speed) u_char *ibuf; int ibufsize; struct tty *tp; - critical_t savecrit; /* * Make the buffer size large enough to handle a softtty interrupt @@ -2284,10 +2278,10 @@ siosetwater(com, speed) * Read current input buffer, if any. Continue with interrupts * disabled. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->iptr != com->ibuf) - sioinput(com, &savecrit); + sioinput(com); /*- * Initialize critical variables, including input buffer watermarks. @@ -2306,7 +2300,7 @@ siosetwater(com, speed) com->ihighwater = ibuf + 3 * ibufsize / 4; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2320,7 +2314,6 @@ comstart(tp) bool_t started; #endif int unit; - critical_t savecrit; unit = DEV_TO_UNIT(tp->t_dev); com = com_addr(unit); @@ -2331,7 +2324,7 @@ comstart(tp) started = FALSE; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (tp->t_state & TS_TTSTOP) { com->state &= ~CS_TTGO; @@ -2369,7 +2362,7 @@ comstart(tp) #endif } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { ttwwakeup(tp); splx(s); @@ -2388,7 +2381,7 @@ comstart(tp) sizeof com->obuf1); com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2409,7 +2402,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { #ifdef CyDebug @@ -2420,7 +2413,7 @@ comstart(tp) sizeof com->obuf2); com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2441,7 +2434,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } tp->t_state |= TS_BUSY; } @@ -2450,12 +2443,12 @@ comstart(tp) ++com->start_real; #endif #if 0 - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif ttwwakeup(tp); splx(s); @@ -2468,11 +2461,10 @@ comstop(tp, rw) { struct com_s *com; bool_t wakeup_etc; - critical_t savecrit; com = com_addr(DEV_TO_UNIT(tp->t_dev)); wakeup_etc = FALSE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (rw & FWRITE) { com->obufs[0].l_queued = FALSE; @@ -2497,7 +2489,7 @@ comstop(tp, rw) com->iptr = com->ibuf; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (wakeup_etc) wakeup(&com->etc); if (rw & FWRITE && com->etc == ETC_NONE) @@ -2513,7 +2505,6 @@ commctl(com, bits, how) { int mcr; int msr; - critical_t savecrit; if (how == DMGET) { if (com->channel_control & CD1400_CCR_RCVEN) @@ -2551,7 +2542,7 @@ commctl(com, bits, how) mcr |= com->mcr_dtr; if (bits & TIOCM_RTS) mcr |= com->mcr_rts; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); switch (how) { case DMSET: @@ -2571,7 +2562,7 @@ commctl(com, bits, how) break; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2633,13 +2624,11 @@ comwakeup(chan) com = com_addr(unit); if (com != NULL && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { - critical_t savecrit; - - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); siointr1(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } } #endif @@ -2659,14 +2648,13 @@ comwakeup(chan) for (errnum = 0; errnum < CE_NTYPES; ++errnum) { u_int delta; u_long total; - critical_t savecrit; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta == 0) continue; total = com->error_counts[errnum] += delta; @@ -2818,7 +2806,6 @@ cd_etc(com, etc) struct com_s *com; int etc; { - critical_t savecrit; /* * We can't change the hardware's ETC state while there are any @@ -2831,7 +2818,7 @@ cd_etc(com, etc) * for the tx to become empty so that the command is sure to be * executed soon after we issue it. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->etc == etc) goto wait; @@ -2842,7 +2829,7 @@ cd_etc(com, etc) && (com->etc == ETC_BREAK_ENDING || com->etc == ETC_BREAK_ENDED || com->etc == ETC_NONE))) { COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return; } com->etc = etc; @@ -2851,7 +2838,7 @@ cd_etc(com, etc) = (com->intr_enable & ~CD1400_SRER_TXRDY) | CD1400_SRER_TXMPTY); wait: COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); while (com->etc == etc && tsleep(&com->etc, TTIPRI | PCATCH, "cyetc", 0) == 0) continue; @@ -2865,7 +2852,6 @@ cd_getreg(com, reg) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; int val; @@ -2875,7 +2861,7 @@ cd_getreg(com, reg) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2883,7 +2869,7 @@ cd_getreg(com, reg) val = cd_inb(iobase, reg, cy_align); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (val); } @@ -2896,7 +2882,6 @@ cd_setreg(com, reg, val) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; @@ -2905,7 +2890,7 @@ cd_setreg(com, reg, val) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2913,7 +2898,7 @@ cd_setreg(com, reg, val) cd_outb(iobase, reg, cy_align, val); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } #ifdef CyDebug diff --git a/sys/dev/cy/cy_isa.c b/sys/dev/cy/cy_isa.c index 20316ad..0510e72 100644 --- a/sys/dev/cy/cy_isa.c +++ b/sys/dev/cy/cy_isa.c @@ -356,7 +356,7 @@ static int cd_getreg __P((struct com_s *com, int reg)); static void cd_setreg __P((struct com_s *com, int reg, int val)); static timeout_t siodtrwakeup; static void comhardclose __P((struct com_s *com)); -static void sioinput __P((struct com_s *com, critical_t *savecrit)); +static void sioinput __P((struct com_s *com)); #if 0 static void siointr1 __P((struct com_s *com)); #endif @@ -673,7 +673,6 @@ sioopen(dev, flag, mode, td) int s; struct tty *tp; int unit; - critical_t savecrit; mynor = minor(dev); unit = MINOR_TO_UNIT(mynor); @@ -775,7 +774,7 @@ open_top: } } - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); @@ -784,7 +783,7 @@ open_top: outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #else /* !0 */ /* * Flush fifos. This requires a full channel reset which @@ -795,7 +794,7 @@ open_top: CD1400_CCR_CMDRESET | CD1400_CCR_CHANRESET); cd1400_channel_cmd(com, com->channel_control); - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->prev_modem_status = com->last_modem_status = cd_getreg(com, CD1400_MSVR2); @@ -803,7 +802,7 @@ open_top: com->intr_enable = CD1400_SRER_MDMCH | CD1400_SRER_RXDATA); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif /* 0 */ /* * Handle initial DCD. Callout devices get a fake initial @@ -886,7 +885,6 @@ comhardclose(com) int s; struct tty *tp; int unit; - critical_t savecrit; unit = com->unit; iobase = com->iobase; @@ -900,12 +898,12 @@ comhardclose(com) outb(iobase + com_cfcr, com->cfcr_image &= ~CFCR_SBREAK); #else /* XXX */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->etc = ETC_NONE; cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); cd1400_channel_cmd(com, CD1400_CCR_CMDRESET | CD1400_CCR_FTF); #endif @@ -913,11 +911,11 @@ comhardclose(com) #if 0 outb(iobase + com_ier, 0); #else - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); cd_setreg(com, CD1400_SRER, com->intr_enable = 0); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif tp = com->tp; if ((tp->t_cflag & HUPCL) @@ -1013,9 +1011,8 @@ siodtrwakeup(chan) * b) needs to return with COM_LOCK() held. */ static void -sioinput(com, savecrit) +sioinput(com) struct com_s *com; - critical_t *savecrit; { u_char *buf; int incc; @@ -1046,7 +1043,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); incc = com->iptr - buf; if (tp->t_rawq.c_cc + incc > tp->t_ihiwat && (com->state & CS_RTS_IFLOW @@ -1067,7 +1064,7 @@ sioinput(com, savecrit) tp->t_lflag &= ~FLUSHO; comstart(tp); } - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } else { @@ -1078,7 +1075,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); line_status = buf[com->ierroff]; recv_data = *buf++; if (line_status @@ -1093,7 +1090,7 @@ sioinput(com, savecrit) recv_data |= TTY_PE; } (*linesw[tp->t_line].l_rint)(recv_data, tp); - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } @@ -1767,7 +1764,6 @@ static void siopoll(void *arg) { int unit; - critical_t savecrit; #ifdef CyDebug ++cy_timeouts; @@ -1790,7 +1786,7 @@ repeat: * (actually never opened devices) so that we don't * loop. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; @@ -1800,7 +1796,7 @@ repeat: } com_events -= incc; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (incc != 0) log(LOG_DEBUG, "sio%d: %d events for device with no tp\n", @@ -1808,36 +1804,36 @@ repeat: continue; } if (com->iptr != com->ibuf) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (com->state & CS_CHECKMSR) { u_char delta_modem_status; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) (tp, com->prev_modem_status & MSR_DCD); } if (com->extra_state & CSE_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->extra_state &= ~CSE_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (!(com->state & CS_BUSY)) { tp->t_state &= ~TS_BUSY; ttwwakeup(com->tp); @@ -1849,12 +1845,12 @@ repeat: } } if (com->state & CS_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); (*linesw[tp->t_line].l_start)(tp); } if (com_events == 0) @@ -1883,7 +1879,6 @@ comparam(tp, t) u_char opt; int s; int unit; - critical_t savecrit; /* do historical conversions */ if (t->c_ispeed == 0) @@ -2031,14 +2026,14 @@ comparam(tp, t) if (cflag & CCTS_OFLOW) opt |= CD1400_COR2_CCTS_OFLOW; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (opt != com->cor[1]) { cor_change |= CD1400_CCR_COR2; cd_setreg(com, CD1400_COR2, com->cor[1] = opt); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); /* * set channel option register 3 - @@ -2159,7 +2154,7 @@ comparam(tp, t) * XXX should have done this long ago, but there is too much state * to change all atomically. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->state &= ~CS_TTGO; @@ -2227,7 +2222,7 @@ comparam(tp, t) } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); splx(s); comstart(tp); if (com->ibufold != NULL) { @@ -2246,7 +2241,6 @@ siosetwater(com, speed) u_char *ibuf; int ibufsize; struct tty *tp; - critical_t savecrit; /* * Make the buffer size large enough to handle a softtty interrupt @@ -2284,10 +2278,10 @@ siosetwater(com, speed) * Read current input buffer, if any. Continue with interrupts * disabled. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->iptr != com->ibuf) - sioinput(com, &savecrit); + sioinput(com); /*- * Initialize critical variables, including input buffer watermarks. @@ -2306,7 +2300,7 @@ siosetwater(com, speed) com->ihighwater = ibuf + 3 * ibufsize / 4; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2320,7 +2314,6 @@ comstart(tp) bool_t started; #endif int unit; - critical_t savecrit; unit = DEV_TO_UNIT(tp->t_dev); com = com_addr(unit); @@ -2331,7 +2324,7 @@ comstart(tp) started = FALSE; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (tp->t_state & TS_TTSTOP) { com->state &= ~CS_TTGO; @@ -2369,7 +2362,7 @@ comstart(tp) #endif } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { ttwwakeup(tp); splx(s); @@ -2388,7 +2381,7 @@ comstart(tp) sizeof com->obuf1); com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2409,7 +2402,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { #ifdef CyDebug @@ -2420,7 +2413,7 @@ comstart(tp) sizeof com->obuf2); com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2441,7 +2434,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } tp->t_state |= TS_BUSY; } @@ -2450,12 +2443,12 @@ comstart(tp) ++com->start_real; #endif #if 0 - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif ttwwakeup(tp); splx(s); @@ -2468,11 +2461,10 @@ comstop(tp, rw) { struct com_s *com; bool_t wakeup_etc; - critical_t savecrit; com = com_addr(DEV_TO_UNIT(tp->t_dev)); wakeup_etc = FALSE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (rw & FWRITE) { com->obufs[0].l_queued = FALSE; @@ -2497,7 +2489,7 @@ comstop(tp, rw) com->iptr = com->ibuf; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (wakeup_etc) wakeup(&com->etc); if (rw & FWRITE && com->etc == ETC_NONE) @@ -2513,7 +2505,6 @@ commctl(com, bits, how) { int mcr; int msr; - critical_t savecrit; if (how == DMGET) { if (com->channel_control & CD1400_CCR_RCVEN) @@ -2551,7 +2542,7 @@ commctl(com, bits, how) mcr |= com->mcr_dtr; if (bits & TIOCM_RTS) mcr |= com->mcr_rts; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); switch (how) { case DMSET: @@ -2571,7 +2562,7 @@ commctl(com, bits, how) break; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2633,13 +2624,11 @@ comwakeup(chan) com = com_addr(unit); if (com != NULL && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { - critical_t savecrit; - - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); siointr1(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } } #endif @@ -2659,14 +2648,13 @@ comwakeup(chan) for (errnum = 0; errnum < CE_NTYPES; ++errnum) { u_int delta; u_long total; - critical_t savecrit; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta == 0) continue; total = com->error_counts[errnum] += delta; @@ -2818,7 +2806,6 @@ cd_etc(com, etc) struct com_s *com; int etc; { - critical_t savecrit; /* * We can't change the hardware's ETC state while there are any @@ -2831,7 +2818,7 @@ cd_etc(com, etc) * for the tx to become empty so that the command is sure to be * executed soon after we issue it. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->etc == etc) goto wait; @@ -2842,7 +2829,7 @@ cd_etc(com, etc) && (com->etc == ETC_BREAK_ENDING || com->etc == ETC_BREAK_ENDED || com->etc == ETC_NONE))) { COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return; } com->etc = etc; @@ -2851,7 +2838,7 @@ cd_etc(com, etc) = (com->intr_enable & ~CD1400_SRER_TXRDY) | CD1400_SRER_TXMPTY); wait: COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); while (com->etc == etc && tsleep(&com->etc, TTIPRI | PCATCH, "cyetc", 0) == 0) continue; @@ -2865,7 +2852,6 @@ cd_getreg(com, reg) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; int val; @@ -2875,7 +2861,7 @@ cd_getreg(com, reg) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2883,7 +2869,7 @@ cd_getreg(com, reg) val = cd_inb(iobase, reg, cy_align); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (val); } @@ -2896,7 +2882,6 @@ cd_setreg(com, reg, val) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; @@ -2905,7 +2890,7 @@ cd_setreg(com, reg, val) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2913,7 +2898,7 @@ cd_setreg(com, reg, val) cd_outb(iobase, reg, cy_align, val); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } #ifdef CyDebug diff --git a/sys/dev/sound/isa/mpu.c b/sys/dev/sound/isa/mpu.c index c0d305e..ed9b746 100644 --- a/sys/dev/sound/isa/mpu.c +++ b/sys/dev/sound/isa/mpu.c @@ -241,7 +241,6 @@ mpu_probe2(device_t dev) sc_p scp; int unit, i; intrmask_t irqp0, irqp1; - critical_t savecrit; scp = device_get_softc(dev); unit = device_get_unit(dev); @@ -267,7 +266,7 @@ mpu_probe2(device_t dev) * Idea-stolen-from: sys/isa/sio.c:sioprobe() */ - savecrit = critical_enter(); + critical_enter(); /* * See the initial irq. We have to do this now, @@ -279,7 +278,7 @@ mpu_probe2(device_t dev) /* Switch to uart mode. */ if (mpu_uartmode(scp) != 0) { - critical_exit(savecrit); + critical_exit(); printf("mpu%d: mode switching failed.\n", unit); mpu_releaseres(scp, dev); return (ENXIO); @@ -298,7 +297,7 @@ mpu_probe2(device_t dev) break; } if (irqp1 == irqp0) { - critical_exit(savecrit); + critical_exit(); printf("mpu%d: switching the mode gave no interrupt.\n", unit); mpu_releaseres(scp, dev); return (ENXIO); @@ -307,13 +306,13 @@ mpu_probe2(device_t dev) no_irq: /* Wait to see an ACK. */ if (mpu_waitack(scp) != 0) { - critical_exit(savecrit); + critical_exit(); printf("mpu%d: not acked.\n", unit); mpu_releaseres(scp, dev); return (ENXIO); } - critical_exit(savecrit); + critical_exit(); if (device_get_flags(dev) & MPU_DF_NO_IRQ) scp->irq_val = 0; diff --git a/sys/i386/i386/db_interface.c b/sys/i386/i386/db_interface.c index 9708cad..000085b 100644 --- a/sys/i386/i386/db_interface.c +++ b/sys/i386/i386/db_interface.c @@ -324,10 +324,10 @@ Debugger(msg) return; if (atomic_cmpset_acq_int(&in_Debugger, 0, 1)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); db_printf("Debugger(\"%s\")\n", msg); breakpoint(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); atomic_store_rel_int(&in_Debugger, 0); } } diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c index 59f3147..99fa502 100644 --- a/sys/i386/i386/genassym.c +++ b/sys/i386/i386/genassym.c @@ -205,7 +205,6 @@ ASSYM(VM86_FRAMESIZE, sizeof(struct vm86frame)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSECNT, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); #ifdef PC98 #include <machine/bus.h> diff --git a/sys/i386/i386/initcpu.c b/sys/i386/i386/initcpu.c index b32c786..8a9a2a5 100644 --- a/sys/i386/i386/initcpu.c +++ b/sys/i386/i386/initcpu.c @@ -646,7 +646,7 @@ enable_K5_wt_alloc(void) * a stepping of 4 or greater. */ if (((cpu_id & 0xf0) > 0) && ((cpu_id & 0x0f) > 3)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); msr = rdmsr(0x83); /* HWCR */ wrmsr(0x83, msr & !(0x10)); @@ -678,7 +678,7 @@ enable_K5_wt_alloc(void) msr=rdmsr(0x83); wrmsr(0x83, msr|0x10); /* enable write allocate */ - critical_exit(savecrit); + cpu_critical_exit(savecrit); } } diff --git a/sys/i386/i386/perfmon.c b/sys/i386/i386/perfmon.c index ed0b53e..4d6c9a2 100644 --- a/sys/i386/i386/perfmon.c +++ b/sys/i386/i386/perfmon.c @@ -124,11 +124,11 @@ perfmon_setup(int pmc, unsigned int control) perfmon_inuse |= (1 << pmc); control &= ~(PMCF_SYS_FLAGS << 16); - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); ctl_shadow[pmc] = control; writectl(pmc); wrmsr(msr_pmc[pmc], pmc_shadow[pmc] = 0); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return 0; } @@ -169,11 +169,11 @@ perfmon_start(int pmc) return EINVAL; if (perfmon_inuse & (1 << pmc)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); ctl_shadow[pmc] |= (PMCF_EN << 16); wrmsr(msr_pmc[pmc], pmc_shadow[pmc]); writectl(pmc); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return 0; } return EBUSY; @@ -188,11 +188,11 @@ perfmon_stop(int pmc) return EINVAL; if (perfmon_inuse & (1 << pmc)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); pmc_shadow[pmc] = rdmsr(msr_pmc[pmc]) & 0xffffffffffULL; ctl_shadow[pmc] &= ~(PMCF_EN << 16); writectl(pmc); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return 0; } return EBUSY; diff --git a/sys/i386/i386/sys_machdep.c b/sys/i386/i386/sys_machdep.c index a88acc9..b7beea9 100644 --- a/sys/i386/i386/sys_machdep.c +++ b/sys/i386/i386/sys_machdep.c @@ -532,13 +532,13 @@ i386_set_ldt(td, args) } /* Fill in range */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); error = copyin(uap->descs, &((union descriptor *)(pldt->ldt_base))[uap->start], uap->num * sizeof(union descriptor)); if (!error) td->td_retval[0] = uap->start; - critical_exit(savecrit); + cpu_critical_exit(savecrit); return(error); } diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index 322b5e7..4f1dab2 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -153,10 +153,10 @@ cpu_fork(td1, p2, flags) #ifdef DEV_NPX if (td1 == curthread) td1->td_pcb->pcb_gs = rgs(); - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (PCPU_GET(fpcurthread) == td1) npxsave(&td1->td_pcb->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #endif /* Point the pcb to the top of the stack */ diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h index 4a9300a..a4f57cb 100644 --- a/sys/i386/include/cpufunc.h +++ b/sys/i386/include/cpufunc.h @@ -41,6 +41,7 @@ #define _MACHINE_CPUFUNC_H_ #include <sys/cdefs.h> +#include <machine/psl.h> __BEGIN_DECLS #define readb(va) (*(volatile u_int8_t *) (va)) @@ -51,6 +52,8 @@ __BEGIN_DECLS #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) +#define CRITICAL_FORK (read_eflags() | PSL_I) + #ifdef __GNUC__ #ifdef SWTCH_OPTIM_STATS @@ -548,7 +551,7 @@ load_dr7(u_int sel) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t eflags; @@ -558,7 +561,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t eflags) +cpu_critical_exit(critical_t eflags) { write_eflags(eflags); } @@ -597,8 +600,8 @@ u_int rfs __P((void)); u_int rgs __P((void)); void load_fs __P((u_int sel)); void load_gs __P((u_int sel)); -critical_t critical_enter __P((void)); -void critical_exit __P((critical_t eflags)); +critical_t cpu_critical_enter __P((void)); +void cpu_critical_exit __P((critical_t eflags)); #endif /* __GNUC__ */ diff --git a/sys/i386/include/mutex.h b/sys/i386/include/mutex.h index ae37b23..e68bce7 100644 --- a/sys/i386/include/mutex.h +++ b/sys/i386/include/mutex.h @@ -35,13 +35,10 @@ #ifndef LOCORE #ifdef _KERNEL -#include <machine/psl.h> /* Global locks */ extern struct mtx clock_lock; -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= PSL_I; while (0) - /* * Assembly macros (for internal use only) *------------------------------------------------------------------------------ @@ -246,51 +243,20 @@ extern struct mtx clock_lock; * locks) in the near future, however. */ #define MTX_LOCK_SPIN(lck, flags) \ - pushl %eax ; \ - pushl %ecx ; \ - pushl %ebx ; \ - movl $(MTX_UNOWNED) , %eax ; \ - movl PCPU(CURTHREAD), %ebx ; \ - pushfl ; \ - popl %ecx ; \ - cli ; \ - MPLOCKED cmpxchgl %ebx, lck+MTX_LOCK ; \ - jz 2f ; \ - cmpl lck+MTX_LOCK, %ebx ; \ - je 3f ; \ pushl $0 ; \ pushl $0 ; \ - pushl %ecx ; \ pushl $flags ; \ pushl $lck ; \ - call _mtx_lock_spin ; \ - addl $0x14, %esp ; \ - jmp 1f ; \ -3: movl lck+MTX_RECURSECNT, %ebx ; \ - incl %ebx ; \ - movl %ebx, lck+MTX_RECURSECNT ; \ - jmp 1f ; \ -2: movl %ecx, lck+MTX_SAVECRIT ; \ -1: popl %ebx ; \ - popl %ecx ; \ - popl %eax + call _mtx_lock_spin_flags ; \ + addl $0x10, %esp ; \ #define MTX_UNLOCK_SPIN(lck) \ - pushl %edx ; \ - pushl %eax ; \ - movl lck+MTX_SAVECRIT, %edx ; \ - movl lck+MTX_RECURSECNT, %eax ; \ - testl %eax, %eax ; \ - jne 2f ; \ - movl $(MTX_UNOWNED), %eax ; \ - xchgl %eax, lck+MTX_LOCK ; \ - pushl %edx ; \ - popfl ; \ - jmp 1f ; \ -2: decl %eax ; \ - movl %eax, lck+MTX_RECURSECNT ; \ -1: popl %eax ; \ - popl %edx + pushl $0 ; \ + pushl $0 ; \ + pushl $0 ; \ + pushl $lck ; \ + call _mtx_unlock_spin_flags ; \ + addl $0x10, %esp ; \ /* * XXX: These two are broken right now and need to be made to work for diff --git a/sys/i386/isa/cy.c b/sys/i386/isa/cy.c index 20316ad..0510e72 100644 --- a/sys/i386/isa/cy.c +++ b/sys/i386/isa/cy.c @@ -356,7 +356,7 @@ static int cd_getreg __P((struct com_s *com, int reg)); static void cd_setreg __P((struct com_s *com, int reg, int val)); static timeout_t siodtrwakeup; static void comhardclose __P((struct com_s *com)); -static void sioinput __P((struct com_s *com, critical_t *savecrit)); +static void sioinput __P((struct com_s *com)); #if 0 static void siointr1 __P((struct com_s *com)); #endif @@ -673,7 +673,6 @@ sioopen(dev, flag, mode, td) int s; struct tty *tp; int unit; - critical_t savecrit; mynor = minor(dev); unit = MINOR_TO_UNIT(mynor); @@ -775,7 +774,7 @@ open_top: } } - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); @@ -784,7 +783,7 @@ open_top: outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #else /* !0 */ /* * Flush fifos. This requires a full channel reset which @@ -795,7 +794,7 @@ open_top: CD1400_CCR_CMDRESET | CD1400_CCR_CHANRESET); cd1400_channel_cmd(com, com->channel_control); - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->prev_modem_status = com->last_modem_status = cd_getreg(com, CD1400_MSVR2); @@ -803,7 +802,7 @@ open_top: com->intr_enable = CD1400_SRER_MDMCH | CD1400_SRER_RXDATA); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif /* 0 */ /* * Handle initial DCD. Callout devices get a fake initial @@ -886,7 +885,6 @@ comhardclose(com) int s; struct tty *tp; int unit; - critical_t savecrit; unit = com->unit; iobase = com->iobase; @@ -900,12 +898,12 @@ comhardclose(com) outb(iobase + com_cfcr, com->cfcr_image &= ~CFCR_SBREAK); #else /* XXX */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->etc = ETC_NONE; cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); cd1400_channel_cmd(com, CD1400_CCR_CMDRESET | CD1400_CCR_FTF); #endif @@ -913,11 +911,11 @@ comhardclose(com) #if 0 outb(iobase + com_ier, 0); #else - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); cd_setreg(com, CD1400_SRER, com->intr_enable = 0); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif tp = com->tp; if ((tp->t_cflag & HUPCL) @@ -1013,9 +1011,8 @@ siodtrwakeup(chan) * b) needs to return with COM_LOCK() held. */ static void -sioinput(com, savecrit) +sioinput(com) struct com_s *com; - critical_t *savecrit; { u_char *buf; int incc; @@ -1046,7 +1043,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); incc = com->iptr - buf; if (tp->t_rawq.c_cc + incc > tp->t_ihiwat && (com->state & CS_RTS_IFLOW @@ -1067,7 +1064,7 @@ sioinput(com, savecrit) tp->t_lflag &= ~FLUSHO; comstart(tp); } - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } else { @@ -1078,7 +1075,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); line_status = buf[com->ierroff]; recv_data = *buf++; if (line_status @@ -1093,7 +1090,7 @@ sioinput(com, savecrit) recv_data |= TTY_PE; } (*linesw[tp->t_line].l_rint)(recv_data, tp); - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } @@ -1767,7 +1764,6 @@ static void siopoll(void *arg) { int unit; - critical_t savecrit; #ifdef CyDebug ++cy_timeouts; @@ -1790,7 +1786,7 @@ repeat: * (actually never opened devices) so that we don't * loop. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; @@ -1800,7 +1796,7 @@ repeat: } com_events -= incc; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (incc != 0) log(LOG_DEBUG, "sio%d: %d events for device with no tp\n", @@ -1808,36 +1804,36 @@ repeat: continue; } if (com->iptr != com->ibuf) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (com->state & CS_CHECKMSR) { u_char delta_modem_status; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) (tp, com->prev_modem_status & MSR_DCD); } if (com->extra_state & CSE_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->extra_state &= ~CSE_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (!(com->state & CS_BUSY)) { tp->t_state &= ~TS_BUSY; ttwwakeup(com->tp); @@ -1849,12 +1845,12 @@ repeat: } } if (com->state & CS_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); (*linesw[tp->t_line].l_start)(tp); } if (com_events == 0) @@ -1883,7 +1879,6 @@ comparam(tp, t) u_char opt; int s; int unit; - critical_t savecrit; /* do historical conversions */ if (t->c_ispeed == 0) @@ -2031,14 +2026,14 @@ comparam(tp, t) if (cflag & CCTS_OFLOW) opt |= CD1400_COR2_CCTS_OFLOW; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (opt != com->cor[1]) { cor_change |= CD1400_CCR_COR2; cd_setreg(com, CD1400_COR2, com->cor[1] = opt); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); /* * set channel option register 3 - @@ -2159,7 +2154,7 @@ comparam(tp, t) * XXX should have done this long ago, but there is too much state * to change all atomically. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->state &= ~CS_TTGO; @@ -2227,7 +2222,7 @@ comparam(tp, t) } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); splx(s); comstart(tp); if (com->ibufold != NULL) { @@ -2246,7 +2241,6 @@ siosetwater(com, speed) u_char *ibuf; int ibufsize; struct tty *tp; - critical_t savecrit; /* * Make the buffer size large enough to handle a softtty interrupt @@ -2284,10 +2278,10 @@ siosetwater(com, speed) * Read current input buffer, if any. Continue with interrupts * disabled. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->iptr != com->ibuf) - sioinput(com, &savecrit); + sioinput(com); /*- * Initialize critical variables, including input buffer watermarks. @@ -2306,7 +2300,7 @@ siosetwater(com, speed) com->ihighwater = ibuf + 3 * ibufsize / 4; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2320,7 +2314,6 @@ comstart(tp) bool_t started; #endif int unit; - critical_t savecrit; unit = DEV_TO_UNIT(tp->t_dev); com = com_addr(unit); @@ -2331,7 +2324,7 @@ comstart(tp) started = FALSE; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (tp->t_state & TS_TTSTOP) { com->state &= ~CS_TTGO; @@ -2369,7 +2362,7 @@ comstart(tp) #endif } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { ttwwakeup(tp); splx(s); @@ -2388,7 +2381,7 @@ comstart(tp) sizeof com->obuf1); com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2409,7 +2402,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { #ifdef CyDebug @@ -2420,7 +2413,7 @@ comstart(tp) sizeof com->obuf2); com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2441,7 +2434,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } tp->t_state |= TS_BUSY; } @@ -2450,12 +2443,12 @@ comstart(tp) ++com->start_real; #endif #if 0 - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif ttwwakeup(tp); splx(s); @@ -2468,11 +2461,10 @@ comstop(tp, rw) { struct com_s *com; bool_t wakeup_etc; - critical_t savecrit; com = com_addr(DEV_TO_UNIT(tp->t_dev)); wakeup_etc = FALSE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (rw & FWRITE) { com->obufs[0].l_queued = FALSE; @@ -2497,7 +2489,7 @@ comstop(tp, rw) com->iptr = com->ibuf; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (wakeup_etc) wakeup(&com->etc); if (rw & FWRITE && com->etc == ETC_NONE) @@ -2513,7 +2505,6 @@ commctl(com, bits, how) { int mcr; int msr; - critical_t savecrit; if (how == DMGET) { if (com->channel_control & CD1400_CCR_RCVEN) @@ -2551,7 +2542,7 @@ commctl(com, bits, how) mcr |= com->mcr_dtr; if (bits & TIOCM_RTS) mcr |= com->mcr_rts; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); switch (how) { case DMSET: @@ -2571,7 +2562,7 @@ commctl(com, bits, how) break; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2633,13 +2624,11 @@ comwakeup(chan) com = com_addr(unit); if (com != NULL && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { - critical_t savecrit; - - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); siointr1(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } } #endif @@ -2659,14 +2648,13 @@ comwakeup(chan) for (errnum = 0; errnum < CE_NTYPES; ++errnum) { u_int delta; u_long total; - critical_t savecrit; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta == 0) continue; total = com->error_counts[errnum] += delta; @@ -2818,7 +2806,6 @@ cd_etc(com, etc) struct com_s *com; int etc; { - critical_t savecrit; /* * We can't change the hardware's ETC state while there are any @@ -2831,7 +2818,7 @@ cd_etc(com, etc) * for the tx to become empty so that the command is sure to be * executed soon after we issue it. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->etc == etc) goto wait; @@ -2842,7 +2829,7 @@ cd_etc(com, etc) && (com->etc == ETC_BREAK_ENDING || com->etc == ETC_BREAK_ENDED || com->etc == ETC_NONE))) { COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return; } com->etc = etc; @@ -2851,7 +2838,7 @@ cd_etc(com, etc) = (com->intr_enable & ~CD1400_SRER_TXRDY) | CD1400_SRER_TXMPTY); wait: COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); while (com->etc == etc && tsleep(&com->etc, TTIPRI | PCATCH, "cyetc", 0) == 0) continue; @@ -2865,7 +2852,6 @@ cd_getreg(com, reg) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; int val; @@ -2875,7 +2861,7 @@ cd_getreg(com, reg) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2883,7 +2869,7 @@ cd_getreg(com, reg) val = cd_inb(iobase, reg, cy_align); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (val); } @@ -2896,7 +2882,6 @@ cd_setreg(com, reg, val) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; @@ -2905,7 +2890,7 @@ cd_setreg(com, reg, val) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2913,7 +2898,7 @@ cd_setreg(com, reg, val) cd_outb(iobase, reg, cy_align, val); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } #ifdef CyDebug diff --git a/sys/i386/isa/npx.c b/sys/i386/isa/npx.c index ff8c4cf..34a822a 100644 --- a/sys/i386/isa/npx.c +++ b/sys/i386/isa/npx.c @@ -515,7 +515,7 @@ npxinit(control) * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -527,7 +527,7 @@ npxinit(control) if (PCPU_GET(curpcb) != NULL) fpusave(&PCPU_GET(curpcb)->pcb_save); start_emulating(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } /* @@ -539,10 +539,10 @@ npxexit(td) { critical_t savecrit; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #ifdef NPX_DEBUG if (npx_exists) { u_int masked_exceptions; @@ -762,7 +762,7 @@ npxtrap() PCPU_GET(fpcurthread), curthread, npx_exists); panic("npxtrap from nowhere"); } - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -783,7 +783,7 @@ npxtrap() GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -807,7 +807,7 @@ npxdna() PCPU_GET(fpcurthread), curthread); panic("npxdna"); } - s = critical_enter(); + s = cpu_critical_enter(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -829,7 +829,7 @@ npxdna() * first FPU instruction after a context switch. */ fpurstor(&PCPU_GET(curpcb)->pcb_save); - critical_exit(s); + cpu_critical_exit(s); return (1); } diff --git a/sys/ia64/ia64/db_interface.c b/sys/ia64/ia64/db_interface.c index 9a254da..3d576cd 100644 --- a/sys/ia64/ia64/db_interface.c +++ b/sys/ia64/ia64/db_interface.c @@ -336,7 +336,7 @@ kdb_trap(int vector, struct trapframe *regs) __asm __volatile("flushrs"); /* so we can look at them */ - s = critical_enter(); + s = cpu_critical_enter(); #if 0 db_printf("stopping %x\n", PCPU_GET(other_cpus)); @@ -359,7 +359,7 @@ kdb_trap(int vector, struct trapframe *regs) restart_cpus(stopped_cpus); #endif - critical_exit(s); + cpu_critical_exit(s); *regs = ddb_regs; diff --git a/sys/ia64/ia64/genassym.c b/sys/ia64/ia64/genassym.c index fbe60e5..5370da0 100644 --- a/sys/ia64/ia64/genassym.c +++ b/sys/ia64/ia64/genassym.c @@ -71,7 +71,6 @@ ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSE, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); ASSYM(MTX_UNOWNED, MTX_UNOWNED); ASSYM(TD_PROC, offsetof(struct thread, td_proc)); diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 34b129d..a144db6 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -577,7 +577,7 @@ pmap_invalidate_all(pmap_t pmap) KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), ("invalidating TLB for non-current pmap")); - psr = critical_enter(); + psr = cpu_critical_enter(); addr = pmap_ptc_e_base; for (i = 0; i < pmap_ptc_e_count1; i++) { for (j = 0; j < pmap_ptc_e_count2; j++) { @@ -586,7 +586,7 @@ pmap_invalidate_all(pmap_t pmap) } addr += pmap_ptc_e_stride1; } - critical_exit(psr); + cpu_critical_exit(psr); } static u_int32_t diff --git a/sys/ia64/ia64/sapic.c b/sys/ia64/ia64/sapic.c index 6af5cc0..e13b8d5 100644 --- a/sys/ia64/ia64/sapic.c +++ b/sys/ia64/ia64/sapic.c @@ -82,10 +82,10 @@ sapic_read_rte(struct sapic *sa, int which, { u_int32_t *p = (u_int32_t *) rte; critical_t c; - c = critical_enter(); + c = cpu_critical_enter(); p[0] = sapic_read(sa, SAPIC_RTE_BASE + 2*which); p[1] = sapic_read(sa, SAPIC_RTE_BASE + 2*which + 1); - critical_exit(c); + cpu_critical_exit(c); } #endif @@ -96,10 +96,10 @@ sapic_write_rte(struct sapic *sa, int which, { u_int32_t *p = (u_int32_t *) rte; critical_t c; - c = critical_enter(); + c = cpu_critical_enter(); sapic_write(sa, SAPIC_RTE_BASE + 2*which, p[0]); sapic_write(sa, SAPIC_RTE_BASE + 2*which + 1, p[1]); - critical_exit(c); + cpu_critical_exit(c); } struct sapic * diff --git a/sys/ia64/include/cpufunc.h b/sys/ia64/include/cpufunc.h index 45390e9..c2d074f 100644 --- a/sys/ia64/include/cpufunc.h +++ b/sys/ia64/include/cpufunc.h @@ -34,6 +34,8 @@ #include <sys/types.h> #include <machine/ia64_cpu.h> +#define CRITICAL_FORK (ia64_get_psr() |= IA64_PSR_I) + #ifdef __GNUC__ static __inline void @@ -283,7 +285,7 @@ enable_intr(void) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t psr; @@ -293,7 +295,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t psr) +cpu_critical_exit(critical_t psr) { __asm __volatile ("mov psr.l=%0;; srlz.d" :: "r" (psr)); } diff --git a/sys/ia64/include/mutex.h b/sys/ia64/include/mutex.h index b14d7ae..96d8f3d 100644 --- a/sys/ia64/include/mutex.h +++ b/sys/ia64/include/mutex.h @@ -32,17 +32,7 @@ #ifndef _MACHINE_MUTEX_H_ #define _MACHINE_MUTEX_H_ -#include <machine/ia64_cpu.h> - -#ifndef LOCORE - -#ifdef _KERNEL - -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= IA64_PSR_I; while (0) - -#endif /* _KERNEL */ - -#else /* !LOCORE */ +#ifdef LOCORE /* * Simple assembly macros to get and release non-recursive spin locks diff --git a/sys/ia64/include/profile.h b/sys/ia64/include/profile.h index 038c805..8d93c5c 100644 --- a/sys/ia64/include/profile.h +++ b/sys/ia64/include/profile.h @@ -100,9 +100,9 @@ _mcount: \n\ * The following two macros do splhigh and splx respectively. */ #define MCOUNT_ENTER(s) \n\ - _c = critical_enter() + _c = cpu_critical_enter() #define MCOUNT_EXIT(s) \n\ - (void)critical_exit(_c) + cpu_critical_exit(_c) #define MCOUNT_DECL(s) critical_t c; #ifdef GUPROF struct gmonparam; diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index bb52a34..bc03078 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -467,9 +467,6 @@ again: if (p1->p_sflag & PS_PROFIL) startprofclock(p2); mtx_unlock_spin(&sched_lock); - /* - * We start off holding one spinlock after fork: sched_lock. - */ PROC_LOCK(p1); p2->p_ucred = crhold(p1->p_ucred); p2->p_thread.td_ucred = crhold(p2->p_ucred); /* XXXKSE */ @@ -766,10 +763,8 @@ fork_exit(callout, arg, frame) */ sched_lock.mtx_lock = (uintptr_t)td; sched_lock.mtx_recurse = 0; - /* - * XXX: We really shouldn't have to do this. - */ - mtx_intr_enable(&sched_lock); + td->td_critnest = 1; + td->td_savecrit = CRITICAL_FORK; CTR3(KTR_PROC, "fork_exit: new proc %p (pid %d, %s)", p, p->p_pid, p->p_comm); if (PCPU_GET(switchtime.tv_sec) == 0) diff --git a/sys/kern/kern_idle.c b/sys/kern/kern_idle.c index 92e5cb3..4dee96d 100644 --- a/sys/kern/kern_idle.c +++ b/sys/kern/kern_idle.c @@ -47,8 +47,10 @@ idle_setup(void *dummy) error = kthread_create(idle_proc, NULL, &p, RFSTOPPED | RFHIGHPID, "idle: cpu%d", pc->pc_cpuid); pc->pc_idlethread = &p->p_thread; - if (pc->pc_curthread == NULL) + if (pc->pc_curthread == NULL) { pc->pc_curthread = pc->pc_idlethread; + pc->pc_idlethread->td_critnest = 0; + } #else error = kthread_create(idle_proc, NULL, &p, RFSTOPPED | RFHIGHPID, "idle"); diff --git a/sys/kern/kern_ktr.c b/sys/kern/kern_ktr.c index 309d6e3..596a724 100644 --- a/sys/kern/kern_ktr.c +++ b/sys/kern/kern_ktr.c @@ -133,9 +133,9 @@ ktr_tracepoint(u_int mask, const char *format, u_long arg1, u_long arg2, td = curthread; if (td->td_inktr) return; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (((1 << KTR_CPU) & ktr_cpumask) == 0) { - critical_exit(savecrit); + cpu_critical_exit(savecrit); return; } td->td_inktr++; @@ -145,7 +145,7 @@ ktr_tracepoint(u_int mask, const char *format, u_long arg1, u_long arg2, } while (atomic_cmpset_rel_int(&ktr_idx, saveindex, newindex) == 0); entry = &ktr_buf[saveindex]; entry->ktr_cpu = KTR_CPU; - critical_exit(savecrit); + cpu_critical_exit(savecrit); nanotime(&entry->ktr_tv); #ifdef KTR_EXTEND entry->ktr_filename = filename; diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index d4809af..852b570 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -427,8 +427,7 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) * is handled inline. */ void -_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, - int line) +_mtx_lock_spin(struct mtx *m, int opts, const char *file, int line) { int i = 0; @@ -440,7 +439,7 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, break; /* Give interrupts a chance while we spin. */ - critical_exit(mtx_crit); + critical_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 1000000) continue; @@ -454,10 +453,9 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, panic("spin lock %s held by %p for > 5 seconds", m->mtx_object.lo_name, (void *)m->mtx_lock); } - mtx_crit = critical_enter(); + critical_enter(); } - m->mtx_savecrit = mtx_crit; if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c index 65054df..1a2afa4 100644 --- a/sys/kern/kern_switch.c +++ b/sys/kern/kern_switch.c @@ -69,6 +69,31 @@ setrunqueue(struct thread *td) runq_add(&runq, td->td_kse); } +/* Critical sections that prevent preemption. */ +void +critical_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_critnest == 0) + td->td_savecrit = cpu_critical_enter(); + td->td_critnest++; +} + +void +critical_exit(void) +{ + struct thread *td; + + td = curthread; + if (td->td_critnest == 1) { + td->td_critnest = 0; + cpu_critical_exit(td->td_savecrit); + } else + td->td_critnest--; +} + /* * Clear the status bit of the queue corresponding to priority level pri, * indicating that it is empty. diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 7034c0a..fce470f 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -698,7 +698,6 @@ mi_switch() #if 0 register struct rlimit *rlim; #endif - critical_t sched_crit; u_int sched_nest; mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED); @@ -773,14 +772,12 @@ mi_switch() PCPU_SET(switchtime, new_switchtime); CTR3(KTR_PROC, "mi_switch: old proc %p (pid %d, %s)", p, p->p_pid, p->p_comm); - sched_crit = sched_lock.mtx_savecrit; sched_nest = sched_lock.mtx_recurse; td->td_lastcpu = td->td_kse->ke_oncpu; td->td_kse->ke_oncpu = NOCPU; td->td_kse->ke_flags &= ~KEF_NEEDRESCHED; cpu_switch(); td->td_kse->ke_oncpu = PCPU_GET(cpuid); - sched_lock.mtx_savecrit = sched_crit; sched_lock.mtx_recurse = sched_nest; sched_lock.mtx_lock = (uintptr_t)td; CTR3(KTR_PROC, "mi_switch: new proc %p (pid %d, %s)", p, p->p_pid, diff --git a/sys/kern/subr_prof.c b/sys/kern/subr_prof.c index 67629d4..8808651 100644 --- a/sys/kern/subr_prof.c +++ b/sys/kern/subr_prof.c @@ -90,7 +90,6 @@ kmupetext(uintfptr_t nhighpc) struct gmonparam np; /* slightly large */ struct gmonparam *p = &_gmonparam; char *cp; - critical_t savecrit; GIANT_REQUIRED; bcopy(p, &np, sizeof(*p)); @@ -127,7 +126,7 @@ kmupetext(uintfptr_t nhighpc) np.mcount_count = &KCOUNT(&np, PC_TO_I(&np, mcount)); np.mexitcount_count = &KCOUNT(&np, PC_TO_I(&np, mexitcount)); #endif - savecrit = critical_enter(); + critical_enter(); bcopy(p->tos, np.tos, p->tossize); bzero((char *)np.tos + p->tossize, np.tossize - p->tossize); bcopy(p->kcount, np.kcount, p->kcountsize); @@ -137,7 +136,7 @@ kmupetext(uintfptr_t nhighpc) bzero((char *)np.froms + p->fromssize, np.fromssize - p->fromssize); cp = (char *)p->tos; bcopy(&np, p, sizeof(*p)); - critical_exit(savecrit); + critical_exit(); free(cp, M_GPROF); } @@ -156,7 +155,6 @@ kmstartup(dummy) int nullfunc_loop_overhead; int nullfunc_loop_profiled_time; uintfptr_t tmp_addr; - critical_t savecrit; #endif /* @@ -195,7 +193,7 @@ kmstartup(dummy) * Disable interrupts to avoid interference while we calibrate * things. */ - savecrit = critical_enter(); + critical_enter(); /* * Determine overheads. @@ -249,7 +247,7 @@ kmstartup(dummy) p->state = GMON_PROF_OFF; stopguprof(p); - critical_exit(savecrit); + critical_exit(); nullfunc_loop_profiled_time = 0; for (tmp_addr = (uintfptr_t)nullfunc_loop_profiled; diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index a899576..6f17f8f 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -141,9 +141,9 @@ ast(framep) panic("Returning to user mode with mutex(s) held"); #endif mtx_assert(&Giant, MA_NOTOWNED); - s = critical_enter(); + s = cpu_critical_enter(); while ((ke->ke_flags & (KEF_ASTPENDING | KEF_NEEDRESCHED)) != 0) { - critical_exit(s); + cpu_critical_exit(s); td->td_frame = framep; /* * This updates the p_sflag's for the checks below in one @@ -195,13 +195,13 @@ ast(framep) crfree(td->td_ucred); mtx_unlock(&Giant); td->td_ucred = NULL; - s = critical_enter(); + s = cpu_critical_enter(); } mtx_assert(&Giant, MA_NOTOWNED); /* * We need to keep interrupts disabled so that if any further AST's * come in, the interrupt they come in on will be delayed until we * finish returning to userland. We assume that the return to userland - * will perform the equivalent of critical_exit(). + * will perform the equivalent of cpu_critical_exit(). */ } diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index d4809af..852b570 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -427,8 +427,7 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) * is handled inline. */ void -_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, - int line) +_mtx_lock_spin(struct mtx *m, int opts, const char *file, int line) { int i = 0; @@ -440,7 +439,7 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, break; /* Give interrupts a chance while we spin. */ - critical_exit(mtx_crit); + critical_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 1000000) continue; @@ -454,10 +453,9 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, panic("spin lock %s held by %p for > 5 seconds", m->mtx_object.lo_name, (void *)m->mtx_lock); } - mtx_crit = critical_enter(); + critical_enter(); } - m->mtx_savecrit = mtx_crit; if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index 454fd55..5e4dc61 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -252,7 +252,6 @@ static struct mtx all_mtx = { { NULL }, /* mtx_object.lo_list */ NULL }, /* mtx_object.lo_witness */ MTX_UNOWNED, 0, /* mtx_lock, mtx_recurse */ - 0, /* mtx_savecrit */ TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL } /* mtx_contested */ }; @@ -836,7 +835,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line) instance->li_flags--; goto out; } - s = critical_enter(); + s = cpu_critical_enter(); CTR4(KTR_WITNESS, "%s: pid %d removed %s from lle[%d]", __func__, td->td_proc->p_pid, @@ -846,7 +845,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line) for (j = i; j < (*lock_list)->ll_count; j++) (*lock_list)->ll_children[j] = (*lock_list)->ll_children[j + 1]; - critical_exit(s); + cpu_critical_exit(s); if ((*lock_list)->ll_count == 0) { lle = *lock_list; *lock_list = lle->ll_next; @@ -896,7 +895,7 @@ witness_sleep(int check_only, struct lock_object *lock, const char *file, /* * Preemption bad because we need PCPU_PTR(spinlocks) to not change. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); td = curthread; lock_list = &td->td_sleeplocks; again: @@ -931,7 +930,7 @@ again: if (witness_ddb && n) Debugger(__func__); #endif /* DDB */ - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (n); } @@ -1360,9 +1359,9 @@ witness_list(struct thread *td) * Preemption bad because we need PCPU_PTR(spinlocks) to not * change. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); nheld += witness_list_locks(PCPU_PTR(spinlocks)); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } return (nheld); } diff --git a/sys/powerpc/include/cpufunc.h b/sys/powerpc/include/cpufunc.h index 0fd0dd0..25f37d3 100644 --- a/sys/powerpc/include/cpufunc.h +++ b/sys/powerpc/include/cpufunc.h @@ -35,6 +35,8 @@ #include <machine/psl.h> +#define CRITICAL_FORK (mfmsr() |= PSL_EE) + #ifdef __GNUC__ static __inline void @@ -114,7 +116,7 @@ save_intr(void) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { return ((critical_t)save_intr()); @@ -128,7 +130,7 @@ restore_intr(unsigned int msr) } static __inline void -critical_exit(critical_t msr) +cpu_critical_exit(critical_t msr) { return (restore_intr((unsigned int)msr)); diff --git a/sys/powerpc/include/mutex.h b/sys/powerpc/include/mutex.h index 0aeaa74..0b7fe26 100644 --- a/sys/powerpc/include/mutex.h +++ b/sys/powerpc/include/mutex.h @@ -32,22 +32,7 @@ #ifndef _MACHINE_MUTEX_H_ #define _MACHINE_MUTEX_H_ -#ifndef LOCORE - -#ifdef _KERNEL - -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= PSL_EE; while (0) - -/* - * Assembly macros (for internal use only) - *-------------------------------------------------------------------------- - */ - -#define _V(x) __STRING(x) - -#endif /* _KERNEL */ - -#else /* !LOCORE */ +#ifdef LOCORE /* * Simple assembly macros to get and release non-recursive spin locks diff --git a/sys/powerpc/powerpc/genassym.c b/sys/powerpc/powerpc/genassym.c index 7ab6e77..c212c6b 100644 --- a/sys/powerpc/powerpc/genassym.c +++ b/sys/powerpc/powerpc/genassym.c @@ -66,7 +66,6 @@ ASSYM(PC_SWITCHTIME, offsetof(struct pcpu, pc_switchtime)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSECNT, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); ASSYM(PM_KERNELSR, offsetof(struct pmap, pm_sr[KERNEL_SR])); ASSYM(PM_USERSR, offsetof(struct pmap, pm_sr[USER_SR])); diff --git a/sys/sparc64/include/cpufunc.h b/sys/sparc64/include/cpufunc.h index 18f06b5..f9739ad 100644 --- a/sys/sparc64/include/cpufunc.h +++ b/sys/sparc64/include/cpufunc.h @@ -156,6 +156,8 @@ STNC_GEN(u_long, stxa); : : "r" (val), "rI" (xor)); \ } while (0) +#define CRITICAL_FORK (0) + static __inline void breakpoint(void) { @@ -163,7 +165,7 @@ breakpoint(void) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t pil; @@ -173,7 +175,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t pil) +cpu_critical_exit(critical_t pil) { wrpr(pil, pil, 0); } diff --git a/sys/sparc64/include/mutex.h b/sys/sparc64/include/mutex.h index 293f8d6..c9b2e1d 100644 --- a/sys/sparc64/include/mutex.h +++ b/sys/sparc64/include/mutex.h @@ -29,11 +29,4 @@ #ifndef _MACHINE_MUTEX_H_ #define _MACHINE_MUTEX_H_ -static __inline void -mtx_intr_enable(struct mtx *mtx) -{ - - mtx->mtx_savecrit = 0; -} - #endif /* !_MACHINE_MUTEX_H_ */ diff --git a/sys/sparc64/sparc64/intr_machdep.c b/sys/sparc64/sparc64/intr_machdep.c index 3b39f9a..ca28a60 100644 --- a/sys/sparc64/sparc64/intr_machdep.c +++ b/sys/sparc64/sparc64/intr_machdep.c @@ -99,7 +99,7 @@ intr_dequeue(struct trapframe *tf) u_long next; u_long tail; - crit = critical_enter(); + crit = cpu_critical_enter(); iq = PCPU_PTR(iq); for (head = iq->iq_head;; head = next) { for (tail = iq->iq_tail; tail != head;) { @@ -115,7 +115,7 @@ intr_dequeue(struct trapframe *tf) if (head == next) break; } - critical_exit(crit); + cpu_critical_exit(crit); } void diff --git a/sys/sys/_mutex.h b/sys/sys/_mutex.h index 82f3648..ef5de76 100644 --- a/sys/sys/_mutex.h +++ b/sys/sys/_mutex.h @@ -39,7 +39,6 @@ struct mtx { struct lock_object mtx_object; /* Common lock properties. */ volatile uintptr_t mtx_lock; /* owner (and state for sleep locks) */ volatile u_int mtx_recurse; /* number of recursive holds */ - critical_t mtx_savecrit; /* saved flags (for spin locks) */ TAILQ_HEAD(, thread) mtx_blocked; /* threads blocked on this lock */ LIST_ENTRY(mtx) mtx_contested; /* list of all contested locks */ }; diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index d581973..a4eb018 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -101,8 +101,7 @@ void mtx_init(struct mtx *m, const char *description, int opts); void mtx_destroy(struct mtx *m); void _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line); void _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line); -void _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, - const char *file, int line); +void _mtx_lock_spin(struct mtx *m, int opts, const char *file, int line); void _mtx_unlock_spin(struct mtx *m, int opts, const char *file, int line); int _mtx_trylock(struct mtx *m, int opts, const char *file, int line); void _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line); @@ -160,16 +159,13 @@ void mtx_unlock_giant(int s); */ #ifndef _get_spin_lock #define _get_spin_lock(mp, tid, opts, file, line) do { \ - critical_t _mtx_crit; \ - _mtx_crit = critical_enter(); \ + critical_enter(); \ if (!_obtain_lock((mp), (tid))) { \ if ((mp)->mtx_lock == (uintptr_t)(tid)) \ (mp)->mtx_recurse++; \ else \ - _mtx_lock_spin((mp), (opts), _mtx_crit, (file), \ - (line)); \ - } else \ - (mp)->mtx_savecrit = _mtx_crit; \ + _mtx_lock_spin((mp), (opts), (file), (line)); \ + } \ } while (0) #endif @@ -189,16 +185,18 @@ void mtx_unlock_giant(int s); * a function call would be too expensive (at least on some architectures). * Since spin locks are not _too_ common, inlining this code is not too big * a deal. + * + * Since we always perform a critical_enter() when attempting to acquire a + * spin lock, we need to always perform a matching critical_exit() when + * releasing a spin lock. This includes the recursion cases. */ #ifndef _rel_spin_lock #define _rel_spin_lock(mp) do { \ - critical_t _mtx_crit = (mp)->mtx_savecrit; \ if (mtx_recursed((mp))) \ (mp)->mtx_recurse--; \ - else { \ + else \ _release_lock_quick((mp)); \ - critical_exit(_mtx_crit); \ - } \ + critical_exit(); \ } while (0) #endif diff --git a/sys/sys/proc.h b/sys/sys/proc.h index beab316..ebf6d36 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -275,6 +275,8 @@ struct thread { struct trapframe *td_frame; /* (k) */ struct vm_object *td_kstack_obj;/* (a) Kstack object. */ vm_offset_t td_kstack; /* Kernel VA of kstack. */ + u_int td_critnest; /* (k) Critical section nest level. */ + critical_t td_savecrit; /* (k) Saved critical section state. */ }; /* diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 3544b1f..70e99bd 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -121,6 +121,8 @@ void *phashinit __P((int count, struct malloc_type *type, u_long *nentries)); void cpu_boot __P((int)); void cpu_rootconf __P((void)); +void critical_enter __P((void)); +void critical_exit __P((void)); void init_param1 __P((void)); void init_param2 __P((int physpages)); void tablefull __P((const char *)); |
