diff options
61 files changed, 376 insertions, 502 deletions
diff --git a/sys/alpha/alpha/db_interface.c b/sys/alpha/alpha/db_interface.c index fea763d..041b0b9 100644 --- a/sys/alpha/alpha/db_interface.c +++ b/sys/alpha/alpha/db_interface.c @@ -192,7 +192,7 @@ kdb_trap(a0, a1, a2, entry, regs) ddb_regs = *regs; - s = critical_enter(); + s = cpu_critical_enter(); #ifdef SMP #ifdef DIAGNOSTIC @@ -219,7 +219,7 @@ kdb_trap(a0, a1, a2, entry, regs) restart_cpus(stopped_cpus); #endif - critical_exit(s); + cpu_critical_exit(s); *regs = ddb_regs; diff --git a/sys/alpha/alpha/genassym.c b/sys/alpha/alpha/genassym.c index 8d68d73..e989675 100644 --- a/sys/alpha/alpha/genassym.c +++ b/sys/alpha/alpha/genassym.c @@ -76,7 +76,6 @@ ASSYM(PC_IDLEPCBPHYS, offsetof(struct pcpu, pc_idlepcbphys)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSE, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); ASSYM(MTX_UNOWNED, MTX_UNOWNED); ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); diff --git a/sys/alpha/alpha/interrupt.c b/sys/alpha/alpha/interrupt.c index 29bb74f..3143856 100644 --- a/sys/alpha/alpha/interrupt.c +++ b/sys/alpha/alpha/interrupt.c @@ -99,13 +99,13 @@ interrupt(a0, a1, a2, framep) * Find our per-cpu globals. */ #ifdef SMP - s = critical_enter(); + s = cpu_critical_enter(); #endif pcpup = (struct pcpu *) alpha_pal_rdval(); td = curthread; #ifdef SMP td->td_md.md_kernnest++; - critical_exit(s); + cpu_critical_exit(s); #endif atomic_add_int(&td->td_intr_nesting_level, 1); #ifndef KSTACK_GUARD diff --git a/sys/alpha/alpha/machdep.c b/sys/alpha/alpha/machdep.c index ef763a9..a6f181c 100644 --- a/sys/alpha/alpha/machdep.c +++ b/sys/alpha/alpha/machdep.c @@ -2007,11 +2007,11 @@ alpha_fpstate_check(struct thread *td) #ifndef SMP critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (td->td_pcb->pcb_hw.apcb_flags & ALPHA_PCB_FLAGS_FEN) if (td != PCPU_GET(fpcurthread)) panic("alpha_check_fpcurthread: bogus"); - critical_exit(s); + cpu_critical_exit(s); #endif } @@ -2033,7 +2033,7 @@ alpha_fpstate_save(struct thread *td, int write) { critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (td != NULL && td == PCPU_GET(fpcurthread)) { /* * If curthread != fpcurthread, then we need to enable FEN @@ -2068,7 +2068,7 @@ alpha_fpstate_save(struct thread *td, int write) alpha_pal_wrfen(0); } } - critical_exit(s); + cpu_critical_exit(s); } /* @@ -2081,7 +2081,7 @@ alpha_fpstate_drop(struct thread *td) { critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) { if (td == curthread) { /* @@ -2097,7 +2097,7 @@ alpha_fpstate_drop(struct thread *td) } PCPU_SET(fpcurthread, NULL); } - critical_exit(s); + cpu_critical_exit(s); } /* @@ -2112,7 +2112,7 @@ alpha_fpstate_switch(struct thread *td) /* * Enable FEN so that we can access the fp registers. */ - s = critical_enter(); + s = cpu_critical_enter(); alpha_pal_wrfen(1); if (PCPU_GET(fpcurthread)) { /* @@ -2139,7 +2139,7 @@ alpha_fpstate_switch(struct thread *td) } td->td_md.md_flags |= MDP_FPUSED; - critical_exit(s); + cpu_critical_exit(s); } /* diff --git a/sys/alpha/alpha/prom.c b/sys/alpha/alpha/prom.c index 54d913a..d4dd267 100644 --- a/sys/alpha/alpha/prom.c +++ b/sys/alpha/alpha/prom.c @@ -193,7 +193,7 @@ enter_prom() pt_entry_t *lev1map; critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); if (!prom_mapped) { #ifdef SIMOS @@ -232,7 +232,7 @@ leave_prom(s) lev1map[0] = saved_pte[0]; /* XXX */ prom_cache_sync(); /* XXX */ } - critical_exit(s); + cpu_critical_exit(s); } static void @@ -272,7 +272,7 @@ prom_halt(halt) /* * Turn off interrupts, for sanity. */ - critical_enter(); + cpu_critical_enter(); /* * Set "boot request" part of the CPU state depending on what diff --git a/sys/alpha/alpha/trap.c b/sys/alpha/alpha/trap.c index 4c7694d..99278fa 100644 --- a/sys/alpha/alpha/trap.c +++ b/sys/alpha/alpha/trap.c @@ -270,13 +270,13 @@ trap(a0, a1, a2, entry, framep) * Find our per-cpu globals. */ #ifdef SMP - s = critical_enter(); + s = cpu_critical_enter(); #endif pcpup = (struct pcpu *) alpha_pal_rdval(); td = curthread; #ifdef SMP td->td_md.md_kernnest++; - critical_exit(s); + cpu_critical_exit(s); #endif p = td->td_proc; @@ -672,13 +672,13 @@ syscall(code, framep) * Find our per-cpu globals. */ #ifdef SMP - s = critical_enter(); + s = cpu_critical_enter(); #endif pcpup = (struct pcpu *) alpha_pal_rdval(); td = curthread; #ifdef SMP td->td_md.md_kernnest++; - critical_exit(s); + cpu_critical_exit(s); #endif p = td->td_proc; diff --git a/sys/alpha/include/cpufunc.h b/sys/alpha/include/cpufunc.h index 51a1fdc..dbee911 100644 --- a/sys/alpha/include/cpufunc.h +++ b/sys/alpha/include/cpufunc.h @@ -35,6 +35,8 @@ #include <machine/chipset.h> #include <machine/alpha_cpu.h> +#define CRITICAL_FORK (ALPHA_PSL_IPL_0) + #ifdef __GNUC__ static __inline void @@ -46,13 +48,13 @@ breakpoint(void) #endif static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { return (alpha_pal_swpipl(ALPHA_PSL_IPL_MCES)); } static __inline void -critical_exit(critical_t ipl) +cpu_critical_exit(critical_t ipl) { alpha_pal_swpipl(ipl); } diff --git a/sys/alpha/include/mutex.h b/sys/alpha/include/mutex.h index 970049b..7300c08 100644 --- a/sys/alpha/include/mutex.h +++ b/sys/alpha/include/mutex.h @@ -39,8 +39,6 @@ /* Global locks */ extern struct mtx clock_lock; -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit = ALPHA_PSL_IPL_0; while (0) - #endif /* _KERNEL */ #else /* !LOCORE */ diff --git a/sys/alpha/pci/cia.c b/sys/alpha/pci/cia.c index c2bcafc..09436e3 100644 --- a/sys/alpha/pci/cia.c +++ b/sys/alpha/pci/cia.c @@ -156,12 +156,12 @@ cia_swiz_set_hae_mem(void *arg, u_int32_t pa) u_int32_t msb = pa & REG1; critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); cia_hae_mem = (cia_hae_mem & ~REG1) | msb; REGVAL(CIA_CSR_HAE_MEM) = cia_hae_mem; alpha_mb(); cia_hae_mem = REGVAL(CIA_CSR_HAE_MEM); - critical_exit(s); + cpu_critical_exit(s); } return pa & ~REG1; } @@ -226,7 +226,7 @@ cia_sgmap_invalidate_pyxis(void) int i; critical_t s; - s = critical_enter(); + s = cpu_critical_enter(); /* * Put the Pyxis into PCI loopback mode. @@ -257,7 +257,7 @@ cia_sgmap_invalidate_pyxis(void) REGVAL(CIA_CSR_CTRL) = ctrl; alpha_mb(); - critical_exit(s); + cpu_critical_exit(s); } static void diff --git a/sys/alpha/pci/t2.c b/sys/alpha/pci/t2.c index c8ade0c..d9d0b9d 100644 --- a/sys/alpha/pci/t2.c +++ b/sys/alpha/pci/t2.c @@ -118,14 +118,14 @@ t2_set_hae_mem(void *arg, u_int32_t pa) msb = pa & 0xf8000000; pa -= msb; msb >>= 27; /* t2 puts high bits in the bottom of the register */ - s = critical_enter(); + s = cpu_critical_enter(); if (msb != t2_hae_mem[hose]) { t2_hae_mem[hose] = msb; t2_csr[hose]->hae0_1 = t2_hae_mem[hose]; alpha_mb(); t2_hae_mem[hose] = t2_csr[hose]->hae0_1; } - critical_exit(s); + cpu_critical_exit(s); } return pa; } diff --git a/sys/alpha/pci/t2_pci.c b/sys/alpha/pci/t2_pci.c index 87a3dca..b50555f 100644 --- a/sys/alpha/pci/t2_pci.c +++ b/sys/alpha/pci/t2_pci.c @@ -92,7 +92,7 @@ t2_pcib_maxslots(device_t dev) #define T2_TYPE1_SETUP(b,s,old_hae3) if((b)) { \ do { \ - (s) = critical_enter(); \ + (s) = cpu_critical_enter(); \ (old_hae3) = REGVAL(T2_HAE0_3); \ alpha_mb(); \ REGVAL(T2_HAE0_3) = (old_hae3) | (1<<30); \ @@ -105,7 +105,7 @@ t2_pcib_maxslots(device_t dev) alpha_mb(); \ REGVAL(T2_HAE0_3) = (old_hae3); \ alpha_mb(); \ - critical_exit((s)); \ + cpu_critical_exit((s)); \ } while(0); \ } diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c index 9708cad..000085b 100644 --- a/sys/amd64/amd64/db_interface.c +++ b/sys/amd64/amd64/db_interface.c @@ -324,10 +324,10 @@ Debugger(msg) return; if (atomic_cmpset_acq_int(&in_Debugger, 0, 1)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); db_printf("Debugger(\"%s\")\n", msg); breakpoint(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); atomic_store_rel_int(&in_Debugger, 0); } } diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c index ff8c4cf..34a822a 100644 --- a/sys/amd64/amd64/fpu.c +++ b/sys/amd64/amd64/fpu.c @@ -515,7 +515,7 @@ npxinit(control) * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -527,7 +527,7 @@ npxinit(control) if (PCPU_GET(curpcb) != NULL) fpusave(&PCPU_GET(curpcb)->pcb_save); start_emulating(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } /* @@ -539,10 +539,10 @@ npxexit(td) { critical_t savecrit; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #ifdef NPX_DEBUG if (npx_exists) { u_int masked_exceptions; @@ -762,7 +762,7 @@ npxtrap() PCPU_GET(fpcurthread), curthread, npx_exists); panic("npxtrap from nowhere"); } - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -783,7 +783,7 @@ npxtrap() GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -807,7 +807,7 @@ npxdna() PCPU_GET(fpcurthread), curthread); panic("npxdna"); } - s = critical_enter(); + s = cpu_critical_enter(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -829,7 +829,7 @@ npxdna() * first FPU instruction after a context switch. */ fpurstor(&PCPU_GET(curpcb)->pcb_save); - critical_exit(s); + cpu_critical_exit(s); return (1); } diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index 59f3147..99fa502 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -205,7 +205,6 @@ ASSYM(VM86_FRAMESIZE, sizeof(struct vm86frame)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSECNT, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); #ifdef PC98 #include <machine/bus.h> diff --git a/sys/amd64/amd64/initcpu.c b/sys/amd64/amd64/initcpu.c index b32c786..8a9a2a5 100644 --- a/sys/amd64/amd64/initcpu.c +++ b/sys/amd64/amd64/initcpu.c @@ -646,7 +646,7 @@ enable_K5_wt_alloc(void) * a stepping of 4 or greater. */ if (((cpu_id & 0xf0) > 0) && ((cpu_id & 0x0f) > 3)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); msr = rdmsr(0x83); /* HWCR */ wrmsr(0x83, msr & !(0x10)); @@ -678,7 +678,7 @@ enable_K5_wt_alloc(void) msr=rdmsr(0x83); wrmsr(0x83, msr|0x10); /* enable write allocate */ - critical_exit(savecrit); + cpu_critical_exit(savecrit); } } diff --git a/sys/amd64/amd64/sys_machdep.c b/sys/amd64/amd64/sys_machdep.c index a88acc9..b7beea9 100644 --- a/sys/amd64/amd64/sys_machdep.c +++ b/sys/amd64/amd64/sys_machdep.c @@ -532,13 +532,13 @@ i386_set_ldt(td, args) } /* Fill in range */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); error = copyin(uap->descs, &((union descriptor *)(pldt->ldt_base))[uap->start], uap->num * sizeof(union descriptor)); if (!error) td->td_retval[0] = uap->start; - critical_exit(savecrit); + cpu_critical_exit(savecrit); return(error); } diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c index 322b5e7..4f1dab2 100644 --- a/sys/amd64/amd64/vm_machdep.c +++ b/sys/amd64/amd64/vm_machdep.c @@ -153,10 +153,10 @@ cpu_fork(td1, p2, flags) #ifdef DEV_NPX if (td1 == curthread) td1->td_pcb->pcb_gs = rgs(); - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (PCPU_GET(fpcurthread) == td1) npxsave(&td1->td_pcb->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #endif /* Point the pcb to the top of the stack */ diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h index 4a9300a..a4f57cb 100644 --- a/sys/amd64/include/cpufunc.h +++ b/sys/amd64/include/cpufunc.h @@ -41,6 +41,7 @@ #define _MACHINE_CPUFUNC_H_ #include <sys/cdefs.h> +#include <machine/psl.h> __BEGIN_DECLS #define readb(va) (*(volatile u_int8_t *) (va)) @@ -51,6 +52,8 @@ __BEGIN_DECLS #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) +#define CRITICAL_FORK (read_eflags() | PSL_I) + #ifdef __GNUC__ #ifdef SWTCH_OPTIM_STATS @@ -548,7 +551,7 @@ load_dr7(u_int sel) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t eflags; @@ -558,7 +561,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t eflags) +cpu_critical_exit(critical_t eflags) { write_eflags(eflags); } @@ -597,8 +600,8 @@ u_int rfs __P((void)); u_int rgs __P((void)); void load_fs __P((u_int sel)); void load_gs __P((u_int sel)); -critical_t critical_enter __P((void)); -void critical_exit __P((critical_t eflags)); +critical_t cpu_critical_enter __P((void)); +void cpu_critical_exit __P((critical_t eflags)); #endif /* __GNUC__ */ diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h index ae37b23..e68bce7 100644 --- a/sys/amd64/include/mutex.h +++ b/sys/amd64/include/mutex.h @@ -35,13 +35,10 @@ #ifndef LOCORE #ifdef _KERNEL -#include <machine/psl.h> /* Global locks */ extern struct mtx clock_lock; -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= PSL_I; while (0) - /* * Assembly macros (for internal use only) *------------------------------------------------------------------------------ @@ -246,51 +243,20 @@ extern struct mtx clock_lock; * locks) in the near future, however. */ #define MTX_LOCK_SPIN(lck, flags) \ - pushl %eax ; \ - pushl %ecx ; \ - pushl %ebx ; \ - movl $(MTX_UNOWNED) , %eax ; \ - movl PCPU(CURTHREAD), %ebx ; \ - pushfl ; \ - popl %ecx ; \ - cli ; \ - MPLOCKED cmpxchgl %ebx, lck+MTX_LOCK ; \ - jz 2f ; \ - cmpl lck+MTX_LOCK, %ebx ; \ - je 3f ; \ pushl $0 ; \ pushl $0 ; \ - pushl %ecx ; \ pushl $flags ; \ pushl $lck ; \ - call _mtx_lock_spin ; \ - addl $0x14, %esp ; \ - jmp 1f ; \ -3: movl lck+MTX_RECURSECNT, %ebx ; \ - incl %ebx ; \ - movl %ebx, lck+MTX_RECURSECNT ; \ - jmp 1f ; \ -2: movl %ecx, lck+MTX_SAVECRIT ; \ -1: popl %ebx ; \ - popl %ecx ; \ - popl %eax + call _mtx_lock_spin_flags ; \ + addl $0x10, %esp ; \ #define MTX_UNLOCK_SPIN(lck) \ - pushl %edx ; \ - pushl %eax ; \ - movl lck+MTX_SAVECRIT, %edx ; \ - movl lck+MTX_RECURSECNT, %eax ; \ - testl %eax, %eax ; \ - jne 2f ; \ - movl $(MTX_UNOWNED), %eax ; \ - xchgl %eax, lck+MTX_LOCK ; \ - pushl %edx ; \ - popfl ; \ - jmp 1f ; \ -2: decl %eax ; \ - movl %eax, lck+MTX_RECURSECNT ; \ -1: popl %eax ; \ - popl %edx + pushl $0 ; \ + pushl $0 ; \ + pushl $0 ; \ + pushl $lck ; \ + call _mtx_unlock_spin_flags ; \ + addl $0x10, %esp ; \ /* * XXX: These two are broken right now and need to be made to work for diff --git a/sys/amd64/isa/npx.c b/sys/amd64/isa/npx.c index ff8c4cf..34a822a 100644 --- a/sys/amd64/isa/npx.c +++ b/sys/amd64/isa/npx.c @@ -515,7 +515,7 @@ npxinit(control) * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -527,7 +527,7 @@ npxinit(control) if (PCPU_GET(curpcb) != NULL) fpusave(&PCPU_GET(curpcb)->pcb_save); start_emulating(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } /* @@ -539,10 +539,10 @@ npxexit(td) { critical_t savecrit; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #ifdef NPX_DEBUG if (npx_exists) { u_int masked_exceptions; @@ -762,7 +762,7 @@ npxtrap() PCPU_GET(fpcurthread), curthread, npx_exists); panic("npxtrap from nowhere"); } - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -783,7 +783,7 @@ npxtrap() GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -807,7 +807,7 @@ npxdna() PCPU_GET(fpcurthread), curthread); panic("npxdna"); } - s = critical_enter(); + s = cpu_critical_enter(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -829,7 +829,7 @@ npxdna() * first FPU instruction after a context switch. */ fpurstor(&PCPU_GET(curpcb)->pcb_save); - critical_exit(s); + cpu_critical_exit(s); return (1); } diff --git a/sys/dev/bktr/bktr_os.h b/sys/dev/bktr/bktr_os.h index 21d3f60..1740723 100644 --- a/sys/dev/bktr/bktr_os.h +++ b/sys/dev/bktr/bktr_os.h @@ -61,9 +61,9 @@ void free_bktr_mem(bktr_ptr_t, bus_dmamap_t, vm_offset_t); /************************************/ #if defined(__FreeBSD__) #if (__FreeBSD_version >=500000) -#define DECLARE_INTR_MASK(s) critical_t s -#define DISABLE_INTR(s) s = critical_enter() -#define ENABLE_INTR(s) critical_exit(s) +#define DECLARE_INTR_MASK(s) /* no need to declare 's' */ +#define DISABLE_INTR(s) critical_enter() +#define ENABLE_INTR(s) critical_exit() #else #define DECLARE_INTR_MASK(s) intrmask_t s #define DISABLE_INTR(s) s=spltty() diff --git a/sys/dev/cy/cy.c b/sys/dev/cy/cy.c index 20316ad..0510e72 100644 --- a/sys/dev/cy/cy.c +++ b/sys/dev/cy/cy.c @@ -356,7 +356,7 @@ static int cd_getreg __P((struct com_s *com, int reg)); static void cd_setreg __P((struct com_s *com, int reg, int val)); static timeout_t siodtrwakeup; static void comhardclose __P((struct com_s *com)); -static void sioinput __P((struct com_s *com, critical_t *savecrit)); +static void sioinput __P((struct com_s *com)); #if 0 static void siointr1 __P((struct com_s *com)); #endif @@ -673,7 +673,6 @@ sioopen(dev, flag, mode, td) int s; struct tty *tp; int unit; - critical_t savecrit; mynor = minor(dev); unit = MINOR_TO_UNIT(mynor); @@ -775,7 +774,7 @@ open_top: } } - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); @@ -784,7 +783,7 @@ open_top: outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #else /* !0 */ /* * Flush fifos. This requires a full channel reset which @@ -795,7 +794,7 @@ open_top: CD1400_CCR_CMDRESET | CD1400_CCR_CHANRESET); cd1400_channel_cmd(com, com->channel_control); - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->prev_modem_status = com->last_modem_status = cd_getreg(com, CD1400_MSVR2); @@ -803,7 +802,7 @@ open_top: com->intr_enable = CD1400_SRER_MDMCH | CD1400_SRER_RXDATA); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif /* 0 */ /* * Handle initial DCD. Callout devices get a fake initial @@ -886,7 +885,6 @@ comhardclose(com) int s; struct tty *tp; int unit; - critical_t savecrit; unit = com->unit; iobase = com->iobase; @@ -900,12 +898,12 @@ comhardclose(com) outb(iobase + com_cfcr, com->cfcr_image &= ~CFCR_SBREAK); #else /* XXX */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->etc = ETC_NONE; cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); cd1400_channel_cmd(com, CD1400_CCR_CMDRESET | CD1400_CCR_FTF); #endif @@ -913,11 +911,11 @@ comhardclose(com) #if 0 outb(iobase + com_ier, 0); #else - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); cd_setreg(com, CD1400_SRER, com->intr_enable = 0); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif tp = com->tp; if ((tp->t_cflag & HUPCL) @@ -1013,9 +1011,8 @@ siodtrwakeup(chan) * b) needs to return with COM_LOCK() held. */ static void -sioinput(com, savecrit) +sioinput(com) struct com_s *com; - critical_t *savecrit; { u_char *buf; int incc; @@ -1046,7 +1043,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); incc = com->iptr - buf; if (tp->t_rawq.c_cc + incc > tp->t_ihiwat && (com->state & CS_RTS_IFLOW @@ -1067,7 +1064,7 @@ sioinput(com, savecrit) tp->t_lflag &= ~FLUSHO; comstart(tp); } - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } else { @@ -1078,7 +1075,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); line_status = buf[com->ierroff]; recv_data = *buf++; if (line_status @@ -1093,7 +1090,7 @@ sioinput(com, savecrit) recv_data |= TTY_PE; } (*linesw[tp->t_line].l_rint)(recv_data, tp); - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } @@ -1767,7 +1764,6 @@ static void siopoll(void *arg) { int unit; - critical_t savecrit; #ifdef CyDebug ++cy_timeouts; @@ -1790,7 +1786,7 @@ repeat: * (actually never opened devices) so that we don't * loop. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; @@ -1800,7 +1796,7 @@ repeat: } com_events -= incc; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (incc != 0) log(LOG_DEBUG, "sio%d: %d events for device with no tp\n", @@ -1808,36 +1804,36 @@ repeat: continue; } if (com->iptr != com->ibuf) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (com->state & CS_CHECKMSR) { u_char delta_modem_status; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) (tp, com->prev_modem_status & MSR_DCD); } if (com->extra_state & CSE_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->extra_state &= ~CSE_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (!(com->state & CS_BUSY)) { tp->t_state &= ~TS_BUSY; ttwwakeup(com->tp); @@ -1849,12 +1845,12 @@ repeat: } } if (com->state & CS_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); (*linesw[tp->t_line].l_start)(tp); } if (com_events == 0) @@ -1883,7 +1879,6 @@ comparam(tp, t) u_char opt; int s; int unit; - critical_t savecrit; /* do historical conversions */ if (t->c_ispeed == 0) @@ -2031,14 +2026,14 @@ comparam(tp, t) if (cflag & CCTS_OFLOW) opt |= CD1400_COR2_CCTS_OFLOW; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (opt != com->cor[1]) { cor_change |= CD1400_CCR_COR2; cd_setreg(com, CD1400_COR2, com->cor[1] = opt); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); /* * set channel option register 3 - @@ -2159,7 +2154,7 @@ comparam(tp, t) * XXX should have done this long ago, but there is too much state * to change all atomically. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->state &= ~CS_TTGO; @@ -2227,7 +2222,7 @@ comparam(tp, t) } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); splx(s); comstart(tp); if (com->ibufold != NULL) { @@ -2246,7 +2241,6 @@ siosetwater(com, speed) u_char *ibuf; int ibufsize; struct tty *tp; - critical_t savecrit; /* * Make the buffer size large enough to handle a softtty interrupt @@ -2284,10 +2278,10 @@ siosetwater(com, speed) * Read current input buffer, if any. Continue with interrupts * disabled. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->iptr != com->ibuf) - sioinput(com, &savecrit); + sioinput(com); /*- * Initialize critical variables, including input buffer watermarks. @@ -2306,7 +2300,7 @@ siosetwater(com, speed) com->ihighwater = ibuf + 3 * ibufsize / 4; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2320,7 +2314,6 @@ comstart(tp) bool_t started; #endif int unit; - critical_t savecrit; unit = DEV_TO_UNIT(tp->t_dev); com = com_addr(unit); @@ -2331,7 +2324,7 @@ comstart(tp) started = FALSE; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (tp->t_state & TS_TTSTOP) { com->state &= ~CS_TTGO; @@ -2369,7 +2362,7 @@ comstart(tp) #endif } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { ttwwakeup(tp); splx(s); @@ -2388,7 +2381,7 @@ comstart(tp) sizeof com->obuf1); com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2409,7 +2402,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { #ifdef CyDebug @@ -2420,7 +2413,7 @@ comstart(tp) sizeof com->obuf2); com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2441,7 +2434,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } tp->t_state |= TS_BUSY; } @@ -2450,12 +2443,12 @@ comstart(tp) ++com->start_real; #endif #if 0 - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif ttwwakeup(tp); splx(s); @@ -2468,11 +2461,10 @@ comstop(tp, rw) { struct com_s *com; bool_t wakeup_etc; - critical_t savecrit; com = com_addr(DEV_TO_UNIT(tp->t_dev)); wakeup_etc = FALSE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (rw & FWRITE) { com->obufs[0].l_queued = FALSE; @@ -2497,7 +2489,7 @@ comstop(tp, rw) com->iptr = com->ibuf; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (wakeup_etc) wakeup(&com->etc); if (rw & FWRITE && com->etc == ETC_NONE) @@ -2513,7 +2505,6 @@ commctl(com, bits, how) { int mcr; int msr; - critical_t savecrit; if (how == DMGET) { if (com->channel_control & CD1400_CCR_RCVEN) @@ -2551,7 +2542,7 @@ commctl(com, bits, how) mcr |= com->mcr_dtr; if (bits & TIOCM_RTS) mcr |= com->mcr_rts; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); switch (how) { case DMSET: @@ -2571,7 +2562,7 @@ commctl(com, bits, how) break; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2633,13 +2624,11 @@ comwakeup(chan) com = com_addr(unit); if (com != NULL && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { - critical_t savecrit; - - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); siointr1(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } } #endif @@ -2659,14 +2648,13 @@ comwakeup(chan) for (errnum = 0; errnum < CE_NTYPES; ++errnum) { u_int delta; u_long total; - critical_t savecrit; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta == 0) continue; total = com->error_counts[errnum] += delta; @@ -2818,7 +2806,6 @@ cd_etc(com, etc) struct com_s *com; int etc; { - critical_t savecrit; /* * We can't change the hardware's ETC state while there are any @@ -2831,7 +2818,7 @@ cd_etc(com, etc) * for the tx to become empty so that the command is sure to be * executed soon after we issue it. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->etc == etc) goto wait; @@ -2842,7 +2829,7 @@ cd_etc(com, etc) && (com->etc == ETC_BREAK_ENDING || com->etc == ETC_BREAK_ENDED || com->etc == ETC_NONE))) { COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return; } com->etc = etc; @@ -2851,7 +2838,7 @@ cd_etc(com, etc) = (com->intr_enable & ~CD1400_SRER_TXRDY) | CD1400_SRER_TXMPTY); wait: COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); while (com->etc == etc && tsleep(&com->etc, TTIPRI | PCATCH, "cyetc", 0) == 0) continue; @@ -2865,7 +2852,6 @@ cd_getreg(com, reg) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; int val; @@ -2875,7 +2861,7 @@ cd_getreg(com, reg) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2883,7 +2869,7 @@ cd_getreg(com, reg) val = cd_inb(iobase, reg, cy_align); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (val); } @@ -2896,7 +2882,6 @@ cd_setreg(com, reg, val) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; @@ -2905,7 +2890,7 @@ cd_setreg(com, reg, val) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2913,7 +2898,7 @@ cd_setreg(com, reg, val) cd_outb(iobase, reg, cy_align, val); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } #ifdef CyDebug diff --git a/sys/dev/cy/cy_isa.c b/sys/dev/cy/cy_isa.c index 20316ad..0510e72 100644 --- a/sys/dev/cy/cy_isa.c +++ b/sys/dev/cy/cy_isa.c @@ -356,7 +356,7 @@ static int cd_getreg __P((struct com_s *com, int reg)); static void cd_setreg __P((struct com_s *com, int reg, int val)); static timeout_t siodtrwakeup; static void comhardclose __P((struct com_s *com)); -static void sioinput __P((struct com_s *com, critical_t *savecrit)); +static void sioinput __P((struct com_s *com)); #if 0 static void siointr1 __P((struct com_s *com)); #endif @@ -673,7 +673,6 @@ sioopen(dev, flag, mode, td) int s; struct tty *tp; int unit; - critical_t savecrit; mynor = minor(dev); unit = MINOR_TO_UNIT(mynor); @@ -775,7 +774,7 @@ open_top: } } - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); @@ -784,7 +783,7 @@ open_top: outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #else /* !0 */ /* * Flush fifos. This requires a full channel reset which @@ -795,7 +794,7 @@ open_top: CD1400_CCR_CMDRESET | CD1400_CCR_CHANRESET); cd1400_channel_cmd(com, com->channel_control); - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->prev_modem_status = com->last_modem_status = cd_getreg(com, CD1400_MSVR2); @@ -803,7 +802,7 @@ open_top: com->intr_enable = CD1400_SRER_MDMCH | CD1400_SRER_RXDATA); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif /* 0 */ /* * Handle initial DCD. Callout devices get a fake initial @@ -886,7 +885,6 @@ comhardclose(com) int s; struct tty *tp; int unit; - critical_t savecrit; unit = com->unit; iobase = com->iobase; @@ -900,12 +898,12 @@ comhardclose(com) outb(iobase + com_cfcr, com->cfcr_image &= ~CFCR_SBREAK); #else /* XXX */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->etc = ETC_NONE; cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); cd1400_channel_cmd(com, CD1400_CCR_CMDRESET | CD1400_CCR_FTF); #endif @@ -913,11 +911,11 @@ comhardclose(com) #if 0 outb(iobase + com_ier, 0); #else - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); cd_setreg(com, CD1400_SRER, com->intr_enable = 0); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif tp = com->tp; if ((tp->t_cflag & HUPCL) @@ -1013,9 +1011,8 @@ siodtrwakeup(chan) * b) needs to return with COM_LOCK() held. */ static void -sioinput(com, savecrit) +sioinput(com) struct com_s *com; - critical_t *savecrit; { u_char *buf; int incc; @@ -1046,7 +1043,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); incc = com->iptr - buf; if (tp->t_rawq.c_cc + incc > tp->t_ihiwat && (com->state & CS_RTS_IFLOW @@ -1067,7 +1064,7 @@ sioinput(com, savecrit) tp->t_lflag &= ~FLUSHO; comstart(tp); } - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } else { @@ -1078,7 +1075,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); line_status = buf[com->ierroff]; recv_data = *buf++; if (line_status @@ -1093,7 +1090,7 @@ sioinput(com, savecrit) recv_data |= TTY_PE; } (*linesw[tp->t_line].l_rint)(recv_data, tp); - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } @@ -1767,7 +1764,6 @@ static void siopoll(void *arg) { int unit; - critical_t savecrit; #ifdef CyDebug ++cy_timeouts; @@ -1790,7 +1786,7 @@ repeat: * (actually never opened devices) so that we don't * loop. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; @@ -1800,7 +1796,7 @@ repeat: } com_events -= incc; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (incc != 0) log(LOG_DEBUG, "sio%d: %d events for device with no tp\n", @@ -1808,36 +1804,36 @@ repeat: continue; } if (com->iptr != com->ibuf) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (com->state & CS_CHECKMSR) { u_char delta_modem_status; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) (tp, com->prev_modem_status & MSR_DCD); } if (com->extra_state & CSE_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->extra_state &= ~CSE_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (!(com->state & CS_BUSY)) { tp->t_state &= ~TS_BUSY; ttwwakeup(com->tp); @@ -1849,12 +1845,12 @@ repeat: } } if (com->state & CS_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); (*linesw[tp->t_line].l_start)(tp); } if (com_events == 0) @@ -1883,7 +1879,6 @@ comparam(tp, t) u_char opt; int s; int unit; - critical_t savecrit; /* do historical conversions */ if (t->c_ispeed == 0) @@ -2031,14 +2026,14 @@ comparam(tp, t) if (cflag & CCTS_OFLOW) opt |= CD1400_COR2_CCTS_OFLOW; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (opt != com->cor[1]) { cor_change |= CD1400_CCR_COR2; cd_setreg(com, CD1400_COR2, com->cor[1] = opt); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); /* * set channel option register 3 - @@ -2159,7 +2154,7 @@ comparam(tp, t) * XXX should have done this long ago, but there is too much state * to change all atomically. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->state &= ~CS_TTGO; @@ -2227,7 +2222,7 @@ comparam(tp, t) } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); splx(s); comstart(tp); if (com->ibufold != NULL) { @@ -2246,7 +2241,6 @@ siosetwater(com, speed) u_char *ibuf; int ibufsize; struct tty *tp; - critical_t savecrit; /* * Make the buffer size large enough to handle a softtty interrupt @@ -2284,10 +2278,10 @@ siosetwater(com, speed) * Read current input buffer, if any. Continue with interrupts * disabled. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->iptr != com->ibuf) - sioinput(com, &savecrit); + sioinput(com); /*- * Initialize critical variables, including input buffer watermarks. @@ -2306,7 +2300,7 @@ siosetwater(com, speed) com->ihighwater = ibuf + 3 * ibufsize / 4; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2320,7 +2314,6 @@ comstart(tp) bool_t started; #endif int unit; - critical_t savecrit; unit = DEV_TO_UNIT(tp->t_dev); com = com_addr(unit); @@ -2331,7 +2324,7 @@ comstart(tp) started = FALSE; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (tp->t_state & TS_TTSTOP) { com->state &= ~CS_TTGO; @@ -2369,7 +2362,7 @@ comstart(tp) #endif } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { ttwwakeup(tp); splx(s); @@ -2388,7 +2381,7 @@ comstart(tp) sizeof com->obuf1); com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2409,7 +2402,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { #ifdef CyDebug @@ -2420,7 +2413,7 @@ comstart(tp) sizeof com->obuf2); com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2441,7 +2434,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } tp->t_state |= TS_BUSY; } @@ -2450,12 +2443,12 @@ comstart(tp) ++com->start_real; #endif #if 0 - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif ttwwakeup(tp); splx(s); @@ -2468,11 +2461,10 @@ comstop(tp, rw) { struct com_s *com; bool_t wakeup_etc; - critical_t savecrit; com = com_addr(DEV_TO_UNIT(tp->t_dev)); wakeup_etc = FALSE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (rw & FWRITE) { com->obufs[0].l_queued = FALSE; @@ -2497,7 +2489,7 @@ comstop(tp, rw) com->iptr = com->ibuf; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (wakeup_etc) wakeup(&com->etc); if (rw & FWRITE && com->etc == ETC_NONE) @@ -2513,7 +2505,6 @@ commctl(com, bits, how) { int mcr; int msr; - critical_t savecrit; if (how == DMGET) { if (com->channel_control & CD1400_CCR_RCVEN) @@ -2551,7 +2542,7 @@ commctl(com, bits, how) mcr |= com->mcr_dtr; if (bits & TIOCM_RTS) mcr |= com->mcr_rts; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); switch (how) { case DMSET: @@ -2571,7 +2562,7 @@ commctl(com, bits, how) break; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2633,13 +2624,11 @@ comwakeup(chan) com = com_addr(unit); if (com != NULL && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { - critical_t savecrit; - - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); siointr1(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } } #endif @@ -2659,14 +2648,13 @@ comwakeup(chan) for (errnum = 0; errnum < CE_NTYPES; ++errnum) { u_int delta; u_long total; - critical_t savecrit; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta == 0) continue; total = com->error_counts[errnum] += delta; @@ -2818,7 +2806,6 @@ cd_etc(com, etc) struct com_s *com; int etc; { - critical_t savecrit; /* * We can't change the hardware's ETC state while there are any @@ -2831,7 +2818,7 @@ cd_etc(com, etc) * for the tx to become empty so that the command is sure to be * executed soon after we issue it. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->etc == etc) goto wait; @@ -2842,7 +2829,7 @@ cd_etc(com, etc) && (com->etc == ETC_BREAK_ENDING || com->etc == ETC_BREAK_ENDED || com->etc == ETC_NONE))) { COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return; } com->etc = etc; @@ -2851,7 +2838,7 @@ cd_etc(com, etc) = (com->intr_enable & ~CD1400_SRER_TXRDY) | CD1400_SRER_TXMPTY); wait: COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); while (com->etc == etc && tsleep(&com->etc, TTIPRI | PCATCH, "cyetc", 0) == 0) continue; @@ -2865,7 +2852,6 @@ cd_getreg(com, reg) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; int val; @@ -2875,7 +2861,7 @@ cd_getreg(com, reg) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2883,7 +2869,7 @@ cd_getreg(com, reg) val = cd_inb(iobase, reg, cy_align); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (val); } @@ -2896,7 +2882,6 @@ cd_setreg(com, reg, val) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; @@ -2905,7 +2890,7 @@ cd_setreg(com, reg, val) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2913,7 +2898,7 @@ cd_setreg(com, reg, val) cd_outb(iobase, reg, cy_align, val); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } #ifdef CyDebug diff --git a/sys/dev/sound/isa/mpu.c b/sys/dev/sound/isa/mpu.c index c0d305e..ed9b746 100644 --- a/sys/dev/sound/isa/mpu.c +++ b/sys/dev/sound/isa/mpu.c @@ -241,7 +241,6 @@ mpu_probe2(device_t dev) sc_p scp; int unit, i; intrmask_t irqp0, irqp1; - critical_t savecrit; scp = device_get_softc(dev); unit = device_get_unit(dev); @@ -267,7 +266,7 @@ mpu_probe2(device_t dev) * Idea-stolen-from: sys/isa/sio.c:sioprobe() */ - savecrit = critical_enter(); + critical_enter(); /* * See the initial irq. We have to do this now, @@ -279,7 +278,7 @@ mpu_probe2(device_t dev) /* Switch to uart mode. */ if (mpu_uartmode(scp) != 0) { - critical_exit(savecrit); + critical_exit(); printf("mpu%d: mode switching failed.\n", unit); mpu_releaseres(scp, dev); return (ENXIO); @@ -298,7 +297,7 @@ mpu_probe2(device_t dev) break; } if (irqp1 == irqp0) { - critical_exit(savecrit); + critical_exit(); printf("mpu%d: switching the mode gave no interrupt.\n", unit); mpu_releaseres(scp, dev); return (ENXIO); @@ -307,13 +306,13 @@ mpu_probe2(device_t dev) no_irq: /* Wait to see an ACK. */ if (mpu_waitack(scp) != 0) { - critical_exit(savecrit); + critical_exit(); printf("mpu%d: not acked.\n", unit); mpu_releaseres(scp, dev); return (ENXIO); } - critical_exit(savecrit); + critical_exit(); if (device_get_flags(dev) & MPU_DF_NO_IRQ) scp->irq_val = 0; diff --git a/sys/i386/i386/db_interface.c b/sys/i386/i386/db_interface.c index 9708cad..000085b 100644 --- a/sys/i386/i386/db_interface.c +++ b/sys/i386/i386/db_interface.c @@ -324,10 +324,10 @@ Debugger(msg) return; if (atomic_cmpset_acq_int(&in_Debugger, 0, 1)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); db_printf("Debugger(\"%s\")\n", msg); breakpoint(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); atomic_store_rel_int(&in_Debugger, 0); } } diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c index 59f3147..99fa502 100644 --- a/sys/i386/i386/genassym.c +++ b/sys/i386/i386/genassym.c @@ -205,7 +205,6 @@ ASSYM(VM86_FRAMESIZE, sizeof(struct vm86frame)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSECNT, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); #ifdef PC98 #include <machine/bus.h> diff --git a/sys/i386/i386/initcpu.c b/sys/i386/i386/initcpu.c index b32c786..8a9a2a5 100644 --- a/sys/i386/i386/initcpu.c +++ b/sys/i386/i386/initcpu.c @@ -646,7 +646,7 @@ enable_K5_wt_alloc(void) * a stepping of 4 or greater. */ if (((cpu_id & 0xf0) > 0) && ((cpu_id & 0x0f) > 3)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); msr = rdmsr(0x83); /* HWCR */ wrmsr(0x83, msr & !(0x10)); @@ -678,7 +678,7 @@ enable_K5_wt_alloc(void) msr=rdmsr(0x83); wrmsr(0x83, msr|0x10); /* enable write allocate */ - critical_exit(savecrit); + cpu_critical_exit(savecrit); } } diff --git a/sys/i386/i386/perfmon.c b/sys/i386/i386/perfmon.c index ed0b53e..4d6c9a2 100644 --- a/sys/i386/i386/perfmon.c +++ b/sys/i386/i386/perfmon.c @@ -124,11 +124,11 @@ perfmon_setup(int pmc, unsigned int control) perfmon_inuse |= (1 << pmc); control &= ~(PMCF_SYS_FLAGS << 16); - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); ctl_shadow[pmc] = control; writectl(pmc); wrmsr(msr_pmc[pmc], pmc_shadow[pmc] = 0); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return 0; } @@ -169,11 +169,11 @@ perfmon_start(int pmc) return EINVAL; if (perfmon_inuse & (1 << pmc)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); ctl_shadow[pmc] |= (PMCF_EN << 16); wrmsr(msr_pmc[pmc], pmc_shadow[pmc]); writectl(pmc); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return 0; } return EBUSY; @@ -188,11 +188,11 @@ perfmon_stop(int pmc) return EINVAL; if (perfmon_inuse & (1 << pmc)) { - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); pmc_shadow[pmc] = rdmsr(msr_pmc[pmc]) & 0xffffffffffULL; ctl_shadow[pmc] &= ~(PMCF_EN << 16); writectl(pmc); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return 0; } return EBUSY; diff --git a/sys/i386/i386/sys_machdep.c b/sys/i386/i386/sys_machdep.c index a88acc9..b7beea9 100644 --- a/sys/i386/i386/sys_machdep.c +++ b/sys/i386/i386/sys_machdep.c @@ -532,13 +532,13 @@ i386_set_ldt(td, args) } /* Fill in range */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); error = copyin(uap->descs, &((union descriptor *)(pldt->ldt_base))[uap->start], uap->num * sizeof(union descriptor)); if (!error) td->td_retval[0] = uap->start; - critical_exit(savecrit); + cpu_critical_exit(savecrit); return(error); } diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index 322b5e7..4f1dab2 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -153,10 +153,10 @@ cpu_fork(td1, p2, flags) #ifdef DEV_NPX if (td1 == curthread) td1->td_pcb->pcb_gs = rgs(); - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (PCPU_GET(fpcurthread) == td1) npxsave(&td1->td_pcb->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #endif /* Point the pcb to the top of the stack */ diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h index 4a9300a..a4f57cb 100644 --- a/sys/i386/include/cpufunc.h +++ b/sys/i386/include/cpufunc.h @@ -41,6 +41,7 @@ #define _MACHINE_CPUFUNC_H_ #include <sys/cdefs.h> +#include <machine/psl.h> __BEGIN_DECLS #define readb(va) (*(volatile u_int8_t *) (va)) @@ -51,6 +52,8 @@ __BEGIN_DECLS #define writew(va, d) (*(volatile u_int16_t *) (va) = (d)) #define writel(va, d) (*(volatile u_int32_t *) (va) = (d)) +#define CRITICAL_FORK (read_eflags() | PSL_I) + #ifdef __GNUC__ #ifdef SWTCH_OPTIM_STATS @@ -548,7 +551,7 @@ load_dr7(u_int sel) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t eflags; @@ -558,7 +561,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t eflags) +cpu_critical_exit(critical_t eflags) { write_eflags(eflags); } @@ -597,8 +600,8 @@ u_int rfs __P((void)); u_int rgs __P((void)); void load_fs __P((u_int sel)); void load_gs __P((u_int sel)); -critical_t critical_enter __P((void)); -void critical_exit __P((critical_t eflags)); +critical_t cpu_critical_enter __P((void)); +void cpu_critical_exit __P((critical_t eflags)); #endif /* __GNUC__ */ diff --git a/sys/i386/include/mutex.h b/sys/i386/include/mutex.h index ae37b23..e68bce7 100644 --- a/sys/i386/include/mutex.h +++ b/sys/i386/include/mutex.h @@ -35,13 +35,10 @@ #ifndef LOCORE #ifdef _KERNEL -#include <machine/psl.h> /* Global locks */ extern struct mtx clock_lock; -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= PSL_I; while (0) - /* * Assembly macros (for internal use only) *------------------------------------------------------------------------------ @@ -246,51 +243,20 @@ extern struct mtx clock_lock; * locks) in the near future, however. */ #define MTX_LOCK_SPIN(lck, flags) \ - pushl %eax ; \ - pushl %ecx ; \ - pushl %ebx ; \ - movl $(MTX_UNOWNED) , %eax ; \ - movl PCPU(CURTHREAD), %ebx ; \ - pushfl ; \ - popl %ecx ; \ - cli ; \ - MPLOCKED cmpxchgl %ebx, lck+MTX_LOCK ; \ - jz 2f ; \ - cmpl lck+MTX_LOCK, %ebx ; \ - je 3f ; \ pushl $0 ; \ pushl $0 ; \ - pushl %ecx ; \ pushl $flags ; \ pushl $lck ; \ - call _mtx_lock_spin ; \ - addl $0x14, %esp ; \ - jmp 1f ; \ -3: movl lck+MTX_RECURSECNT, %ebx ; \ - incl %ebx ; \ - movl %ebx, lck+MTX_RECURSECNT ; \ - jmp 1f ; \ -2: movl %ecx, lck+MTX_SAVECRIT ; \ -1: popl %ebx ; \ - popl %ecx ; \ - popl %eax + call _mtx_lock_spin_flags ; \ + addl $0x10, %esp ; \ #define MTX_UNLOCK_SPIN(lck) \ - pushl %edx ; \ - pushl %eax ; \ - movl lck+MTX_SAVECRIT, %edx ; \ - movl lck+MTX_RECURSECNT, %eax ; \ - testl %eax, %eax ; \ - jne 2f ; \ - movl $(MTX_UNOWNED), %eax ; \ - xchgl %eax, lck+MTX_LOCK ; \ - pushl %edx ; \ - popfl ; \ - jmp 1f ; \ -2: decl %eax ; \ - movl %eax, lck+MTX_RECURSECNT ; \ -1: popl %eax ; \ - popl %edx + pushl $0 ; \ + pushl $0 ; \ + pushl $0 ; \ + pushl $lck ; \ + call _mtx_unlock_spin_flags ; \ + addl $0x10, %esp ; \ /* * XXX: These two are broken right now and need to be made to work for diff --git a/sys/i386/isa/cy.c b/sys/i386/isa/cy.c index 20316ad..0510e72 100644 --- a/sys/i386/isa/cy.c +++ b/sys/i386/isa/cy.c @@ -356,7 +356,7 @@ static int cd_getreg __P((struct com_s *com, int reg)); static void cd_setreg __P((struct com_s *com, int reg, int val)); static timeout_t siodtrwakeup; static void comhardclose __P((struct com_s *com)); -static void sioinput __P((struct com_s *com, critical_t *savecrit)); +static void sioinput __P((struct com_s *com)); #if 0 static void siointr1 __P((struct com_s *com)); #endif @@ -673,7 +673,6 @@ sioopen(dev, flag, mode, td) int s; struct tty *tp; int unit; - critical_t savecrit; mynor = minor(dev); unit = MINOR_TO_UNIT(mynor); @@ -775,7 +774,7 @@ open_top: } } - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); (void) inb(com->line_status_port); (void) inb(com->data_port); @@ -784,7 +783,7 @@ open_top: outb(iobase + com_ier, IER_ERXRDY | IER_ETXRDY | IER_ERLS | IER_EMSC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #else /* !0 */ /* * Flush fifos. This requires a full channel reset which @@ -795,7 +794,7 @@ open_top: CD1400_CCR_CMDRESET | CD1400_CCR_CHANRESET); cd1400_channel_cmd(com, com->channel_control); - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->prev_modem_status = com->last_modem_status = cd_getreg(com, CD1400_MSVR2); @@ -803,7 +802,7 @@ open_top: com->intr_enable = CD1400_SRER_MDMCH | CD1400_SRER_RXDATA); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif /* 0 */ /* * Handle initial DCD. Callout devices get a fake initial @@ -886,7 +885,6 @@ comhardclose(com) int s; struct tty *tp; int unit; - critical_t savecrit; unit = com->unit; iobase = com->iobase; @@ -900,12 +898,12 @@ comhardclose(com) outb(iobase + com_cfcr, com->cfcr_image &= ~CFCR_SBREAK); #else /* XXX */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->etc = ETC_NONE; cd_setreg(com, CD1400_COR2, com->cor[1] &= ~CD1400_COR2_ETC); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); cd1400_channel_cmd(com, CD1400_CCR_CMDRESET | CD1400_CCR_FTF); #endif @@ -913,11 +911,11 @@ comhardclose(com) #if 0 outb(iobase + com_ier, 0); #else - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); cd_setreg(com, CD1400_SRER, com->intr_enable = 0); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif tp = com->tp; if ((tp->t_cflag & HUPCL) @@ -1013,9 +1011,8 @@ siodtrwakeup(chan) * b) needs to return with COM_LOCK() held. */ static void -sioinput(com, savecrit) +sioinput(com) struct com_s *com; - critical_t *savecrit; { u_char *buf; int incc; @@ -1046,7 +1043,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); incc = com->iptr - buf; if (tp->t_rawq.c_cc + incc > tp->t_ihiwat && (com->state & CS_RTS_IFLOW @@ -1067,7 +1064,7 @@ sioinput(com, savecrit) tp->t_lflag &= ~FLUSHO; comstart(tp); } - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } else { @@ -1078,7 +1075,7 @@ sioinput(com, savecrit) * that are used everywhere else. */ COM_UNLOCK(); - critical_exit(*savecrit); + critical_exit(); line_status = buf[com->ierroff]; recv_data = *buf++; if (line_status @@ -1093,7 +1090,7 @@ sioinput(com, savecrit) recv_data |= TTY_PE; } (*linesw[tp->t_line].l_rint)(recv_data, tp); - *savecrit = critical_enter(); + critical_enter(); COM_LOCK(); } while (buf < com->iptr); } @@ -1767,7 +1764,6 @@ static void siopoll(void *arg) { int unit; - critical_t savecrit; #ifdef CyDebug ++cy_timeouts; @@ -1790,7 +1786,7 @@ repeat: * (actually never opened devices) so that we don't * loop. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); incc = com->iptr - com->ibuf; com->iptr = com->ibuf; @@ -1800,7 +1796,7 @@ repeat: } com_events -= incc; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (incc != 0) log(LOG_DEBUG, "sio%d: %d events for device with no tp\n", @@ -1808,36 +1804,36 @@ repeat: continue; } if (com->iptr != com->ibuf) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (com->state & CS_CHECKMSR) { u_char delta_modem_status; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); - sioinput(com, &savecrit); + sioinput(com); delta_modem_status = com->last_modem_status ^ com->prev_modem_status; com->prev_modem_status = com->last_modem_status; com_events -= LOTS_OF_EVENTS; com->state &= ~CS_CHECKMSR; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta_modem_status & MSR_DCD) (*linesw[tp->t_line].l_modem) (tp, com->prev_modem_status & MSR_DCD); } if (com->extra_state & CSE_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->extra_state &= ~CSE_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (!(com->state & CS_BUSY)) { tp->t_state &= ~TS_BUSY; ttwwakeup(com->tp); @@ -1849,12 +1845,12 @@ repeat: } } if (com->state & CS_ODONE) { - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com_events -= LOTS_OF_EVENTS; com->state &= ~CS_ODONE; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); (*linesw[tp->t_line].l_start)(tp); } if (com_events == 0) @@ -1883,7 +1879,6 @@ comparam(tp, t) u_char opt; int s; int unit; - critical_t savecrit; /* do historical conversions */ if (t->c_ispeed == 0) @@ -2031,14 +2026,14 @@ comparam(tp, t) if (cflag & CCTS_OFLOW) opt |= CD1400_COR2_CCTS_OFLOW; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (opt != com->cor[1]) { cor_change |= CD1400_CCR_COR2; cd_setreg(com, CD1400_COR2, com->cor[1] = opt); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); /* * set channel option register 3 - @@ -2159,7 +2154,7 @@ comparam(tp, t) * XXX should have done this long ago, but there is too much state * to change all atomically. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); com->state &= ~CS_TTGO; @@ -2227,7 +2222,7 @@ comparam(tp, t) } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); splx(s); comstart(tp); if (com->ibufold != NULL) { @@ -2246,7 +2241,6 @@ siosetwater(com, speed) u_char *ibuf; int ibufsize; struct tty *tp; - critical_t savecrit; /* * Make the buffer size large enough to handle a softtty interrupt @@ -2284,10 +2278,10 @@ siosetwater(com, speed) * Read current input buffer, if any. Continue with interrupts * disabled. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->iptr != com->ibuf) - sioinput(com, &savecrit); + sioinput(com); /*- * Initialize critical variables, including input buffer watermarks. @@ -2306,7 +2300,7 @@ siosetwater(com, speed) com->ihighwater = ibuf + 3 * ibufsize / 4; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2320,7 +2314,6 @@ comstart(tp) bool_t started; #endif int unit; - critical_t savecrit; unit = DEV_TO_UNIT(tp->t_dev); com = com_addr(unit); @@ -2331,7 +2324,7 @@ comstart(tp) started = FALSE; #endif - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (tp->t_state & TS_TTSTOP) { com->state &= ~CS_TTGO; @@ -2369,7 +2362,7 @@ comstart(tp) #endif } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (tp->t_state & (TS_TIMEOUT | TS_TTSTOP)) { ttwwakeup(tp); splx(s); @@ -2388,7 +2381,7 @@ comstart(tp) sizeof com->obuf1); com->obufs[0].l_next = NULL; com->obufs[0].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2409,7 +2402,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } if (tp->t_outq.c_cc != 0 && !com->obufs[1].l_queued) { #ifdef CyDebug @@ -2420,7 +2413,7 @@ comstart(tp) sizeof com->obuf2); com->obufs[1].l_next = NULL; com->obufs[1].l_queued = TRUE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state & CS_BUSY) { qp = com->obufq.l_next; @@ -2441,7 +2434,7 @@ comstart(tp) | CD1400_SRER_TXRDY); } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } tp->t_state |= TS_BUSY; } @@ -2450,12 +2443,12 @@ comstart(tp) ++com->start_real; #endif #if 0 - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->state >= (CS_BUSY | CS_TTGO)) siointr1(com); /* fake interrupt to start output */ COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); #endif ttwwakeup(tp); splx(s); @@ -2468,11 +2461,10 @@ comstop(tp, rw) { struct com_s *com; bool_t wakeup_etc; - critical_t savecrit; com = com_addr(DEV_TO_UNIT(tp->t_dev)); wakeup_etc = FALSE; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (rw & FWRITE) { com->obufs[0].l_queued = FALSE; @@ -2497,7 +2489,7 @@ comstop(tp, rw) com->iptr = com->ibuf; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (wakeup_etc) wakeup(&com->etc); if (rw & FWRITE && com->etc == ETC_NONE) @@ -2513,7 +2505,6 @@ commctl(com, bits, how) { int mcr; int msr; - critical_t savecrit; if (how == DMGET) { if (com->channel_control & CD1400_CCR_RCVEN) @@ -2551,7 +2542,7 @@ commctl(com, bits, how) mcr |= com->mcr_dtr; if (bits & TIOCM_RTS) mcr |= com->mcr_rts; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); switch (how) { case DMSET: @@ -2571,7 +2562,7 @@ commctl(com, bits, how) break; } COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (0); } @@ -2633,13 +2624,11 @@ comwakeup(chan) com = com_addr(unit); if (com != NULL && (com->state >= (CS_BUSY | CS_TTGO) || com->poll)) { - critical_t savecrit; - - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); siointr1(com); COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } } #endif @@ -2659,14 +2648,13 @@ comwakeup(chan) for (errnum = 0; errnum < CE_NTYPES; ++errnum) { u_int delta; u_long total; - critical_t savecrit; - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); delta = com->delta_error_counts[errnum]; com->delta_error_counts[errnum] = 0; COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); if (delta == 0) continue; total = com->error_counts[errnum] += delta; @@ -2818,7 +2806,6 @@ cd_etc(com, etc) struct com_s *com; int etc; { - critical_t savecrit; /* * We can't change the hardware's ETC state while there are any @@ -2831,7 +2818,7 @@ cd_etc(com, etc) * for the tx to become empty so that the command is sure to be * executed soon after we issue it. */ - savecrit = critical_enter(); + critical_enter(); COM_LOCK(); if (com->etc == etc) goto wait; @@ -2842,7 +2829,7 @@ cd_etc(com, etc) && (com->etc == ETC_BREAK_ENDING || com->etc == ETC_BREAK_ENDED || com->etc == ETC_NONE))) { COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return; } com->etc = etc; @@ -2851,7 +2838,7 @@ cd_etc(com, etc) = (com->intr_enable & ~CD1400_SRER_TXRDY) | CD1400_SRER_TXMPTY); wait: COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); while (com->etc == etc && tsleep(&com->etc, TTIPRI | PCATCH, "cyetc", 0) == 0) continue; @@ -2865,7 +2852,6 @@ cd_getreg(com, reg) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; int val; @@ -2875,7 +2861,7 @@ cd_getreg(com, reg) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2883,7 +2869,7 @@ cd_getreg(com, reg) val = cd_inb(iobase, reg, cy_align); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); return (val); } @@ -2896,7 +2882,6 @@ cd_setreg(com, reg, val) struct com_s *basecom; u_char car; int cy_align; - critical_t savecrit; register_t eflags; cy_addr iobase; @@ -2905,7 +2890,7 @@ cd_setreg(com, reg, val) cy_align = com->cy_align; iobase = com->iobase; eflags = read_eflags(); - savecrit = critical_enter(); + critical_enter(); if (eflags & PSL_I) COM_LOCK(); if (basecom->car != car) @@ -2913,7 +2898,7 @@ cd_setreg(com, reg, val) cd_outb(iobase, reg, cy_align, val); if (eflags & PSL_I) COM_UNLOCK(); - critical_exit(savecrit); + critical_exit(); } #ifdef CyDebug diff --git a/sys/i386/isa/npx.c b/sys/i386/isa/npx.c index ff8c4cf..34a822a 100644 --- a/sys/i386/isa/npx.c +++ b/sys/i386/isa/npx.c @@ -515,7 +515,7 @@ npxinit(control) * fnsave to throw away any junk in the fpu. npxsave() initializes * the fpu and sets fpcurthread = NULL as important side effects. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); npxsave(&dummy); stop_emulating(); #ifdef CPU_ENABLE_SSE @@ -527,7 +527,7 @@ npxinit(control) if (PCPU_GET(curpcb) != NULL) fpusave(&PCPU_GET(curpcb)->pcb_save); start_emulating(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } /* @@ -539,10 +539,10 @@ npxexit(td) { critical_t savecrit; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (td == PCPU_GET(fpcurthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); - critical_exit(savecrit); + cpu_critical_exit(savecrit); #ifdef NPX_DEBUG if (npx_exists) { u_int masked_exceptions; @@ -762,7 +762,7 @@ npxtrap() PCPU_GET(fpcurthread), curthread, npx_exists); panic("npxtrap from nowhere"); } - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); /* * Interrupt handling (for another interrupt) may have pushed the @@ -783,7 +783,7 @@ npxtrap() GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (fpetable[status & ((~control & 0x3f) | 0x40)]); } @@ -807,7 +807,7 @@ npxdna() PCPU_GET(fpcurthread), curthread); panic("npxdna"); } - s = critical_enter(); + s = cpu_critical_enter(); stop_emulating(); /* * Record new context early in case frstor causes an IRQ13. @@ -829,7 +829,7 @@ npxdna() * first FPU instruction after a context switch. */ fpurstor(&PCPU_GET(curpcb)->pcb_save); - critical_exit(s); + cpu_critical_exit(s); return (1); } diff --git a/sys/ia64/ia64/db_interface.c b/sys/ia64/ia64/db_interface.c index 9a254da..3d576cd 100644 --- a/sys/ia64/ia64/db_interface.c +++ b/sys/ia64/ia64/db_interface.c @@ -336,7 +336,7 @@ kdb_trap(int vector, struct trapframe *regs) __asm __volatile("flushrs"); /* so we can look at them */ - s = critical_enter(); + s = cpu_critical_enter(); #if 0 db_printf("stopping %x\n", PCPU_GET(other_cpus)); @@ -359,7 +359,7 @@ kdb_trap(int vector, struct trapframe *regs) restart_cpus(stopped_cpus); #endif - critical_exit(s); + cpu_critical_exit(s); *regs = ddb_regs; diff --git a/sys/ia64/ia64/genassym.c b/sys/ia64/ia64/genassym.c index fbe60e5..5370da0 100644 --- a/sys/ia64/ia64/genassym.c +++ b/sys/ia64/ia64/genassym.c @@ -71,7 +71,6 @@ ASSYM(PC_CPUID, offsetof(struct pcpu, pc_cpuid)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSE, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); ASSYM(MTX_UNOWNED, MTX_UNOWNED); ASSYM(TD_PROC, offsetof(struct thread, td_proc)); diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 34b129d..a144db6 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -577,7 +577,7 @@ pmap_invalidate_all(pmap_t pmap) KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), ("invalidating TLB for non-current pmap")); - psr = critical_enter(); + psr = cpu_critical_enter(); addr = pmap_ptc_e_base; for (i = 0; i < pmap_ptc_e_count1; i++) { for (j = 0; j < pmap_ptc_e_count2; j++) { @@ -586,7 +586,7 @@ pmap_invalidate_all(pmap_t pmap) } addr += pmap_ptc_e_stride1; } - critical_exit(psr); + cpu_critical_exit(psr); } static u_int32_t diff --git a/sys/ia64/ia64/sapic.c b/sys/ia64/ia64/sapic.c index 6af5cc0..e13b8d5 100644 --- a/sys/ia64/ia64/sapic.c +++ b/sys/ia64/ia64/sapic.c @@ -82,10 +82,10 @@ sapic_read_rte(struct sapic *sa, int which, { u_int32_t *p = (u_int32_t *) rte; critical_t c; - c = critical_enter(); + c = cpu_critical_enter(); p[0] = sapic_read(sa, SAPIC_RTE_BASE + 2*which); p[1] = sapic_read(sa, SAPIC_RTE_BASE + 2*which + 1); - critical_exit(c); + cpu_critical_exit(c); } #endif @@ -96,10 +96,10 @@ sapic_write_rte(struct sapic *sa, int which, { u_int32_t *p = (u_int32_t *) rte; critical_t c; - c = critical_enter(); + c = cpu_critical_enter(); sapic_write(sa, SAPIC_RTE_BASE + 2*which, p[0]); sapic_write(sa, SAPIC_RTE_BASE + 2*which + 1, p[1]); - critical_exit(c); + cpu_critical_exit(c); } struct sapic * diff --git a/sys/ia64/include/cpufunc.h b/sys/ia64/include/cpufunc.h index 45390e9..c2d074f 100644 --- a/sys/ia64/include/cpufunc.h +++ b/sys/ia64/include/cpufunc.h @@ -34,6 +34,8 @@ #include <sys/types.h> #include <machine/ia64_cpu.h> +#define CRITICAL_FORK (ia64_get_psr() |= IA64_PSR_I) + #ifdef __GNUC__ static __inline void @@ -283,7 +285,7 @@ enable_intr(void) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t psr; @@ -293,7 +295,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t psr) +cpu_critical_exit(critical_t psr) { __asm __volatile ("mov psr.l=%0;; srlz.d" :: "r" (psr)); } diff --git a/sys/ia64/include/mutex.h b/sys/ia64/include/mutex.h index b14d7ae..96d8f3d 100644 --- a/sys/ia64/include/mutex.h +++ b/sys/ia64/include/mutex.h @@ -32,17 +32,7 @@ #ifndef _MACHINE_MUTEX_H_ #define _MACHINE_MUTEX_H_ -#include <machine/ia64_cpu.h> - -#ifndef LOCORE - -#ifdef _KERNEL - -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= IA64_PSR_I; while (0) - -#endif /* _KERNEL */ - -#else /* !LOCORE */ +#ifdef LOCORE /* * Simple assembly macros to get and release non-recursive spin locks diff --git a/sys/ia64/include/profile.h b/sys/ia64/include/profile.h index 038c805..8d93c5c 100644 --- a/sys/ia64/include/profile.h +++ b/sys/ia64/include/profile.h @@ -100,9 +100,9 @@ _mcount: \n\ * The following two macros do splhigh and splx respectively. */ #define MCOUNT_ENTER(s) \n\ - _c = critical_enter() + _c = cpu_critical_enter() #define MCOUNT_EXIT(s) \n\ - (void)critical_exit(_c) + cpu_critical_exit(_c) #define MCOUNT_DECL(s) critical_t c; #ifdef GUPROF struct gmonparam; diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c index bb52a34..bc03078 100644 --- a/sys/kern/kern_fork.c +++ b/sys/kern/kern_fork.c @@ -467,9 +467,6 @@ again: if (p1->p_sflag & PS_PROFIL) startprofclock(p2); mtx_unlock_spin(&sched_lock); - /* - * We start off holding one spinlock after fork: sched_lock. - */ PROC_LOCK(p1); p2->p_ucred = crhold(p1->p_ucred); p2->p_thread.td_ucred = crhold(p2->p_ucred); /* XXXKSE */ @@ -766,10 +763,8 @@ fork_exit(callout, arg, frame) */ sched_lock.mtx_lock = (uintptr_t)td; sched_lock.mtx_recurse = 0; - /* - * XXX: We really shouldn't have to do this. - */ - mtx_intr_enable(&sched_lock); + td->td_critnest = 1; + td->td_savecrit = CRITICAL_FORK; CTR3(KTR_PROC, "fork_exit: new proc %p (pid %d, %s)", p, p->p_pid, p->p_comm); if (PCPU_GET(switchtime.tv_sec) == 0) diff --git a/sys/kern/kern_idle.c b/sys/kern/kern_idle.c index 92e5cb3..4dee96d 100644 --- a/sys/kern/kern_idle.c +++ b/sys/kern/kern_idle.c @@ -47,8 +47,10 @@ idle_setup(void *dummy) error = kthread_create(idle_proc, NULL, &p, RFSTOPPED | RFHIGHPID, "idle: cpu%d", pc->pc_cpuid); pc->pc_idlethread = &p->p_thread; - if (pc->pc_curthread == NULL) + if (pc->pc_curthread == NULL) { pc->pc_curthread = pc->pc_idlethread; + pc->pc_idlethread->td_critnest = 0; + } #else error = kthread_create(idle_proc, NULL, &p, RFSTOPPED | RFHIGHPID, "idle"); diff --git a/sys/kern/kern_ktr.c b/sys/kern/kern_ktr.c index 309d6e3..596a724 100644 --- a/sys/kern/kern_ktr.c +++ b/sys/kern/kern_ktr.c @@ -133,9 +133,9 @@ ktr_tracepoint(u_int mask, const char *format, u_long arg1, u_long arg2, td = curthread; if (td->td_inktr) return; - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); if (((1 << KTR_CPU) & ktr_cpumask) == 0) { - critical_exit(savecrit); + cpu_critical_exit(savecrit); return; } td->td_inktr++; @@ -145,7 +145,7 @@ ktr_tracepoint(u_int mask, const char *format, u_long arg1, u_long arg2, } while (atomic_cmpset_rel_int(&ktr_idx, saveindex, newindex) == 0); entry = &ktr_buf[saveindex]; entry->ktr_cpu = KTR_CPU; - critical_exit(savecrit); + cpu_critical_exit(savecrit); nanotime(&entry->ktr_tv); #ifdef KTR_EXTEND entry->ktr_filename = filename; diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c index d4809af..852b570 100644 --- a/sys/kern/kern_mutex.c +++ b/sys/kern/kern_mutex.c @@ -427,8 +427,7 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) * is handled inline. */ void -_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, - int line) +_mtx_lock_spin(struct mtx *m, int opts, const char *file, int line) { int i = 0; @@ -440,7 +439,7 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, break; /* Give interrupts a chance while we spin. */ - critical_exit(mtx_crit); + critical_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 1000000) continue; @@ -454,10 +453,9 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, panic("spin lock %s held by %p for > 5 seconds", m->mtx_object.lo_name, (void *)m->mtx_lock); } - mtx_crit = critical_enter(); + critical_enter(); } - m->mtx_savecrit = mtx_crit; if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); diff --git a/sys/kern/kern_switch.c b/sys/kern/kern_switch.c index 65054df..1a2afa4 100644 --- a/sys/kern/kern_switch.c +++ b/sys/kern/kern_switch.c @@ -69,6 +69,31 @@ setrunqueue(struct thread *td) runq_add(&runq, td->td_kse); } +/* Critical sections that prevent preemption. */ +void +critical_enter(void) +{ + struct thread *td; + + td = curthread; + if (td->td_critnest == 0) + td->td_savecrit = cpu_critical_enter(); + td->td_critnest++; +} + +void +critical_exit(void) +{ + struct thread *td; + + td = curthread; + if (td->td_critnest == 1) { + td->td_critnest = 0; + cpu_critical_exit(td->td_savecrit); + } else + td->td_critnest--; +} + /* * Clear the status bit of the queue corresponding to priority level pri, * indicating that it is empty. diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c index 7034c0a..fce470f 100644 --- a/sys/kern/kern_synch.c +++ b/sys/kern/kern_synch.c @@ -698,7 +698,6 @@ mi_switch() #if 0 register struct rlimit *rlim; #endif - critical_t sched_crit; u_int sched_nest; mtx_assert(&sched_lock, MA_OWNED | MA_NOTRECURSED); @@ -773,14 +772,12 @@ mi_switch() PCPU_SET(switchtime, new_switchtime); CTR3(KTR_PROC, "mi_switch: old proc %p (pid %d, %s)", p, p->p_pid, p->p_comm); - sched_crit = sched_lock.mtx_savecrit; sched_nest = sched_lock.mtx_recurse; td->td_lastcpu = td->td_kse->ke_oncpu; td->td_kse->ke_oncpu = NOCPU; td->td_kse->ke_flags &= ~KEF_NEEDRESCHED; cpu_switch(); td->td_kse->ke_oncpu = PCPU_GET(cpuid); - sched_lock.mtx_savecrit = sched_crit; sched_lock.mtx_recurse = sched_nest; sched_lock.mtx_lock = (uintptr_t)td; CTR3(KTR_PROC, "mi_switch: new proc %p (pid %d, %s)", p, p->p_pid, diff --git a/sys/kern/subr_prof.c b/sys/kern/subr_prof.c index 67629d4..8808651 100644 --- a/sys/kern/subr_prof.c +++ b/sys/kern/subr_prof.c @@ -90,7 +90,6 @@ kmupetext(uintfptr_t nhighpc) struct gmonparam np; /* slightly large */ struct gmonparam *p = &_gmonparam; char *cp; - critical_t savecrit; GIANT_REQUIRED; bcopy(p, &np, sizeof(*p)); @@ -127,7 +126,7 @@ kmupetext(uintfptr_t nhighpc) np.mcount_count = &KCOUNT(&np, PC_TO_I(&np, mcount)); np.mexitcount_count = &KCOUNT(&np, PC_TO_I(&np, mexitcount)); #endif - savecrit = critical_enter(); + critical_enter(); bcopy(p->tos, np.tos, p->tossize); bzero((char *)np.tos + p->tossize, np.tossize - p->tossize); bcopy(p->kcount, np.kcount, p->kcountsize); @@ -137,7 +136,7 @@ kmupetext(uintfptr_t nhighpc) bzero((char *)np.froms + p->fromssize, np.fromssize - p->fromssize); cp = (char *)p->tos; bcopy(&np, p, sizeof(*p)); - critical_exit(savecrit); + critical_exit(); free(cp, M_GPROF); } @@ -156,7 +155,6 @@ kmstartup(dummy) int nullfunc_loop_overhead; int nullfunc_loop_profiled_time; uintfptr_t tmp_addr; - critical_t savecrit; #endif /* @@ -195,7 +193,7 @@ kmstartup(dummy) * Disable interrupts to avoid interference while we calibrate * things. */ - savecrit = critical_enter(); + critical_enter(); /* * Determine overheads. @@ -249,7 +247,7 @@ kmstartup(dummy) p->state = GMON_PROF_OFF; stopguprof(p); - critical_exit(savecrit); + critical_exit(); nullfunc_loop_profiled_time = 0; for (tmp_addr = (uintfptr_t)nullfunc_loop_profiled; diff --git a/sys/kern/subr_trap.c b/sys/kern/subr_trap.c index a899576..6f17f8f 100644 --- a/sys/kern/subr_trap.c +++ b/sys/kern/subr_trap.c @@ -141,9 +141,9 @@ ast(framep) panic("Returning to user mode with mutex(s) held"); #endif mtx_assert(&Giant, MA_NOTOWNED); - s = critical_enter(); + s = cpu_critical_enter(); while ((ke->ke_flags & (KEF_ASTPENDING | KEF_NEEDRESCHED)) != 0) { - critical_exit(s); + cpu_critical_exit(s); td->td_frame = framep; /* * This updates the p_sflag's for the checks below in one @@ -195,13 +195,13 @@ ast(framep) crfree(td->td_ucred); mtx_unlock(&Giant); td->td_ucred = NULL; - s = critical_enter(); + s = cpu_critical_enter(); } mtx_assert(&Giant, MA_NOTOWNED); /* * We need to keep interrupts disabled so that if any further AST's * come in, the interrupt they come in on will be delayed until we * finish returning to userland. We assume that the return to userland - * will perform the equivalent of critical_exit(). + * will perform the equivalent of cpu_critical_exit(). */ } diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c index d4809af..852b570 100644 --- a/sys/kern/subr_turnstile.c +++ b/sys/kern/subr_turnstile.c @@ -427,8 +427,7 @@ _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line) * is handled inline. */ void -_mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, - int line) +_mtx_lock_spin(struct mtx *m, int opts, const char *file, int line) { int i = 0; @@ -440,7 +439,7 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, break; /* Give interrupts a chance while we spin. */ - critical_exit(mtx_crit); + critical_exit(); while (m->mtx_lock != MTX_UNOWNED) { if (i++ < 1000000) continue; @@ -454,10 +453,9 @@ _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, const char *file, panic("spin lock %s held by %p for > 5 seconds", m->mtx_object.lo_name, (void *)m->mtx_lock); } - mtx_crit = critical_enter(); + critical_enter(); } - m->mtx_savecrit = mtx_crit; if (LOCK_LOG_TEST(&m->mtx_object, opts)) CTR1(KTR_LOCK, "_mtx_lock_spin: %p spin done", m); diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c index 454fd55..5e4dc61 100644 --- a/sys/kern/subr_witness.c +++ b/sys/kern/subr_witness.c @@ -252,7 +252,6 @@ static struct mtx all_mtx = { { NULL }, /* mtx_object.lo_list */ NULL }, /* mtx_object.lo_witness */ MTX_UNOWNED, 0, /* mtx_lock, mtx_recurse */ - 0, /* mtx_savecrit */ TAILQ_HEAD_INITIALIZER(all_mtx.mtx_blocked), { NULL, NULL } /* mtx_contested */ }; @@ -836,7 +835,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line) instance->li_flags--; goto out; } - s = critical_enter(); + s = cpu_critical_enter(); CTR4(KTR_WITNESS, "%s: pid %d removed %s from lle[%d]", __func__, td->td_proc->p_pid, @@ -846,7 +845,7 @@ witness_unlock(struct lock_object *lock, int flags, const char *file, int line) for (j = i; j < (*lock_list)->ll_count; j++) (*lock_list)->ll_children[j] = (*lock_list)->ll_children[j + 1]; - critical_exit(s); + cpu_critical_exit(s); if ((*lock_list)->ll_count == 0) { lle = *lock_list; *lock_list = lle->ll_next; @@ -896,7 +895,7 @@ witness_sleep(int check_only, struct lock_object *lock, const char *file, /* * Preemption bad because we need PCPU_PTR(spinlocks) to not change. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); td = curthread; lock_list = &td->td_sleeplocks; again: @@ -931,7 +930,7 @@ again: if (witness_ddb && n) Debugger(__func__); #endif /* DDB */ - critical_exit(savecrit); + cpu_critical_exit(savecrit); return (n); } @@ -1360,9 +1359,9 @@ witness_list(struct thread *td) * Preemption bad because we need PCPU_PTR(spinlocks) to not * change. */ - savecrit = critical_enter(); + savecrit = cpu_critical_enter(); nheld += witness_list_locks(PCPU_PTR(spinlocks)); - critical_exit(savecrit); + cpu_critical_exit(savecrit); } return (nheld); } diff --git a/sys/powerpc/include/cpufunc.h b/sys/powerpc/include/cpufunc.h index 0fd0dd0..25f37d3 100644 --- a/sys/powerpc/include/cpufunc.h +++ b/sys/powerpc/include/cpufunc.h @@ -35,6 +35,8 @@ #include <machine/psl.h> +#define CRITICAL_FORK (mfmsr() |= PSL_EE) + #ifdef __GNUC__ static __inline void @@ -114,7 +116,7 @@ save_intr(void) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { return ((critical_t)save_intr()); @@ -128,7 +130,7 @@ restore_intr(unsigned int msr) } static __inline void -critical_exit(critical_t msr) +cpu_critical_exit(critical_t msr) { return (restore_intr((unsigned int)msr)); diff --git a/sys/powerpc/include/mutex.h b/sys/powerpc/include/mutex.h index 0aeaa74..0b7fe26 100644 --- a/sys/powerpc/include/mutex.h +++ b/sys/powerpc/include/mutex.h @@ -32,22 +32,7 @@ #ifndef _MACHINE_MUTEX_H_ #define _MACHINE_MUTEX_H_ -#ifndef LOCORE - -#ifdef _KERNEL - -#define mtx_intr_enable(mutex) do (mutex)->mtx_savecrit |= PSL_EE; while (0) - -/* - * Assembly macros (for internal use only) - *-------------------------------------------------------------------------- - */ - -#define _V(x) __STRING(x) - -#endif /* _KERNEL */ - -#else /* !LOCORE */ +#ifdef LOCORE /* * Simple assembly macros to get and release non-recursive spin locks diff --git a/sys/powerpc/powerpc/genassym.c b/sys/powerpc/powerpc/genassym.c index 7ab6e77..c212c6b 100644 --- a/sys/powerpc/powerpc/genassym.c +++ b/sys/powerpc/powerpc/genassym.c @@ -66,7 +66,6 @@ ASSYM(PC_SWITCHTIME, offsetof(struct pcpu, pc_switchtime)); ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); ASSYM(MTX_RECURSECNT, offsetof(struct mtx, mtx_recurse)); -ASSYM(MTX_SAVECRIT, offsetof(struct mtx, mtx_savecrit)); ASSYM(PM_KERNELSR, offsetof(struct pmap, pm_sr[KERNEL_SR])); ASSYM(PM_USERSR, offsetof(struct pmap, pm_sr[USER_SR])); diff --git a/sys/sparc64/include/cpufunc.h b/sys/sparc64/include/cpufunc.h index 18f06b5..f9739ad 100644 --- a/sys/sparc64/include/cpufunc.h +++ b/sys/sparc64/include/cpufunc.h @@ -156,6 +156,8 @@ STNC_GEN(u_long, stxa); : : "r" (val), "rI" (xor)); \ } while (0) +#define CRITICAL_FORK (0) + static __inline void breakpoint(void) { @@ -163,7 +165,7 @@ breakpoint(void) } static __inline critical_t -critical_enter(void) +cpu_critical_enter(void) { critical_t pil; @@ -173,7 +175,7 @@ critical_enter(void) } static __inline void -critical_exit(critical_t pil) +cpu_critical_exit(critical_t pil) { wrpr(pil, pil, 0); } diff --git a/sys/sparc64/include/mutex.h b/sys/sparc64/include/mutex.h index 293f8d6..c9b2e1d 100644 --- a/sys/sparc64/include/mutex.h +++ b/sys/sparc64/include/mutex.h @@ -29,11 +29,4 @@ #ifndef _MACHINE_MUTEX_H_ #define _MACHINE_MUTEX_H_ -static __inline void -mtx_intr_enable(struct mtx *mtx) -{ - - mtx->mtx_savecrit = 0; -} - #endif /* !_MACHINE_MUTEX_H_ */ diff --git a/sys/sparc64/sparc64/intr_machdep.c b/sys/sparc64/sparc64/intr_machdep.c index 3b39f9a..ca28a60 100644 --- a/sys/sparc64/sparc64/intr_machdep.c +++ b/sys/sparc64/sparc64/intr_machdep.c @@ -99,7 +99,7 @@ intr_dequeue(struct trapframe *tf) u_long next; u_long tail; - crit = critical_enter(); + crit = cpu_critical_enter(); iq = PCPU_PTR(iq); for (head = iq->iq_head;; head = next) { for (tail = iq->iq_tail; tail != head;) { @@ -115,7 +115,7 @@ intr_dequeue(struct trapframe *tf) if (head == next) break; } - critical_exit(crit); + cpu_critical_exit(crit); } void diff --git a/sys/sys/_mutex.h b/sys/sys/_mutex.h index 82f3648..ef5de76 100644 --- a/sys/sys/_mutex.h +++ b/sys/sys/_mutex.h @@ -39,7 +39,6 @@ struct mtx { struct lock_object mtx_object; /* Common lock properties. */ volatile uintptr_t mtx_lock; /* owner (and state for sleep locks) */ volatile u_int mtx_recurse; /* number of recursive holds */ - critical_t mtx_savecrit; /* saved flags (for spin locks) */ TAILQ_HEAD(, thread) mtx_blocked; /* threads blocked on this lock */ LIST_ENTRY(mtx) mtx_contested; /* list of all contested locks */ }; diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h index d581973..a4eb018 100644 --- a/sys/sys/mutex.h +++ b/sys/sys/mutex.h @@ -101,8 +101,7 @@ void mtx_init(struct mtx *m, const char *description, int opts); void mtx_destroy(struct mtx *m); void _mtx_lock_sleep(struct mtx *m, int opts, const char *file, int line); void _mtx_unlock_sleep(struct mtx *m, int opts, const char *file, int line); -void _mtx_lock_spin(struct mtx *m, int opts, critical_t mtx_crit, - const char *file, int line); +void _mtx_lock_spin(struct mtx *m, int opts, const char *file, int line); void _mtx_unlock_spin(struct mtx *m, int opts, const char *file, int line); int _mtx_trylock(struct mtx *m, int opts, const char *file, int line); void _mtx_lock_flags(struct mtx *m, int opts, const char *file, int line); @@ -160,16 +159,13 @@ void mtx_unlock_giant(int s); */ #ifndef _get_spin_lock #define _get_spin_lock(mp, tid, opts, file, line) do { \ - critical_t _mtx_crit; \ - _mtx_crit = critical_enter(); \ + critical_enter(); \ if (!_obtain_lock((mp), (tid))) { \ if ((mp)->mtx_lock == (uintptr_t)(tid)) \ (mp)->mtx_recurse++; \ else \ - _mtx_lock_spin((mp), (opts), _mtx_crit, (file), \ - (line)); \ - } else \ - (mp)->mtx_savecrit = _mtx_crit; \ + _mtx_lock_spin((mp), (opts), (file), (line)); \ + } \ } while (0) #endif @@ -189,16 +185,18 @@ void mtx_unlock_giant(int s); * a function call would be too expensive (at least on some architectures). * Since spin locks are not _too_ common, inlining this code is not too big * a deal. + * + * Since we always perform a critical_enter() when attempting to acquire a + * spin lock, we need to always perform a matching critical_exit() when + * releasing a spin lock. This includes the recursion cases. */ #ifndef _rel_spin_lock #define _rel_spin_lock(mp) do { \ - critical_t _mtx_crit = (mp)->mtx_savecrit; \ if (mtx_recursed((mp))) \ (mp)->mtx_recurse--; \ - else { \ + else \ _release_lock_quick((mp)); \ - critical_exit(_mtx_crit); \ - } \ + critical_exit(); \ } while (0) #endif diff --git a/sys/sys/proc.h b/sys/sys/proc.h index beab316..ebf6d36 100644 --- a/sys/sys/proc.h +++ b/sys/sys/proc.h @@ -275,6 +275,8 @@ struct thread { struct trapframe *td_frame; /* (k) */ struct vm_object *td_kstack_obj;/* (a) Kstack object. */ vm_offset_t td_kstack; /* Kernel VA of kstack. */ + u_int td_critnest; /* (k) Critical section nest level. */ + critical_t td_savecrit; /* (k) Saved critical section state. */ }; /* diff --git a/sys/sys/systm.h b/sys/sys/systm.h index 3544b1f..70e99bd 100644 --- a/sys/sys/systm.h +++ b/sys/sys/systm.h @@ -121,6 +121,8 @@ void *phashinit __P((int count, struct malloc_type *type, u_long *nentries)); void cpu_boot __P((int)); void cpu_rootconf __P((void)); +void critical_enter __P((void)); +void critical_exit __P((void)); void init_param1 __P((void)); void init_param2 __P((int physpages)); void tablefull __P((const char *)); |
