From 769e0f974d8929599ba599ac496510fffc90ff34 Mon Sep 17 00:00:00 2001 From: jasone Date: Thu, 7 Sep 2000 01:33:02 +0000 Subject: Major update to the way synchronization is done in the kernel. Highlights include: * Mutual exclusion is used instead of spl*(). See mutex(9). (Note: The alpha port is still in transition and currently uses both.) * Per-CPU idle processes. * Interrupts are run in their own separate kernel threads and can be preempted (i386 only). Partially contributed by: BSDi (BSD/OS) Submissions by (at least): cp, dfr, dillon, grog, jake, jhb, sheldonh --- sys/amd64/include/cpu.h | 12 +- sys/amd64/include/cpufunc.h | 21 +- sys/amd64/include/mptable.h | 88 +++-- sys/amd64/include/mutex.h | 786 ++++++++++++++++++++++++++++++++++++++++++++ sys/amd64/include/pcb.h | 6 +- sys/amd64/include/pcpu.h | 33 ++ sys/amd64/include/smp.h | 38 +-- 7 files changed, 910 insertions(+), 74 deletions(-) create mode 100644 sys/amd64/include/mutex.h (limited to 'sys/amd64/include') diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h index ffabf7f..18822b8 100644 --- a/sys/amd64/include/cpu.h +++ b/sys/amd64/include/cpu.h @@ -46,6 +46,7 @@ #include #include #include +#include /* * definitions of cpu-dependent requirements @@ -86,7 +87,9 @@ * added, we will have an atomicy problem. The type of atomicy we need is * a non-locked orl. */ -#define need_resched() do { astpending = AST_RESCHED|AST_PENDING; } while (0) +#define need_resched() do { \ + PCPU_SET(astpending, AST_RESCHED|AST_PENDING); \ +} while (0) #define resched_wanted() (astpending & AST_RESCHED) /* @@ -109,8 +112,9 @@ * it off (asynchronous need_resched() conflicts are not critical). */ #define signotify(p) aston() - -#define aston() do { astpending |= AST_PENDING; } while (0) +#define aston() do { \ + PCPU_SET(astpending, astpending | AST_PENDING); \ +} while (0) #define astoff() /* @@ -135,7 +139,9 @@ #ifdef _KERNEL extern char btext[]; extern char etext[]; +#ifndef intr_nesting_level extern u_char intr_nesting_level; +#endif void fork_trampoline __P((void)); void fork_return __P((struct proc *, struct trapframe)); diff --git a/sys/amd64/include/cpufunc.h b/sys/amd64/include/cpufunc.h index 9a4052f..39868df 100644 --- a/sys/amd64/include/cpufunc.h +++ b/sys/amd64/include/cpufunc.h @@ -86,20 +86,29 @@ static __inline void disable_intr(void) { __asm __volatile("cli" : : : "memory"); -#ifdef SMP - MPINTR_LOCK(); -#endif } static __inline void enable_intr(void) { -#ifdef SMP - MPINTR_UNLOCK(); -#endif __asm __volatile("sti"); } +static __inline u_int +save_intr(void) +{ + u_int ef; + + __asm __volatile("pushfl; popl %0" : "=r" (ef)); + return (ef); +} + +static __inline void +restore_intr(u_int ef) +{ + __asm __volatile("pushl %0; popfl" : : "r" (ef) : "memory" ); +} + #define HAVE_INLINE_FFS static __inline int diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h index 61c5ecf..95b5759 100644 --- a/sys/amd64/include/mptable.h +++ b/sys/amd64/include/mptable.h @@ -36,6 +36,7 @@ #endif #include +#include #include #include #include @@ -65,6 +66,7 @@ #include #include #include +#include #include #include #include @@ -236,6 +238,8 @@ typedef struct BASETABLE_ENTRY { #define MP_ANNOUNCE_POST 0x19 +/* used to hold the AP's until we are ready to release them */ +struct simplelock ap_boot_lock; /** XXX FIXME: where does this really belong, isa.h/isa.c perhaps? */ int current_postcode; @@ -336,6 +340,7 @@ static int start_all_aps(u_int boot_addr); static void install_ap_tramp(u_int boot_addr); static int start_ap(int logicalCpu, u_int boot_addr); static int apic_int_is_bus_type(int intr, int bus_type); +static void release_aps(void *dummy); /* * Calculate usable address in base memory for AP trampoline code. @@ -403,7 +408,7 @@ found: /* - * Startup the SMP processors. + * Initialize the SMP hardware and the APIC and start up the AP's. */ void mp_start(void) @@ -619,6 +624,9 @@ mp_enable(u_int boot_addr) /* initialize all SMP locks */ init_locks(); + /* obtain the ap_boot_lock */ + s_lock(&ap_boot_lock); + /* start each Application Processor */ start_all_aps(boot_addr); } @@ -1866,9 +1874,6 @@ struct simplelock fast_intr_lock; /* critical region around INTR() routines */ struct simplelock intr_lock; -/* lock regions protected in UP kernel via cli/sti */ -struct simplelock mpintr_lock; - /* lock region used by kernel profiling */ struct simplelock mcount_lock; @@ -1885,26 +1890,16 @@ struct simplelock clock_lock; /* lock around the MP rendezvous */ static struct simplelock smp_rv_lock; +/* only 1 CPU can panic at a time :) */ +struct simplelock panic_lock; + static void init_locks(void) { - /* - * Get the initial mp_lock with a count of 1 for the BSP. - * This uses a LOGICAL cpu ID, ie BSP == 0. - */ - mp_lock = 0x00000001; - -#if 0 - /* ISR uses its own "giant lock" */ - isr_lock = FREE_LOCK; -#endif - #if defined(APIC_INTR_DIAGNOSTIC) && defined(APIC_INTR_DIAGNOSTIC_IRQ) s_lock_init((struct simplelock*)&apic_itrace_debuglock); #endif - s_lock_init((struct simplelock*)&mpintr_lock); - s_lock_init((struct simplelock*)&mcount_lock); s_lock_init((struct simplelock*)&fast_intr_lock); @@ -1912,6 +1907,7 @@ init_locks(void) s_lock_init((struct simplelock*)&imen_lock); s_lock_init((struct simplelock*)&cpl_lock); s_lock_init(&smp_rv_lock); + s_lock_init(&panic_lock); #ifdef USE_COMLOCK s_lock_init((struct simplelock*)&com_lock); @@ -1919,11 +1915,9 @@ init_locks(void) #ifdef USE_CLOCKLOCK s_lock_init((struct simplelock*)&clock_lock); #endif /* USE_CLOCKLOCK */ -} - -/* Wait for all APs to be fully initialized */ -extern int wait_ap(unsigned int); + s_lock_init(&ap_boot_lock); +} /* * start each AP in our list @@ -1987,6 +1981,7 @@ start_all_aps(u_int boot_addr) SMPpt[pg + 4] = 0; /* *prv_PMAP1 */ /* prime data page for it to use */ + SLIST_INSERT_HEAD(&cpuhead, gd, gd_allcpu); gd->gd_cpuid = x; gd->gd_cpu_lockid = x << 24; gd->gd_prv_CMAP1 = &SMPpt[pg + 1]; @@ -2211,7 +2206,6 @@ start_ap(int logical_cpu, u_int boot_addr) return 0; /* return FAILURE */ } - /* * Flush the TLB on all other CPU's * @@ -2348,10 +2342,13 @@ SYSCTL_INT(_machdep, OID_AUTO, forward_roundrobin_enabled, CTLFLAG_RW, void ap_init(void); void -ap_init() +ap_init(void) { u_int apic_id; + /* lock against other AP's that are waking up */ + s_lock(&ap_boot_lock); + /* BSP may have changed PTD while we're waiting for the lock */ cpu_invltlb(); @@ -2397,6 +2394,30 @@ ap_init() smp_started = 1; /* enable IPI's, tlb shootdown, freezes etc */ smp_active = 1; /* historic */ } + + /* let other AP's wake up now */ + s_unlock(&ap_boot_lock); + + /* wait until all the AP's are up */ + while (smp_started == 0) + ; /* nothing */ + + /* + * Set curproc to our per-cpu idleproc so that mutexes have + * something unique to lock with. + */ + PCPU_SET(curproc,idleproc); + PCPU_SET(prevproc,idleproc); + + microuptime(&switchtime); + switchticks = ticks; + + /* ok, now grab sched_lock and enter the scheduler */ + enable_intr(); + mtx_enter(&sched_lock, MTX_SPIN); + cpu_throw(); /* doesn't return */ + + panic("scheduler returned us to ap_init"); } #ifdef BETTER_CLOCK @@ -2453,6 +2474,12 @@ forwarded_statclock(int id, int pscnt, int *astmap) p = checkstate_curproc[id]; cpustate = checkstate_cpustate[id]; + /* XXX */ + if (p->p_ithd) + cpustate = CHECKSTATE_INTR; + else if (p == idleproc) + cpustate = CHECKSTATE_SYS; + switch (cpustate) { case CHECKSTATE_USER: if (p->p_flag & P_PROFIL) @@ -2482,9 +2509,10 @@ forwarded_statclock(int id, int pscnt, int *astmap) if (pscnt > 1) return; - if (!p) + if (p == idleproc) { + p->p_sticks++; cp_time[CP_IDLE]++; - else { + } else { p->p_sticks++; cp_time[CP_SYS]++; } @@ -2510,7 +2538,7 @@ forwarded_statclock(int id, int pscnt, int *astmap) p->p_iticks++; cp_time[CP_INTR]++; } - if (p != NULL) { + if (p != idleproc) { schedclock(p); /* Update resource usage integrals and maximums. */ @@ -2863,3 +2891,11 @@ smp_rendezvous(void (* setup_func)(void *), /* release lock */ s_unlock(&smp_rv_lock); } + +void +release_aps(void *dummy __unused) +{ + s_unlock(&ap_boot_lock); +} + +SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, release_aps, NULL); diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h new file mode 100644 index 0000000..ef0c963 --- /dev/null +++ b/sys/amd64/include/mutex.h @@ -0,0 +1,786 @@ +/*- + * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Berkeley Software Design Inc's name may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from BSDI $Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp Exp $ + * $FreeBSD$ + */ + +#ifndef _MACHINE_MUTEX_H_ +#define _MACHINE_MUTEX_H_ + +#ifndef LOCORE + +#include +#include +#include +#include +#include + +/* + * If kern_mutex.c is being built, compile non-inlined versions of various + * functions so that kernel modules can use them. + */ +#ifndef _KERN_MUTEX_C_ +#define _MTX_INLINE static __inline +#else +#define _MTX_INLINE +#endif + +/* + * Mutex flags + * + * Types + */ +#define MTX_DEF 0x0 /* Default (spin/sleep) */ +#define MTX_SPIN 0x1 /* Spin only lock */ + +/* Options */ +#define MTX_RLIKELY 0x4 /* (opt) Recursion likely */ +#define MTX_NORECURSE 0x8 /* No recursion possible */ +#define MTX_NOSPIN 0x10 /* Don't spin before sleeping */ +#define MTX_NOSWITCH 0x20 /* Do not switch on release */ +#define MTX_FIRST 0x40 /* First spin lock holder */ +#define MTX_TOPHALF 0x80 /* Interrupts not disabled on spin */ + +/* options that should be passed on to mtx_enter_hard, mtx_exit_hard */ +#define MTX_HARDOPTS (MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH) + +/* Flags/value used in mtx_lock */ +#define MTX_RECURSE 0x01 /* (non-spin) lock held recursively */ +#define MTX_CONTESTED 0x02 /* (non-spin) lock contested */ +#define MTX_FLAGMASK ~(MTX_RECURSE | MTX_CONTESTED) +#define MTX_UNOWNED 0x8 /* Cookie for free mutex */ + +struct proc; /* XXX */ + +/* + * Sleep/spin mutex + */ +struct mtx { + volatile u_int mtx_lock; /* lock owner/gate/flags */ + volatile u_short mtx_recurse; /* number of recursive holds */ + u_short mtx_f1; + u_int mtx_savefl; /* saved flags (for spin locks) */ + char *mtx_description; + TAILQ_HEAD(, proc) mtx_blocked; + LIST_ENTRY(mtx) mtx_contested; + struct mtx *mtx_next; /* all locks in system */ + struct mtx *mtx_prev; +#ifdef SMP_DEBUG + /* If you add anything here, adjust the mtxf_t definition below */ + struct witness *mtx_witness; + LIST_ENTRY(mtx) mtx_held; + char *mtx_file; + int mtx_line; +#endif /* SMP_DEBUG */ +}; + +typedef struct mtx mtx_t; + +/* + * Filler for structs which need to remain the same size + * whether or not SMP_DEBUG is turned on. + */ +typedef struct mtxf { +#ifdef SMP_DEBUG + char mtxf_data[0]; +#else + char mtxf_data[4*sizeof(void *) + sizeof(int)]; +#endif +} mtxf_t; + +#define mp_fixme(string) + +#ifdef _KERNEL +/* Misc */ +#define CURTHD ((u_int)CURPROC) /* Current thread ID */ + +/* Prototypes */ +void mtx_init(mtx_t *m, char *description, int flag); +void mtx_enter_hard(mtx_t *, int type, int flags); +void mtx_exit_hard(mtx_t *, int type); +void mtx_destroy(mtx_t *m); + +#if (defined(KLD_MODULE) || defined(_KERN_MUTEX_C_)) +void mtx_enter(mtx_t *mtxp, int type); +int mtx_try_enter(mtx_t *mtxp, int type); +void mtx_exit(mtx_t *mtxp, int type); +#endif + +/* Global locks */ +extern mtx_t sched_lock; +extern mtx_t Giant; + +/* + * Used to replace return with an exit Giant and return. + */ + +#define EGAR(a) \ +do { \ + mtx_exit(&Giant, MTX_DEF); \ + return (a); \ +} while (0) + +#define VEGAR \ +do { \ + mtx_exit(&Giant, MTX_DEF); \ + return; \ +} while (0) + +#define DROP_GIANT() \ +do { \ + int _giantcnt; \ + WITNESS_SAVE_DECL(Giant); \ + \ + WITNESS_SAVE(&Giant, Giant); \ + for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \ + mtx_exit(&Giant, MTX_DEF) + +#define PICKUP_GIANT() \ + mtx_assert(&Giant, MA_NOTOWNED); \ + while (_giantcnt--) \ + mtx_enter(&Giant, MTX_DEF); \ + WITNESS_RESTORE(&Giant, Giant); \ +} while (0) + +#define PARTIAL_PICKUP_GIANT() \ + mtx_assert(&Giant, MA_NOTOWNED); \ + while (_giantcnt--) \ + mtx_enter(&Giant, MTX_DEF); \ + WITNESS_RESTORE(&Giant, Giant) + + +/* + * Debugging + */ +#ifndef SMP_DEBUG +#define mtx_assert(m, what) +#else /* SMP_DEBUG */ + +#define MA_OWNED 1 +#define MA_NOTOWNED 2 +#define mtx_assert(m, what) { \ + switch ((what)) { \ + case MA_OWNED: \ + ASS(mtx_owned((m))); \ + break; \ + case MA_NOTOWNED: \ + ASS(!mtx_owned((m))); \ + break; \ + default: \ + panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__); \ + } \ +} + +#ifdef INVARIANTS +#define ASS(ex) MPASS(ex) +#define MPASS(ex) if (!(ex)) panic("Assertion %s failed at %s:%d", \ + #ex, __FILE__, __LINE__) +#define MPASS2(ex, what) if (!(ex)) panic("Assertion %s failed at %s:%d", \ + what, __FILE__, __LINE__) + +#ifdef MTX_STRS +char STR_IEN[] = "fl & 0x200"; +char STR_IDIS[] = "!(fl & 0x200)"; +#else /* MTX_STRS */ +extern char STR_IEN[]; +extern char STR_IDIS[]; +#endif /* MTX_STRS */ +#define ASS_IEN MPASS2(read_eflags() & 0x200, STR_IEN) +#define ASS_IDIS MPASS2((read_eflags() & 0x200) == 0, STR_IDIS) +#endif /* INVARIANTS */ + +#endif /* SMP_DEBUG */ + +#if !defined(SMP_DEBUG) || !defined(INVARIANTS) +#define ASS(ex) +#define MPASS(ex) +#define MPASS2(ex, where) +#define ASS_IEN +#define ASS_IDIS +#endif /* !defined(SMP_DEBUG) || !defined(INVARIANTS) */ + +#ifdef WITNESS +#ifndef SMP_DEBUG +#error WITNESS requires SMP_DEBUG +#endif /* SMP_DEBUG */ +#define WITNESS_ENTER(m, f) \ + if ((m)->mtx_witness != NULL) \ + witness_enter((m), (f), __FILE__, __LINE__) +#define WITNESS_EXIT(m, f) \ + if ((m)->mtx_witness != NULL) \ + witness_exit((m), (f), __FILE__, __LINE__) + +#define WITNESS_SLEEP(check, m) witness_sleep(check, (m), __FILE__, __LINE__) +#define WITNESS_SAVE_DECL(n) \ + char * __CONCAT(n, __wf); \ + int __CONCAT(n, __wl) + +#define WITNESS_SAVE(m, n) \ +do { \ + if ((m)->mtx_witness != NULL) \ + witness_save(m, &__CONCAT(n, __wf), &__CONCAT(n, __wl)); \ +} while (0) + +#define WITNESS_RESTORE(m, n) \ +do { \ + if ((m)->mtx_witness != NULL) \ + witness_restore(m, __CONCAT(n, __wf), __CONCAT(n, __wl)); \ +} while (0) + +void witness_init(mtx_t *, int flag); +void witness_destroy(mtx_t *); +void witness_enter(mtx_t *, int, char *, int); +void witness_try_enter(mtx_t *, int, char *, int); +void witness_exit(mtx_t *, int, char *, int); +void witness_display(void(*)(const char *fmt, ...)); +void witness_list(struct proc *); +int witness_sleep(int, mtx_t *, char *, int); +void witness_save(mtx_t *, char **, int *); +void witness_restore(mtx_t *, char *, int); +#else /* WITNESS */ +#define WITNESS_ENTER(m, flag) +#define WITNESS_EXIT(m, flag) +#define WITNESS_SLEEP(check, m) +#define WITNESS_SAVE_DECL(n) +#define WITNESS_SAVE(m, n) +#define WITNESS_RESTORE(m, n) + +/* + * flag++ is slezoid way of shutting up unused parameter warning + * in mtx_init() + */ +#define witness_init(m, flag) flag++ +#define witness_destroy(m) +#define witness_enter(m, flag, f, l) +#define witness_try_enter(m, flag, f, l ) +#define witness_exit(m, flag, f, l) +#endif /* WITNESS */ + +/* + * Assembly macros (for internal use only) + *------------------------------------------------------------------------------ + */ + +#define _V(x) __STRING(x) + +#ifndef I386_CPU + +/* + * For 486 and newer processors. + */ + +/* Get a sleep lock, deal with recursion inline. */ +#define _getlock_sleep(mtxp, tid, type) ({ \ + int _res; \ + \ + __asm __volatile ( \ +" movl $" _V(MTX_UNOWNED) ",%%eax;" /* Unowned cookie */ \ +" " MPLOCKED "" \ +" cmpxchgl %3,%1;" /* Try */ \ +" jz 1f;" /* Got it */ \ +" andl $" _V(MTX_FLAGMASK) ",%%eax;" /* turn off spec bits */ \ +" cmpl %%eax,%3;" /* already have it? */ \ +" je 2f;" /* yes, recurse */ \ +" pushl %4;" \ +" pushl %5;" \ +" call mtx_enter_hard;" \ +" addl $8,%%esp;" \ +" jmp 1f;" \ +"2: lock; orl $" _V(MTX_RECURSE) ",%1;" \ +" incw %2;" \ +"1:" \ +"# getlock_sleep" \ + : "=&a" (_res), /* 0 (dummy output) */ \ + "+m" (mtxp->mtx_lock), /* 1 */ \ + "+m" (mtxp->mtx_recurse) /* 2 */ \ + : "r" (tid), /* 3 (input) */ \ + "gi" (type), /* 4 */ \ + "g" (mtxp) /* 5 */ \ + : "memory", "ecx", "edx" /* used */ ); \ +}) + +/* Get a spin lock, handle recursion inline (as the less common case) */ +#define _getlock_spin_block(mtxp, tid, type) ({ \ + int _res; \ + \ + __asm __volatile ( \ +" pushfl;" \ +" cli;" \ +" movl $" _V(MTX_UNOWNED) ",%%eax;" /* Unowned cookie */ \ +" " MPLOCKED "" \ +" cmpxchgl %3,%1;" /* Try */ \ +" jz 2f;" /* got it */ \ +" pushl %4;" \ +" pushl %5;" \ +" call mtx_enter_hard;" /* mtx_enter_hard(mtxp, type, oflags) */ \ +" addl $0xc,%%esp;" \ +" jmp 1f;" \ +"2: popl %2;" /* save flags */ \ +"1:" \ +"# getlock_spin_block" \ + : "=&a" (_res), /* 0 (dummy output) */ \ + "+m" (mtxp->mtx_lock), /* 1 */ \ + "=m" (mtxp->mtx_savefl) /* 2 */ \ + : "r" (tid), /* 3 (input) */ \ + "gi" (type), /* 4 */ \ + "g" (mtxp) /* 5 */ \ + : "memory", "ecx", "edx" /* used */ ); \ +}) + +/* + * Get a lock without any recursion handling. Calls the hard enter function if + * we can't get it inline. + */ +#define _getlock_norecurse(mtxp, tid, type) ({ \ + int _res; \ + \ + __asm __volatile ( \ +" movl $" _V(MTX_UNOWNED) ",%%eax;" /* Unowned cookie */ \ +" " MPLOCKED "" \ +" cmpxchgl %2,%1;" /* Try */ \ +" jz 1f;" /* got it */ \ +" pushl %3;" \ +" pushl %4;" \ +" call mtx_enter_hard;" /* mtx_enter_hard(mtxp, type) */ \ +" addl $8,%%esp;" \ +"1:" \ +"# getlock_norecurse" \ + : "=&a" (_res), /* 0 (dummy output) */ \ + "+m" (mtxp->mtx_lock) /* 1 */ \ + : "r" (tid), /* 2 (input) */ \ + "gi" (type), /* 3 */ \ + "g" (mtxp) /* 4 */ \ + : "memory", "ecx", "edx" /* used */ ); \ +}) + +/* + * Release a sleep lock assuming we haven't recursed on it, recursion is handled + * in the hard function. + */ +#define _exitlock_norecurse(mtxp, tid, type) ({ \ + int _tid = (int)(tid); \ + \ + __asm __volatile ( \ +" " MPLOCKED "" \ +" cmpxchgl %4,%0;" /* try easy rel */ \ +" jz 1f;" /* released! */ \ +" pushl %2;" \ +" pushl %3;" \ +" call mtx_exit_hard;" \ +" addl $8,%%esp;" \ +"1:" \ +"# exitlock_norecurse" \ + : "+m" (mtxp->mtx_lock), /* 0 */ \ + "+a" (_tid) /* 1 */ \ + : "gi" (type), /* 2 (input) */ \ + "g" (mtxp), /* 3 */ \ + "r" (MTX_UNOWNED) /* 4 */ \ + : "memory", "ecx", "edx" /* used */ ); \ +}) + +/* + * Release a sleep lock when its likely we recursed (the code to + * deal with simple recursion is inline). + */ +#define _exitlock(mtxp, tid, type) ({ \ + int _tid = (int)(tid); \ + \ + __asm __volatile ( \ +" " MPLOCKED "" \ +" cmpxchgl %5,%0;" /* try easy rel */ \ +" jz 1f;" /* released! */ \ +" testl $" _V(MTX_RECURSE) ",%%eax;" /* recursed? */ \ +" jnz 3f;" /* handle recursion */ \ + /* Lock not recursed and contested: do the hard way */ \ +" pushl %3;" \ +" pushl %4;" \ +" call mtx_exit_hard;" /* mtx_exit_hard(mtxp,type) */ \ +" addl $8,%%esp;" \ +" jmp 1f;" \ + /* lock recursed, lower recursion level */ \ +"3: decw %1;" /* one less level */ \ +" jnz 1f;" /* still recursed, done */ \ +" lock; andl $~" _V(MTX_RECURSE) ",%0;" /* turn off recurse flag */ \ +"1:" \ +"# exitlock" \ + : "+m" (mtxp->mtx_lock), /* 0 */ \ + "+m" (mtxp->mtx_recurse), /* 1 */ \ + "+a" (_tid) /* 2 */ \ + : "gi" (type), /* 3 (input) */ \ + "g" (mtxp), /* 4 */ \ + "r" (MTX_UNOWNED) /* 5 */ \ + : "memory", "ecx", "edx" /* used */ ); \ +}) + +/* + * Release a spin lock (with possible recursion). + * + * We use cmpxchgl to clear lock (instead of simple store) to flush posting + * buffers and make the change visible to other CPU's. + */ +#define _exitlock_spin(mtxp, inten1, inten2) ({ \ + int _res; \ + \ + __asm __volatile ( \ +" movw %1,%%ax;" \ +" decw %%ax;" \ +" js 1f;" \ +" movw %%ax,%1;" \ +" jmp 2f;" \ +"1: movl %0,%%eax;" \ +" movl $ " _V(MTX_UNOWNED) ",%%ecx;" \ +" " inten1 ";" \ +" " MPLOCKED "" \ +" cmpxchgl %%ecx,%0;" \ +" " inten2 ";" \ +"2:" \ +"# exitlock_spin" \ + : "+m" (mtxp->mtx_lock), /* 0 */ \ + "+m" (mtxp->mtx_recurse), /* 1 */ \ + "=&a" (_res) /* 2 */ \ + : "g" (mtxp->mtx_savefl) /* 3 (used in 'inten') */ \ + : "memory", "ecx" /* used */ ); \ +}) + +#else /* I386_CPU */ + +/* + * For 386 processors only. + */ + +/* Get a sleep lock, deal with recursion inline. */ +#define _getlock_sleep(mp, tid, type) do { \ + if (atomic_cmpset_int(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) { \ + if (((mp)->mtx_lock & MTX_FLAGMASK) != (tid)) \ + mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0); \ + else { \ + atomic_set_int(&(mp)->mtx_lock, MTX_RECURSE); \ + (mp)->mtx_recurse++; \ + } \ + } \ +} while (0) + +/* Get a spin lock, handle recursion inline (as the less common case) */ +#define _getlock_spin_block(mp, tid, type) do { \ + u_int _mtx_fl = read_eflags(); \ + disable_intr(); \ + if (atomic_cmpset_int(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) \ + mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _mtx_fl); \ + else \ + (mp)->mtx_savefl = _mtx_fl; \ +} while (0) + +/* + * Get a lock without any recursion handling. Calls the hard enter function if + * we can't get it inline. + */ +#define _getlock_norecurse(mp, tid, type) do { \ + if (atomic_cmpset_int(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) \ + mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0); \ +} while (0) + +/* + * Release a sleep lock assuming we haven't recursed on it, recursion is handled + * in the hard function. + */ +#define _exitlock_norecurse(mp, tid, type) do { \ + if (atomic_cmpset_int(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0) \ + mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ +} while (0) + +/* + * Release a sleep lock when its likely we recursed (the code to + * deal with simple recursion is inline). + */ +#define _exitlock(mp, tid, type) do { \ + if (atomic_cmpset_int(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0) { \ + if ((mp)->mtx_lock & MTX_RECURSE) { \ + if (--((mp)->mtx_recurse) == 0) \ + atomic_clear_int(&(mp)->mtx_lock, \ + MTX_RECURSE); \ + } else { \ + mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ + } \ + } \ +} while (0) + +/* Release a spin lock (with possible recursion). */ +#define _exitlock_spin(mp, inten1, inten2) do { \ + if ((mp)->mtx_recurse == 0) { \ + atomic_cmpset_int(&(mp)->mtx_lock, (mp)->mtx_lock, \ + MTX_UNOWNED); \ + write_eflags((mp)->mtx_savefl); \ + } else { \ + (mp)->mtx_recurse--; \ + } \ +} while (0) + +#endif /* I386_CPU */ + +/* + * Externally visible mutex functions. + *------------------------------------------------------------------------------ + */ + +/* + * Return non-zero if a mutex is already owned by the current thread. + */ +#define mtx_owned(m) (((m)->mtx_lock & MTX_FLAGMASK) == CURTHD) + +/* Common strings */ +#ifdef MTX_STRS +#ifdef KTR_EXTEND + +/* + * KTR_EXTEND saves file name and line for all entries, so we don't need them + * here. Theoretically we should also change the entries which refer to them + * (from CTR5 to CTR3), but since they're just passed to snprinf as the last + * parameters, it doesn't do any harm to leave them. + */ +char STR_mtx_enter_fmt[] = "GOT %s [%x] r=%d"; +char STR_mtx_exit_fmt[] = "REL %s [%x] r=%d"; +char STR_mtx_try_enter_fmt[] = "TRY_ENTER %s [%x] result=%d"; +#else +char STR_mtx_enter_fmt[] = "GOT %s [%x] at %s:%d r=%d"; +char STR_mtx_exit_fmt[] = "REL %s [%x] at %s:%d r=%d"; +char STR_mtx_try_enter_fmt[] = "TRY_ENTER %s [%x] at %s:%d result=%d"; +#endif +char STR_mtx_bad_type[] = "((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0"; +char STR_mtx_owned[] = "mtx_owned(_mpp)"; +char STR_mtx_recurse[] = "_mpp->mtx_recurse == 0"; +#else /* MTX_STRS */ +extern char STR_mtx_enter_fmt[]; +extern char STR_mtx_bad_type[]; +extern char STR_mtx_exit_fmt[]; +extern char STR_mtx_owned[]; +extern char STR_mtx_recurse[]; +extern char STR_mtx_try_enter_fmt[]; +#endif /* MTX_STRS */ + +#ifndef KLD_MODULE +/* + * Get lock 'm', the macro handles the easy (and most common cases) and leaves + * the slow stuff to the mtx_enter_hard() function. + * + * Note: since type is usually a constant much of this code is optimized out. + */ +_MTX_INLINE void +mtx_enter(mtx_t *mtxp, int type) +{ + mtx_t *_mpp = mtxp; + + /* bits only valid on mtx_exit() */ + MPASS2(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0, + STR_mtx_bad_type); + + do { + if ((type) & MTX_SPIN) { + /* + * Easy cases of spin locks: + * + * 1) We already own the lock and will simply + * recurse on it (if RLIKELY) + * + * 2) The lock is free, we just get it + */ + if ((type) & MTX_RLIKELY) { + /* + * Check for recursion, if we already + * have this lock we just bump the + * recursion count. + */ + if (_mpp->mtx_lock == CURTHD) { + _mpp->mtx_recurse++; + break; /* Done */ + } + } + + if (((type) & MTX_TOPHALF) == 0) { + /* + * If an interrupt thread uses this + * we must block interrupts here. + */ + if ((type) & MTX_FIRST) { + ASS_IEN; + disable_intr(); + _getlock_norecurse(_mpp, CURTHD, + (type) & MTX_HARDOPTS); + } else { + _getlock_spin_block(_mpp, CURTHD, + (type) & MTX_HARDOPTS); + } + } else + _getlock_norecurse(_mpp, CURTHD, + (type) & MTX_HARDOPTS); + } else { + /* Sleep locks */ + if ((type) & MTX_RLIKELY) + _getlock_sleep(_mpp, CURTHD, + (type) & MTX_HARDOPTS); + else + _getlock_norecurse(_mpp, CURTHD, + (type) & MTX_HARDOPTS); + } + } while (0); + WITNESS_ENTER(_mpp, type); + CTR5(KTR_LOCK, STR_mtx_enter_fmt, + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, + (_mpp)->mtx_recurse); +} + +/* + * Attempt to get MTX_DEF lock, return non-zero if lock acquired. + * + * XXX DOES NOT HANDLE RECURSION + */ +_MTX_INLINE int +mtx_try_enter(mtx_t *mtxp, int type) +{ + mtx_t *const _mpp = mtxp; + int _rval; + + _rval = atomic_cmpset_int(&_mpp->mtx_lock, MTX_UNOWNED, CURTHD); +#ifdef SMP_DEBUG + if (_rval && (_mpp)->mtx_witness != NULL) { + ASS((_mpp)->mtx_recurse == 0); + witness_try_enter(_mpp, type, __FILE__, __LINE__); + } +#endif + CTR5(KTR_LOCK, STR_mtx_try_enter_fmt, + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, _rval); + + return _rval; +} + +#define mtx_legal2block() (read_eflags() & 0x200) + +/* + * Release lock m. + */ +_MTX_INLINE void +mtx_exit(mtx_t *mtxp, int type) +{ + mtx_t *const _mpp = mtxp; + + MPASS2(mtx_owned(_mpp), STR_mtx_owned); + WITNESS_EXIT(_mpp, type); + CTR5(KTR_LOCK, STR_mtx_exit_fmt, + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, + (_mpp)->mtx_recurse); + if ((type) & MTX_SPIN) { + if ((type) & MTX_NORECURSE) { + MPASS2(_mpp->mtx_recurse == 0, STR_mtx_recurse); + atomic_cmpset_int(&_mpp->mtx_lock, _mpp->mtx_lock, + MTX_UNOWNED); + if (((type) & MTX_TOPHALF) == 0) { + if ((type) & MTX_FIRST) { + ASS_IDIS; + enable_intr(); + } else + write_eflags(_mpp->mtx_savefl); + } + } else { + if ((type) & MTX_TOPHALF) + _exitlock_spin(_mpp,,); + else { + if ((type) & MTX_FIRST) { + ASS_IDIS; + _exitlock_spin(_mpp,, "sti"); + } else { + _exitlock_spin(_mpp, + "pushl %3", "popfl"); + } + } + } + } else { + /* Handle sleep locks */ + if ((type) & MTX_RLIKELY) + _exitlock(_mpp, CURTHD, (type) & MTX_HARDOPTS); + else { + _exitlock_norecurse(_mpp, CURTHD, + (type) & MTX_HARDOPTS); + } + } +} + +#endif /* KLD_MODULE */ +#endif /* _KERNEL */ + +#else /* !LOCORE */ + +/* + * Simple assembly macros to get and release non-recursive spin locks + */ + +#if defined(I386_CPU) + +#define MTX_EXIT(lck, reg) \ + movl $ MTX_UNOWNED,lck+MTX_LOCK; + +#else /* I386_CPU */ + +#define MTX_ENTER(reg, lck) \ +9: movl $ MTX_UNOWNED,%eax; \ + MPLOCKED \ + cmpxchgl reg,lck+MTX_LOCK; \ + jnz 9b + +/* Must use locked bus op (cmpxchg) when setting to unowned (barrier) */ +#define MTX_EXIT(lck,reg) \ + movl lck+MTX_LOCK,%eax; \ + movl $ MTX_UNOWNED,reg; \ + MPLOCKED \ + cmpxchgl reg,lck+MTX_LOCK; \ + +#define MTX_ENTER_WITH_RECURSION(reg, lck) \ + movl lck+MTX_LOCK,%eax; \ + cmpl PCPU_CURPROC,%eax; \ + jne 9f; \ + incw lck+MTX_RECURSECNT; \ + jmp 8f; \ +9: movl $ MTX_UNOWNED,%eax; \ + MPLOCKED \ + cmpxchgl reg,lck+MTX_LOCK; \ + jnz 9b; \ +8: + +#define MTX_EXIT_WITH_RECURSION(lck,reg) \ + movw lck+MTX_RECURSECNT,%ax; \ + decw %ax; \ + js 9f; \ + movw %ax,lck+MTX_RECURSECNT; \ + jmp 8f; \ +9: movl lck+MTX_LOCK,%eax; \ + movl $ MTX_UNOWNED,reg; \ + MPLOCKED \ + cmpxchgl reg,lck+MTX_LOCK; \ +8: + +#endif /* I386_CPU */ +#endif /* !LOCORE */ +#endif /* __MACHINE_MUTEX_H */ diff --git a/sys/amd64/include/pcb.h b/sys/amd64/include/pcb.h index 08beb5a..1c7af85 100644 --- a/sys/amd64/include/pcb.h +++ b/sys/amd64/include/pcb.h @@ -72,11 +72,7 @@ struct pcb { #define FP_SOFTFP 0x01 /* process using software fltng pnt emulator */ #define PCB_DBREGS 0x02 /* process using debug registers */ caddr_t pcb_onfault; /* copyin/out fault recovery */ -#ifdef SMP - u_long pcb_mpnest; -#else - u_long pcb_mpnest_dontuse; -#endif + int pcb_schednest; int pcb_gs; struct pcb_ext *pcb_ext; /* optional pcb extension */ u_long __pcb_spare[3]; /* adjust to avoid core dump size changes */ diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h index 58bd9cf..440da60 100644 --- a/sys/amd64/include/pcpu.h +++ b/sys/amd64/include/pcpu.h @@ -26,6 +26,20 @@ * $FreeBSD$ */ +#ifndef _MACHINE_GLOBALDATA_H_ +#define _MACHINE_GLOBALDATA_H_ + +#include +#include +#include +#include +#include + +/* XXX */ +#ifdef KTR_PERCPU +#include +#endif + /* * This structure maps out the global data that needs to be kept on a * per-cpu basis. genassym uses this to generate offsets for the assembler @@ -41,11 +55,14 @@ struct globaldata { struct privatespace *gd_prvspace; /* self-reference */ struct proc *gd_curproc; + struct proc *gd_prevproc; struct proc *gd_npxproc; struct pcb *gd_curpcb; + struct proc *gd_idleproc; struct timeval gd_switchtime; struct i386tss gd_common_tss; int gd_switchticks; + int gd_intr_nesting_level; struct segment_descriptor gd_common_tssd; struct segment_descriptor *gd_tss_gdt; #ifdef USER_LDT @@ -67,8 +84,22 @@ struct globaldata { unsigned *gd_prv_PADDR1; #endif u_int gd_astpending; + SLIST_ENTRY(globaldata) gd_allcpu; + int gd_witness_spin_check; +#ifdef KTR_PERCPU +#ifdef KTR + volatile int gd_ktr_idx; + char *gd_ktr_buf; + char gd_ktr_buf_data[KTR_SIZE]; +#endif +#endif }; +extern struct globaldata globaldata; + +SLIST_HEAD(cpuhead, globaldata); +extern struct cpuhead cpuhead; + #ifdef SMP /* * This is the upper (0xff800000) address space layout that is per-cpu. @@ -93,3 +124,5 @@ struct privatespace { extern struct privatespace SMP_prvspace[]; #endif + +#endif /* ! _MACHINE_GLOBALDATA_H_ */ diff --git a/sys/amd64/include/smp.h b/sys/amd64/include/smp.h index 69b716b..20d4fa3 100644 --- a/sys/amd64/include/smp.h +++ b/sys/amd64/include/smp.h @@ -15,6 +15,9 @@ #ifdef _KERNEL +#ifdef I386_CPU +#error SMP not supported with I386_CPU +#endif #if defined(SMP) && !defined(APIC_IO) # error APIC_IO required for SMP, add "options APIC_IO" to your config file. #endif /* SMP && !APIC_IO */ @@ -57,23 +60,6 @@ extern int bootMP_size; /* functions in mpboot.s */ void bootMP __P((void)); -/* global data in mplock.s */ -extern u_int mp_lock; -extern u_int isr_lock; -#ifdef RECURSIVE_MPINTRLOCK -extern u_int mpintr_lock; -#endif /* RECURSIVE_MPINTRLOCK */ - -/* functions in mplock.s */ -void get_mplock __P((void)); -void rel_mplock __P((void)); -int try_mplock __P((void)); -#ifdef RECURSIVE_MPINTRLOCK -void get_mpintrlock __P((void)); -void rel_mpintrlock __P((void)); -int try_mpintrlock __P((void)); -#endif /* RECURSIVE_MPINTRLOCK */ - /* global data in apic_vector.s */ extern volatile u_int stopped_cpus; extern volatile u_int started_cpus; @@ -185,23 +171,7 @@ extern int smp_started; extern volatile int smp_idle_loops; #endif /* !LOCORE */ -#else /* !SMP && !APIC_IO */ - -/* - * Create dummy MP lock empties - */ - -static __inline void -get_mplock(void) -{ -} - -static __inline void -rel_mplock(void) -{ -} - -#endif +#endif /* SMP && !APIC_IO */ #endif /* _KERNEL */ #endif /* _MACHINE_SMP_H_ */ -- cgit v1.1