diff options
author | jasone <jasone@FreeBSD.org> | 2000-09-07 01:33:02 +0000 |
---|---|---|
committer | jasone <jasone@FreeBSD.org> | 2000-09-07 01:33:02 +0000 |
commit | 769e0f974d8929599ba599ac496510fffc90ff34 (patch) | |
tree | 9387522900085835de81e7830e570ef3f6b3ea80 /sys/powerpc | |
parent | acf1927de02afda4855ec278b1128fd9446405ea (diff) | |
download | FreeBSD-src-769e0f974d8929599ba599ac496510fffc90ff34.zip FreeBSD-src-769e0f974d8929599ba599ac496510fffc90ff34.tar.gz |
Major update to the way synchronization is done in the kernel. Highlights
include:
* Mutual exclusion is used instead of spl*(). See mutex(9). (Note: The
alpha port is still in transition and currently uses both.)
* Per-CPU idle processes.
* Interrupts are run in their own separate kernel threads and can be
preempted (i386 only).
Partially contributed by: BSDi (BSD/OS)
Submissions by (at least): cp, dfr, dillon, grog, jake, jhb, sheldonh
Diffstat (limited to 'sys/powerpc')
-rw-r--r-- | sys/powerpc/aim/vm_machdep.c | 11 | ||||
-rw-r--r-- | sys/powerpc/include/globaldata.h | 79 | ||||
-rw-r--r-- | sys/powerpc/include/globals.h | 63 | ||||
-rw-r--r-- | sys/powerpc/include/mutex.h | 563 | ||||
-rw-r--r-- | sys/powerpc/include/pcpu.h | 79 | ||||
-rw-r--r-- | sys/powerpc/powerpc/genassym.c | 19 | ||||
-rw-r--r-- | sys/powerpc/powerpc/vm_machdep.c | 11 |
7 files changed, 817 insertions, 8 deletions
diff --git a/sys/powerpc/aim/vm_machdep.c b/sys/powerpc/aim/vm_machdep.c index 8baea02..3831d67 100644 --- a/sys/powerpc/aim/vm_machdep.c +++ b/sys/powerpc/aim/vm_machdep.c @@ -84,6 +84,7 @@ #include <machine/fpu.h> #include <machine/md_var.h> #include <machine/prom.h> +#include <machine/mutex.h> #include <vm/vm.h> #include <vm/vm_param.h> @@ -246,8 +247,10 @@ cpu_exit(p) alpha_fpstate_drop(p); (void) splhigh(); + mtx_enter(&sched_lock, MTX_SPIN); + mtx_exit(&Giant, MTX_DEF); cnt.v_swtch++; - cpu_switch(p); + cpu_switch(); panic("cpu_exit"); } @@ -358,7 +361,7 @@ vunmapbuf(bp) } /* - * Force reset the processor by invalidating the entire address space! + * Reset back to firmware. */ void cpu_reset() @@ -416,7 +419,7 @@ vm_page_zero_idle() return(0); #ifdef SMP - if (try_mplock()) { + if (KLOCK_ENTER(M_TRY)) { #endif s = splvm(); m = vm_page_list_find(PQ_FREE, free_rover, FALSE); @@ -447,7 +450,7 @@ vm_page_zero_idle() free_rover = (free_rover + PQ_PRIME2) & PQ_L2_MASK; splx(s); #ifdef SMP - rel_mplock(); + KLOCK_EXIT; #endif return (1); #ifdef SMP diff --git a/sys/powerpc/include/globaldata.h b/sys/powerpc/include/globaldata.h new file mode 100644 index 0000000..b246bb1 --- /dev/null +++ b/sys/powerpc/include/globaldata.h @@ -0,0 +1,79 @@ +/*- + * Copyright (c) 1999 Luoqi Chen <luoqi@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_GLOBALDATA_H_ +#define _MACHINE_GLOBALDATA_H_ + +#ifdef _KERNEL + +#include <sys/queue.h> + +/* + * This structure maps out the global data that needs to be kept on a + * per-cpu basis. genassym uses this to generate offsets for the assembler + * code, which also provides external symbols so that C can get at them as + * though they were really globals. This structure is pointed to by + * the per-cpu system value (see alpha_pal_rdval() and alpha_pal_wrval()). + * Inside the kernel, the globally reserved register t7 is used to + * point at the globaldata structure. + */ +struct globaldata { + struct alpha_pcb gd_idlepcb; /* pcb for idling */ + struct proc *gd_curproc; /* current process */ + struct proc *gd_idleproc; /* idle process */ + struct proc *gd_fpcurproc; /* fp state owner */ + struct pcb *gd_curpcb; /* current pcb */ + struct timeval gd_switchtime; + int gd_switchticks; + u_int gd_cpuno; /* this cpu number */ + u_int gd_other_cpus; /* all other cpus */ + int gd_inside_intr; + u_int64_t gd_idlepcbphys; /* pa of gd_idlepcb */ + u_int64_t gd_pending_ipis; /* pending IPI events */ + u_int32_t gd_next_asn; /* next ASN to allocate */ + u_int32_t gd_current_asngen; /* ASN rollover check */ + u_int32_t gd_intr_nesting_level; /* interrupt recursion */ + + u_int gd_astpending; + SLIST_ENTRY(globaldata) gd_allcpu; +#ifdef KTR_PERCPU + volatile int gd_ktr_idx; /* Index into trace table */ + char *gd_ktr_buf; + char gd_ktr_buf_data[0]; +#endif +}; + +SLIST_HEAD(cpuhead, globaldata); +extern struct cpuhead cpuhead; + +void globaldata_init(struct globaldata *pcpu, int cpuno, size_t sz); +struct globaldata *globaldata_find(int cpuno); + +#endif /* _KERNEL */ + +#endif /* !_MACHINE_GLOBALDATA_H_ */ diff --git a/sys/powerpc/include/globals.h b/sys/powerpc/include/globals.h new file mode 100644 index 0000000..303efdf --- /dev/null +++ b/sys/powerpc/include/globals.h @@ -0,0 +1,63 @@ +/*- + * Copyright (c) 1999 Luoqi Chen <luoqi@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_GLOBALS_H_ +#define _MACHINE_GLOBALS_H_ + +#ifdef _KERNEL + +register struct globaldata *globalp __asm__("$8"); + +#if 1 +#define GLOBALP globalp +#else +#define GLOBALP ((struct globaldata *) alpha_pal_rdval()) +#endif + +#define PCPU_GET(name) (GLOBALP->gd_##name) +#define PCPU_SET(name,value) (GLOBALP->gd_##name = (value)) + +/* + * The following set of macros works for UP kernel as well, but for maximum + * performance we allow the global variables to be accessed directly. On the + * other hand, kernel modules should always use these macros to maintain + * portability between UP and SMP kernels. + */ +#define CURPROC PCPU_GET(curproc) +#define curproc PCPU_GET(curproc) +#define idleproc PCPU_GET(idleproc) +#define curpcb PCPU_GET(curpcb) +#define fpcurproc PCPU_GET(fpcurproc) +#define switchtime PCPU_GET(switchtime) +#define switchticks PCPU_GET(switchticks) +#define cpuid PCPU_GET(cpuno) +#define prevproc PCPU_GET(curproc) /* XXX - until ithreads */ + +#endif /* _KERNEL */ + +#endif /* !_MACHINE_GLOBALS_H_ */ diff --git a/sys/powerpc/include/mutex.h b/sys/powerpc/include/mutex.h new file mode 100644 index 0000000..ac13b8c --- /dev/null +++ b/sys/powerpc/include/mutex.h @@ -0,0 +1,563 @@ +/*- + * Copyright (c) 1997 Berkeley Software Design, Inc. All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * 3. Berkeley Software Design Inc's name may not be used to endorse or + * promote products derived from this software without specific prior + * written permission. + * + * THIS SOFTWARE IS PROVIDED BY BERKELEY SOFTWARE DESIGN INC ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL BERKELEY SOFTWARE DESIGN INC BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * from BSDI $Id: mutex.h,v 2.7.2.35 2000/04/27 03:10:26 cp Exp $ + * $FreeBSD$ + */ + + +#ifndef _MACHINE_MUTEX_H_ +#define _MACHINE_MUTEX_H_ + +#ifndef LOCORE + +#include <sys/queue.h> +#include <machine/atomic.h> +#include <machine/cpufunc.h> +#include <machine/globaldata.h> + +/* + * Mutex flags + * + * Types + */ +#define MTX_DEF 0x1 /* Default (spin/sleep) */ +#define MTX_SPIN 0x2 /* Spin only lock */ + +/* Options */ +#define MTX_RLIKELY 0x4 /* (opt) Recursion likely */ +#define MTX_NORECURSE 0x8 /* No recursion possible */ +#define MTX_NOSPIN 0x10 /* Don't spin before sleeping */ +#define MTX_NOSWITCH 0x20 /* Do not switch on release */ +#define MTX_FIRST 0x40 /* First spin lock holder */ +#define MTX_TOPHALF 0x80 /* Interrupts not disabled on spin */ + +/* options that should be passed on to mtx_enter_hard, mtx_exit_hard */ +#define MTX_HARDOPTS (MTX_DEF | MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH) + +/* Flags/value used in mtx_lock */ +#define MTX_RECURSE 0x01 /* (non-spin) lock held recursively */ +#define MTX_CONTESTED 0x02 /* (non-spin) lock contested */ +#define MTX_FLAGMASK ~(MTX_RECURSE | MTX_CONTESTED) +#define MTX_UNOWNED 0x8 /* Cookie for free mutex */ + +struct proc; /* XXX */ + +/* + * Sleep/spin mutex + */ +struct mtx { + volatile u_int64_t mtx_lock; /* lock owner/gate/flags */ + volatile u_int32_t mtx_recurse; /* number of recursive holds */ + u_int32_t mtx_saveipl; /* saved ipl (for spin locks) */ + char *mtx_description; + TAILQ_HEAD(, proc) mtx_blocked; + LIST_ENTRY(mtx) mtx_contested; + struct mtx *mtx_next; /* all locks in system */ + struct mtx *mtx_prev; +#ifdef SMP_DEBUG + /* If you add anything here, adjust the mtxf_t definition below */ + struct witness *mtx_witness; + LIST_ENTRY(mtx) mtx_held; + char *mtx_file; + int mtx_line; +#endif /* SMP_DEBUG */ +}; + +typedef struct mtx mtx_t; + +/* + * Filler for structs which need to remain the same size + * whether or not SMP_DEBUG is turned on. + */ +typedef struct mtxf { +#ifdef SMP_DEBUG + char mtxf_data[0]; +#else + char mtxf_data[4*sizeof(void *) + sizeof(int)]; +#endif +} mtxf_t; + +#define mp_fixme(string) + +#ifdef _KERNEL +/* Misc */ +#define CURTHD ((u_int64_t)CURPROC) /* Current thread ID */ + +/* Prototypes */ +void mtx_init(mtx_t *m, char *description, int flag); +void mtx_enter_hard(mtx_t *, int type, int ipl); +void mtx_exit_hard(mtx_t *, int type); +void mtx_destroy(mtx_t *m); + +/* Global locks */ +extern mtx_t sched_lock; +extern mtx_t Giant; + +/* + * Used to replace return with an exit Giant and return. + */ + +#define EGAR(a) \ +do { \ + mtx_exit(&Giant, MTX_DEF); \ + return (a); \ +} while (0) + +#define VEGAR \ +do { \ + mtx_exit(&Giant, MTX_DEF); \ + return; \ +} while (0) + +#define DROP_GIANT() \ +do { \ + int _giantcnt; \ + WITNESS_SAVE_DECL(Giant); \ + \ + WITNESS_SAVE(&Giant, Giant); \ + for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++) \ + mtx_exit(&Giant, MTX_DEF) + +#define PICKUP_GIANT() \ + mtx_assert(&Giant, MA_NOTOWNED); \ + while (_giantcnt--) \ + mtx_enter(&Giant, MTX_DEF); \ + WITNESS_RESTORE(&Giant, Giant); \ +} while (0) + +#define PARTIAL_PICKUP_GIANT() \ + mtx_assert(&Giant, MA_NOTOWNED); \ + while (_giantcnt--) \ + mtx_enter(&Giant, MTX_DEF); \ + WITNESS_RESTORE(&Giant, Giant) + + +/* + * Debugging + */ +#ifndef SMP_DEBUG +#define mtx_assert(m, what) +#else /* SMP_DEBUG */ + +#define MA_OWNED 1 +#define MA_NOTOWNED 2 +#define mtx_assert(m, what) { \ + switch ((what)) { \ + case MA_OWNED: \ + ASS(mtx_owned((m))); \ + break; \ + case MA_NOTOWNED: \ + ASS(!mtx_owned((m))); \ + break; \ + default: \ + panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__); \ + } \ +} + +#ifdef INVARIANTS +#define ASS(ex) MPASS(ex) +#define MPASS(ex) if (!(ex)) panic("Assertion %s failed at %s:%d", \ + #ex, __FILE__, __LINE__) +#define MPASS2(ex, what) if (!(ex)) panic("Assertion %s failed at %s:%d", \ + what, __FILE__, __LINE__) + +#ifdef MTX_STRS +char STR_IEN[] = "fl & 0x200"; +char STR_IDIS[] = "!(fl & 0x200)"; +#else /* MTX_STRS */ +extern char STR_IEN[]; +extern char STR_IDIS[]; +#endif /* MTX_STRS */ +#define ASS_IEN MPASS2((alpha_pal_rdps & ALPHA_PSL_IPL_MASK) + == ALPHA_PSL_IPL_HIGH, STR_IEN) +#define ASS_IDIS MPASS2((alpha_pal_rdps & ALPHA_PSL_IPL_MASK) + != ALPHA_PSL_IPL_HIGH, STR_IDIS) +#endif /* INVARIANTS */ + +#endif /* SMP_DEBUG */ + +#if !defined(SMP_DEBUG) || !defined(INVARIANTS) +#define ASS(ex) +#define MPASS(ex) +#define MPASS2(ex, where) +#define ASS_IEN +#define ASS_IDIS +#endif /* !defined(SMP_DEBUG) || !defined(INVARIANTS) */ + +#ifdef WITNESS +#ifndef SMP_DEBUG +#error WITNESS requires SMP_DEBUG +#endif /* SMP_DEBUG */ +#define WITNESS_ENTER(m, f) \ + if ((m)->mtx_witness != NULL) \ + witness_enter((m), (f), __FILE__, __LINE__) +#define WITNESS_EXIT(m, f) \ + if ((m)->mtx_witness != NULL) \ + witness_exit((m), (f), __FILE__, __LINE__) + +#define WITNESS_SLEEP(check, m) witness_sleep(check, (m), __FILE__, __LINE__) +#define WITNESS_SAVE_DECL(n) \ + char * __CONCAT(n, __wf); \ + int __CONCAT(n, __wl) + +#define WITNESS_SAVE(m, n) \ +do { \ + if ((m)->mtx_witness != NULL) \ + witness_save(m, &__CONCAT(n, __wf), &__CONCAT(n, __wl)); \ +} while (0) + +#define WITNESS_RESTORE(m, n) \ +do { \ + if ((m)->mtx_witness != NULL) \ + witness_restore(m, __CONCAT(n, __wf), __CONCAT(n, __wl)); \ +} while (0) + +void witness_init(mtx_t *, int flag); +void witness_destroy(mtx_t *); +void witness_enter(mtx_t *, int, char *, int); +void witness_try_enter(mtx_t *, int, char *, int); +void witness_exit(mtx_t *, int, char *, int); +void witness_display(void(*)(const char *fmt, ...)); +void witness_list(struct proc *); +int witness_sleep(int, mtx_t *, char *, int); +void witness_save(mtx_t *, char **, int *); +void witness_restore(mtx_t *, char *, int); +#else /* WITNESS */ +#define WITNESS_ENTER(m, flag) +#define WITNESS_EXIT(m, flag) +#define WITNESS_SLEEP(check, m) +#define WITNESS_SAVE_DECL(n) +#define WITNESS_SAVE(m, n) +#define WITNESS_RESTORE(m, n) + +/* + * flag++ is slezoid way of shutting up unused parameter warning + * in mtx_init() + */ +#define witness_init(m, flag) flag++ +#define witness_destroy(m) +#define witness_enter(m, flag, f, l) +#define witness_try_enter(m, flag, f, l ) +#define witness_exit(m, flag, f, l) +#endif /* WITNESS */ + +/* + * Assembly macros (for internal use only) + *-------------------------------------------------------------------------- + */ + +/* + * Get a sleep lock, deal with recursion inline + */ + +#define _V(x) __STRING(x) + +#define _getlock_sleep(mp, tid, type) do { \ + if (atomic_cmpset_64(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) { \ + if (((mp)->mtx_lock & MTX_FLAGMASK) != (tid)) \ + mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0); \ + else { \ + if (((mp)->mtx_lock & MTX_RECURSE) == 0) \ + atomic_set_64(&(mp)->mtx_lock, MTX_RECURSE); \ + (mp)->mtx_recurse++; \ + } \ + } else { \ + alpha_mb(); \ + } \ +} while (0) + +/* + * Get a spin lock, handle recusion inline (as the less common case) + */ + +#define _getlock_spin_block(mp, tid, type) do { \ + u_int _ipl = alpha_pal_rdps() & ALPHA_PSL_IPL_MASK; \ + if (atomic_cmpset_64(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) \ + mtx_enter_hard(mp, (type) & MTX_HARDOPTS, _ipl); \ + else { \ + alpha_mb(); \ + (mp)->mtx_saveipl = _ipl; \ + } \ +} while (0) + +/* + * Get a lock without any recursion handling. Calls the hard enter + * function if we can't get it inline. + */ + +#define _getlock_norecurse(mp, tid, type) do { \ + if (atomic_cmpset_64(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) \ + mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0); \ + else \ + alpha_mb(); \ +} while (0) + +/* + * Release a sleep lock assuming we haven't recursed on it, recursion is + * handled in the hard function. + */ + +#define _exitlock_norecurse(mp, tid, type) do { \ + alpha_mb(); \ + if (atomic_cmpset_64(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0) \ + mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ +} while (0) + +/* + * Release a sleep lock when its likely we recursed (the code to + * deal with simple recursion is inline). + */ + +#define _exitlock(mp, tid, type) do { \ + alpha_mb(); \ + if (atomic_cmpset_64(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0) {\ + if (((mp)->mtx_lock & MTX_RECURSE) && \ + (--(mp)->mtx_recurse == 0)) \ + atomic_clear_64(&(mp)->mtx_lock, MTX_RECURSE); \ + else \ + mtx_exit_hard((mp), (type) & MTX_HARDOPTS); \ + } \ +} while (0) + +/* + * Release a spin lock (with possible recursion) + */ + +#define _exitlock_spin(mp) do { \ + int _ipl = (mp)->mtx_saveipl; \ + alpha_mb(); \ + if ((mp)->mtx_recurse == 0 || (--(mp)->mtx_recurse) == 0) \ + atomic_cmpset_64(&(mp)->mtx_lock, (mp)->mtx_lock, \ + MTX_UNOWNED); \ + alpha_pal_swpipl(_ipl); \ +} while (0) + +/* + * Externally visible mutex functions + *------------------------------------------------------------------------ + */ + +/* + * Return non-zero if a mutex is already owned by the current thread + */ +#define mtx_owned(m) (((m)->mtx_lock & MTX_FLAGMASK) == CURTHD) + +/* Common strings */ +#ifdef MTX_STRS +char STR_mtx_enter_fmt[] = "GOT %s [%p] at %s:%d r=%d"; +char STR_mtx_bad_type[] = "((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0"; +char STR_mtx_exit_fmt[] = "REL %s [%p] at %s:%d r=%d"; +char STR_mtx_owned[] = "mtx_owned(_mpp)"; +char STR_mtx_recurse[] = "_mpp->mtx_recurse == 0"; +char STR_mtx_try_enter_fmt[] = "TRY_ENTER %s [%p] at %s:%d result=%d"; +#else /* MTX_STRS */ +extern char STR_mtx_enter_fmt[]; +extern char STR_mtx_bad_type[]; +extern char STR_mtx_exit_fmt[]; +extern char STR_mtx_owned[]; +extern char STR_mtx_recurse[]; +extern char STR_mtx_try_enter_fmt[]; +#endif /* MTX_STRS */ + +/* + * Get lock 'm', the macro handles the easy (and most common cases) and + * leaves the slow stuff to the mtx_enter_hard() function. + * + * Note: since type is usually a constant much of this code is optimized out + */ +#define mtx_enter(mtxp, type) do { \ + mtx_t * _mpp = mtxp; \ + \ + /* bits only valid on mtx_exit() */ \ + MPASS2(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0, STR_mtx_bad_type); \ + \ + do { \ + if ((type) & MTX_SPIN) { \ + /* \ + * Easy cases of spin locks: \ + * \ + * 1) We already own the lock and will simply \ + * recurse on it (if RLIKELY) \ + * \ + * 2) The lock is free, we just get it \ + */ \ + if ((type) & MTX_RLIKELY) { \ + /* \ + * Check for recursion, if we already \ + * have this lock we just bump the \ + * recursion count. \ + */ \ + if (_mpp->mtx_lock == CURTHD) { \ + _mpp->mtx_recurse++; \ + break; /* Done */ \ + } \ + } \ + \ + if (((type) & MTX_TOPHALF) == 0) \ + /* \ + * If an interrupt thread uses this \ + * we must block interrupts here. \ + */ \ + _getlock_spin_block(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + else \ + _getlock_norecurse(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + } else { \ + /* Sleep locks */ \ + if ((type) & MTX_RLIKELY) \ + _getlock_sleep(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + else \ + _getlock_norecurse(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + } \ + } while (0); \ + WITNESS_ENTER(_mpp, type); \ + CTR5(KTR_LOCK, STR_mtx_enter_fmt, \ + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, \ + (_mpp)->mtx_recurse); \ +} while (0) + +/* + * Attempt to get MTX_DEF lock, return non-zero if lock acquired + * + * XXX DOES NOT HANDLE RECURSION + */ +#ifdef SMP_DEBUG +#define mtx_try_enter(mtxp, type) ({ \ + mtx_t *const _mpp = mtxp; \ + int _rval; \ + \ + _rval = atomic_cmpset_int(&_mpp->mtx_lock, MTX_UNOWNED, CURTHD);\ + if (_rval && (_mpp)->mtx_witness != NULL) { \ + ASS((_mpp)->mtx_recurse == 0); \ + witness_try_enter(_mpp, type, __FILE__, __LINE__); \ + } \ + CTR5(KTR_LOCK, STR_mtx_try_enter_fmt, \ + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, \ + _rval); \ + _rval; \ +}) + +#else /* SMP_DEBUG */ + +#define mtx_try_enter(mtxp, type) ({ \ + mtx_t *const _mpp = mtxp; \ + int _rval; \ + \ + _rval = atomic_cmpset_int(&_mpp->mtx_lock, MTX_UNOWNED, CURTHD);\ + CTR5(KTR_LOCK, STR_mtx_try_enter_fmt, \ + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, \ + _rval); \ + _rval; \ +}) + +#endif /* SMP_DEBUG */ + +#if 0 +#define mtx_legal2block() ({ \ + register int _l2b; \ + __asm __volatile ( \ +" pushfl;" \ +" popl %%eax;" \ +" andl $0x200, %%eax;" \ + : "=a" (_l2b) \ + : \ + : "cc"); \ + _l2b; \ +}) +#endif + +#define mtx_legal2block() (read_eflags() & 0x200) + +/* + * Release lock m + */ +#define mtx_exit(mtxp, type) do { \ + mtx_t *const _mpp = mtxp; \ + \ + MPASS2(mtx_owned(_mpp), STR_mtx_owned); \ + WITNESS_EXIT(_mpp, type); \ + CTR5(KTR_LOCK, STR_mtx_exit_fmt, \ + (_mpp)->mtx_description, (_mpp), __FILE__, __LINE__, \ + (_mpp)->mtx_recurse); \ + if ((type) & MTX_SPIN) { \ + if ((type) & MTX_NORECURSE) { \ + MPASS2(_mpp->mtx_recurse == 0, STR_mtx_recurse); \ + atomic_cmpset_64(&_mpp->mtx_lock, _mpp->mtx_lock, \ + MTX_UNOWNED); \ + if (((type) & MTX_TOPHALF) == 0) { \ + splx(_mpp->mtx_saveipl); \ + } \ + } else \ + if ((type) & MTX_TOPHALF) \ + _exitlock_norecurse(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + else \ + _exitlock_spin(_mpp); \ + } else { \ + /* Handle sleep locks */ \ + if ((type) & MTX_RLIKELY) \ + _exitlock(_mpp, CURTHD, (type) & MTX_HARDOPTS); \ + else \ + _exitlock_norecurse(_mpp, CURTHD, \ + (type) & MTX_HARDOPTS); \ + } \ +} while (0) +#endif /* _KERNEL */ + +#else /* !LOCORE */ + +/* + * Simple assembly macros to get and release non-recursive spin locks + */ +#define MTX_ENTER(lck) \ + call_pal PAL_OSF1_rdps; \ + and v0, ALPHA_PSL_IPL_MASK, v0; \ +1: ldq_l a0, lck+MTX_LOCK; \ + cmpeq a0, MTX_UNOWNED, a1; \ + beq a1, 1b; \ + ldq a0, PC_CURPROC(globalp); \ + stq_c a0, lck+MTX_LOCK; \ + beq a0, 1b; \ + mb; \ + stl v0, lck+MTX_SAVEIPL; \ + ldq a0, ALPHA_PSL_IPL_HIGH; \ + call_pal PSL_OSF1_swpipl + +#define MTX_EXIT(lck) \ + mb; \ + ldiq a0, MTX_UNOWNED; \ + stq a0, lck+MTX_LOCK; \ + ldl a0, lck+MTX_SAVEIPL; \ + call_pal PAL_OSF1_swpipl + +#endif /* !LOCORE */ + +#endif /* __MACHINE_MUTEX_H */ diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h new file mode 100644 index 0000000..b246bb1 --- /dev/null +++ b/sys/powerpc/include/pcpu.h @@ -0,0 +1,79 @@ +/*- + * Copyright (c) 1999 Luoqi Chen <luoqi@freebsd.org> + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND + * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE + * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE + * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE + * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL + * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS + * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) + * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT + * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY + * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF + * SUCH DAMAGE. + * + * $FreeBSD$ + */ + +#ifndef _MACHINE_GLOBALDATA_H_ +#define _MACHINE_GLOBALDATA_H_ + +#ifdef _KERNEL + +#include <sys/queue.h> + +/* + * This structure maps out the global data that needs to be kept on a + * per-cpu basis. genassym uses this to generate offsets for the assembler + * code, which also provides external symbols so that C can get at them as + * though they were really globals. This structure is pointed to by + * the per-cpu system value (see alpha_pal_rdval() and alpha_pal_wrval()). + * Inside the kernel, the globally reserved register t7 is used to + * point at the globaldata structure. + */ +struct globaldata { + struct alpha_pcb gd_idlepcb; /* pcb for idling */ + struct proc *gd_curproc; /* current process */ + struct proc *gd_idleproc; /* idle process */ + struct proc *gd_fpcurproc; /* fp state owner */ + struct pcb *gd_curpcb; /* current pcb */ + struct timeval gd_switchtime; + int gd_switchticks; + u_int gd_cpuno; /* this cpu number */ + u_int gd_other_cpus; /* all other cpus */ + int gd_inside_intr; + u_int64_t gd_idlepcbphys; /* pa of gd_idlepcb */ + u_int64_t gd_pending_ipis; /* pending IPI events */ + u_int32_t gd_next_asn; /* next ASN to allocate */ + u_int32_t gd_current_asngen; /* ASN rollover check */ + u_int32_t gd_intr_nesting_level; /* interrupt recursion */ + + u_int gd_astpending; + SLIST_ENTRY(globaldata) gd_allcpu; +#ifdef KTR_PERCPU + volatile int gd_ktr_idx; /* Index into trace table */ + char *gd_ktr_buf; + char gd_ktr_buf_data[0]; +#endif +}; + +SLIST_HEAD(cpuhead, globaldata); +extern struct cpuhead cpuhead; + +void globaldata_init(struct globaldata *pcpu, int cpuno, size_t sz); +struct globaldata *globaldata_find(int cpuno); + +#endif /* _KERNEL */ + +#endif /* !_MACHINE_GLOBALDATA_H_ */ diff --git a/sys/powerpc/powerpc/genassym.c b/sys/powerpc/powerpc/genassym.c index a67f2d1..066d87b 100644 --- a/sys/powerpc/powerpc/genassym.c +++ b/sys/powerpc/powerpc/genassym.c @@ -51,8 +51,11 @@ #include <sys/socket.h> #include <sys/resource.h> #include <sys/resourcevar.h> +#include <sys/ktr.h> #include <machine/frame.h> #include <machine/chipset.h> +#include <machine/globaldata.h> +#include <machine/mutex.h> #include <sys/vmmeter.h> #include <vm/vm.h> #include <vm/vm_param.h> @@ -66,6 +69,21 @@ #include <nfs/nfs.h> #include <nfs/nfsdiskless.h> +#include "opt_smp.h" + +ASSYM(GD_CURPROC, offsetof(struct globaldata, gd_curproc)); +ASSYM(GD_FPCURPROC, offsetof(struct globaldata, gd_fpcurproc)); +ASSYM(GD_CURPCB, offsetof(struct globaldata, gd_curpcb)); +ASSYM(GD_SWITCHTIME, offsetof(struct globaldata, gd_switchtime)); +ASSYM(GD_CPUNO, offsetof(struct globaldata, gd_cpuno)); +ASSYM(GD_IDLEPCBPHYS, offsetof(struct globaldata, gd_idlepcbphys)); +ASSYM(GD_ASTPENDING, offsetof(struct globaldata, gd_astpending)); + +ASSYM(MTX_LOCK, offsetof(struct mtx, mtx_lock)); +ASSYM(MTX_RECURSE, offsetof(struct mtx, mtx_recurse)); +ASSYM(MTX_SAVEIPL, offsetof(struct mtx, mtx_saveipl)); +ASSYM(MTX_UNOWNED, MTX_UNOWNED); + ASSYM(P_ADDR, offsetof(struct proc, p_addr)); ASSYM(P_MD_FLAGS, offsetof(struct proc, p_md.md_flags)); ASSYM(P_MD_PCBPADDR, offsetof(struct proc, p_md.md_pcbpaddr)); @@ -81,6 +99,7 @@ ASSYM(PTESIZE, PTESIZE); ASSYM(U_PCB_ONFAULT, offsetof(struct user, u_pcb.pcb_onfault)); ASSYM(U_PCB_HWPCB_KSP, offsetof(struct user, u_pcb.pcb_hw.apcb_ksp)); ASSYM(U_PCB_CONTEXT, offsetof(struct user, u_pcb.pcb_context)); +ASSYM(U_PCB_SCHEDNEST, offsetof(struct user, u_pcb.pcb_schednest)); ASSYM(PCB_HW, offsetof(struct pcb, pcb_hw)); diff --git a/sys/powerpc/powerpc/vm_machdep.c b/sys/powerpc/powerpc/vm_machdep.c index 8baea02..3831d67 100644 --- a/sys/powerpc/powerpc/vm_machdep.c +++ b/sys/powerpc/powerpc/vm_machdep.c @@ -84,6 +84,7 @@ #include <machine/fpu.h> #include <machine/md_var.h> #include <machine/prom.h> +#include <machine/mutex.h> #include <vm/vm.h> #include <vm/vm_param.h> @@ -246,8 +247,10 @@ cpu_exit(p) alpha_fpstate_drop(p); (void) splhigh(); + mtx_enter(&sched_lock, MTX_SPIN); + mtx_exit(&Giant, MTX_DEF); cnt.v_swtch++; - cpu_switch(p); + cpu_switch(); panic("cpu_exit"); } @@ -358,7 +361,7 @@ vunmapbuf(bp) } /* - * Force reset the processor by invalidating the entire address space! + * Reset back to firmware. */ void cpu_reset() @@ -416,7 +419,7 @@ vm_page_zero_idle() return(0); #ifdef SMP - if (try_mplock()) { + if (KLOCK_ENTER(M_TRY)) { #endif s = splvm(); m = vm_page_list_find(PQ_FREE, free_rover, FALSE); @@ -447,7 +450,7 @@ vm_page_zero_idle() free_rover = (free_rover + PQ_PRIME2) & PQ_L2_MASK; splx(s); #ifdef SMP - rel_mplock(); + KLOCK_EXIT; #endif return (1); #ifdef SMP |