- Make the mutex code almost completely machine independent. This greatly

  reducues the maintenance load for the mutex code.  The only MD portions
  of the mutex code are in machine/mutex.h now, which include the assembly
  macros for handling mutexes as well as optionally overriding the mutex
  micro-operations.  For example, we use optimized micro-ops on the x86
  platform #ifndef I386_CPU.
- Change the behavior of the SMP_DEBUG kernel option.  In the new code,
  mtx_assert() only depends on INVARIANTS, allowing other kernel developers
  to have working mutex assertiions without having to include all of the
  mutex debugging code.  The SMP_DEBUG kernel option has been renamed to
  MUTEX_DEBUG and now just controls extra mutex debugging code.
- Abolish the ugly mtx_f hack.  Instead, we dynamically allocate
  seperate mtx_debug structures on the fly in mtx_init, except for mutexes
  that are initiated very early in the boot process.   These mutexes
  are declared using a special MUTEX_DECLARE() macro, and use a new
  flag MTX_COLD when calling mtx_init.  This is still somewhat hackish,
  but it is less evil than the mtx_f filler struct, and the mtx struct is
  now the same size with and without mutex debugging code.
- Add some micro-micro-operation macros for doing the actual atomic
  operations on the mutex mtx_lock field to make it easier for other archs
  to override/optimize mutex ops if needed.  These new tiny ops also clean
  up the code in some places by replacing long atomic operation function
  calls that spanned 2-3 lines with a short 1-line macro call.
- Don't call mi_switch() from mtx_enter_hard() when we block while trying
  to obtain a sleep mutex.  Calling mi_switch() would bogusly release
  Giant before switching to the next process.  Instead, inline most of the
  code from mi_switch() in the mtx_enter_hard() function.  Note that when
  we finally kill Giant we can back this out and go back to calling
  mi_switch().

1 files changed, 12 insertions, 468 deletions

diff --git a/sys/alpha/include/mutex.h b/sys/alpha/include/mutex.h
index 12b12a8..5ccc28d 100644
--- a/sys/alpha/include/mutex.h
+++ b/sys/alpha/include/mutex.h
@@ -33,293 +33,39 @@
 #define _MACHINE_MUTEX_H_
 
 #ifndef LOCORE
-#include <sys/queue.h>
 
 #ifdef _KERNEL
-#include <sys/ktr.h>
-#include <sys/proc.h>	/* Needed for curproc. */
-#include <machine/atomic.h>
-#include <machine/cpufunc.h>
-#include <machine/globaldata.h>
-#include <machine/globals.h>
-
-/*
- * If kern_mutex.c is being built, compile non-inlined versions of various
- * functions so that kernel modules can use them.
- */
-#ifndef _KERN_MUTEX_C_
-#define _MTX_INLINE	static __inline
-#else
-#define _MTX_INLINE
-#endif
-
-/*
- * Mutex flags
- *
- * Types
- */
-#define	MTX_DEF		0x1		/* Default (spin/sleep) */
-#define MTX_SPIN	0x2		/* Spin only lock */
-
-/* Options */
-#define	MTX_RLIKELY	0x4		/* (opt) Recursion likely */
-#define	MTX_NORECURSE	0x8		/* No recursion possible */
-#define	MTX_NOSPIN	0x10		/* Don't spin before sleeping */
-#define	MTX_NOSWITCH	0x20		/* Do not switch on release */
-#define	MTX_FIRST	0x40		/* First spin lock holder */
-#define MTX_TOPHALF	0x80		/* Interrupts not disabled on spin */
-
-/* options that should be passed on to mtx_enter_hard, mtx_exit_hard */
-#define	MTX_HARDOPTS	(MTX_DEF | MTX_SPIN | MTX_FIRST | MTX_TOPHALF | MTX_NOSWITCH)
-
-/* Flags/value used in mtx_lock */
-#define	MTX_RECURSE	0x01		/* (non-spin) lock held recursively */
-#define	MTX_CONTESTED	0x02		/* (non-spin) lock contested */
-#define	MTX_FLAGMASK	~(MTX_RECURSE | MTX_CONTESTED)
-#define MTX_UNOWNED	0x8		/* Cookie for free mutex */
-
-#endif	/* _KERNEL */
-
-/*
- * Sleep/spin mutex
- */
-struct mtx {
-	volatile u_int64_t mtx_lock;	/* lock owner/gate/flags */
-	volatile u_int32_t mtx_recurse;	/* number of recursive holds */
-	u_int32_t	mtx_saveipl;	/* saved ipl (for spin locks) */
-	char		*mtx_description;
-	TAILQ_HEAD(, proc) mtx_blocked;
-	LIST_ENTRY(mtx) mtx_contested;
-	struct mtx	*mtx_next;	/* all locks in system */
-	struct mtx	*mtx_prev;
-#ifdef SMP_DEBUG
-	/* If you add anything here, adjust the mtxf_t definition below */
-	struct witness	*mtx_witness;
-	LIST_ENTRY(mtx)	mtx_held;
-	const char	*mtx_file;
-	int		 mtx_line;
-#endif /* SMP_DEBUG */
-};
-
-/*
- * Filler for structs which need to remain the same size
- * whether or not SMP_DEBUG is turned on.
- */
-typedef struct mtxf {
-#ifdef SMP_DEBUG
-	char	mtxf_data[0];
-#else
-	char	mtxf_data[4*sizeof(void *) + sizeof(int)];
-#endif
-} mtxf_t;
-
-#define mp_fixme(string)
-
-#ifdef _KERNEL
-/* Misc */
-#define CURTHD	((u_int64_t)CURPROC)	/* Current thread ID */
-
-/* Prototypes */
-void	mtx_init(struct mtx *m, char *description, int flag);
-void	mtx_enter_hard(struct mtx *, int type, int ipl);
-void	mtx_exit_hard(struct mtx *, int type);
-void	mtx_destroy(struct mtx *m);
-
-/*
- * Wrap the following functions with cpp macros so that filenames and line
- * numbers are embedded in the code correctly.
- */
-#if (defined(KLD_MODULE) || defined(_KERN_MUTEX_C_))
-void	_mtx_enter(struct mtx *mtxp, int type, const char *file, int line);
-int	_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line);
-void	_mtx_exit(struct mtx *mtxp, int type, const char *file, int line);
-#endif
-
-#define	mtx_enter(mtxp, type)						\
-	_mtx_enter((mtxp), (type), __FILE__, __LINE__)
-
-#define	mtx_try_enter(mtxp, type)					\
-	_mtx_try_enter((mtxp), (type), __FILE__, __LINE__)
-
-#define	mtx_exit(mtxp, type)						\
-	_mtx_exit((mtxp), (type), __FILE__, __LINE__)
-
-/* Global locks */
-extern struct mtx sched_lock;
-extern struct mtx Giant;
-
-/*
- * Used to replace return with an exit Giant and return.
- */
-
-#define EGAR(a)				\
-do {					\
-	mtx_exit(&Giant, MTX_DEF);	\
-	return (a);			\
-} while (0)
-
-#define VEGAR				\
-do {					\
-	mtx_exit(&Giant, MTX_DEF);	\
-	return;				\
-} while (0)
-
-#define DROP_GIANT()						\
-do {								\
-	int _giantcnt;						\
-	WITNESS_SAVE_DECL(Giant);				\
-								\
-	WITNESS_SAVE(&Giant, Giant);				\
-	for (_giantcnt = 0; mtx_owned(&Giant); _giantcnt++)	\
-		mtx_exit(&Giant, MTX_DEF)
-
-#define PICKUP_GIANT()						\
-	mtx_assert(&Giant, MA_NOTOWNED);			\
-	while (_giantcnt--)					\
-		mtx_enter(&Giant, MTX_DEF);			\
-	WITNESS_RESTORE(&Giant, Giant);				\
-} while (0)
-
-#define PARTIAL_PICKUP_GIANT()					\
-	mtx_assert(&Giant, MA_NOTOWNED);			\
-	while (_giantcnt--)					\
-		mtx_enter(&Giant, MTX_DEF);			\
-	WITNESS_RESTORE(&Giant, Giant)
-
 
 /*
  * Debugging
  */
-#ifndef SMP_DEBUG
-#define mtx_assert(m, what)
-#else	/* SMP_DEBUG */
+#ifdef MUTEX_DEBUG
 
-#define MA_OWNED	1
-#define MA_NOTOWNED	2
-#define mtx_assert(m, what) {						\
-	switch ((what)) {						\
-	case MA_OWNED:							\
-		ASS(mtx_owned((m)));					\
-		break;							\
-	case MA_NOTOWNED:						\
-		ASS(!mtx_owned((m)));					\
-		break;							\
-	default:							\
-		panic("unknown mtx_assert at %s:%d", __FILE__, __LINE__); \
-	}								\
-}
-
-#ifdef INVARIANTS
-#define	ASS(ex) MPASS(ex)
-#define MPASS(ex) if (!(ex)) panic("Assertion %s failed at %s:%d", \
-                #ex, __FILE__, __LINE__)
-#define MPASS2(ex, what) if (!(ex)) panic("Assertion %s failed at %s:%d", \
-                what, __FILE__, __LINE__)
-
-#ifdef MTX_STRS
-char STR_IEN[] = "fl & 0x200";
-char STR_IDIS[] = "!(fl & 0x200)";
-#else	/* MTX_STRS */
+#ifdef _KERN_MUTEX_C_
+char STR_IEN[] = "ps & IPL != IPL_HIGH";
+char STR_IDIS[] = "ps & IPL == IPL_HIGH";
+char STR_SIEN[] = "mpp->mtx_saveintr != IPL_HIGH";
+#else	/* _KERN_MUTEX_C_ */
 extern char STR_IEN[];
 extern char STR_IDIS[];
-#endif	/* MTX_STRS */
+extern char STR_SIEN[];
+#endif	/* _KERN_MUTEX_C_ */
+
+#endif	/* MUTEX_DEBUG */
+
 #define	ASS_IEN		MPASS2((alpha_pal_rdps() & ALPHA_PSL_IPL_MASK)	\
 			       == ALPHA_PSL_IPL_HIGH, STR_IEN)
 #define	ASS_IDIS	MPASS2((alpha_pal_rdps() & ALPHA_PSL_IPL_MASK)	\
 			       != ALPHA_PSL_IPL_HIGH, STR_IDIS)
-#endif	/* INVARIANTS */
-
-#endif	/* SMP_DEBUG */
-
-#if !defined(SMP_DEBUG) || !defined(INVARIANTS)
-#define ASS(ex)
-#define	MPASS(ex)
-#define	MPASS2(ex, where)
-#define	ASS_IEN
-#define ASS_IDIS
-#endif	/* !defined(SMP_DEBUG) || !defined(INVARIANTS) */
-
-#ifdef	WITNESS
-#ifndef	SMP_DEBUG
-#error	WITNESS requires SMP_DEBUG
-#endif	/* SMP_DEBUG */
-#define WITNESS_ENTER(m, t, f, l)					\
-	if ((m)->mtx_witness != NULL)					\
-		witness_enter((m), (t), (f), (l))
-#define WITNESS_EXIT(m, t, f, l)					\
-	if ((m)->mtx_witness != NULL)					\
-		witness_exit((m), (t), (f), (l))
-
-#define	WITNESS_SLEEP(check, m) witness_sleep(check, (m), __FILE__, __LINE__)
-#define	WITNESS_SAVE_DECL(n)						\
-	const char * __CONCAT(n, __wf);					\
-	int __CONCAT(n, __wl)
-
-#define	WITNESS_SAVE(m, n) 						\
-do {									\
-	if ((m)->mtx_witness != NULL) 					\
-	    witness_save(m, &__CONCAT(n, __wf), &__CONCAT(n, __wl));	\
-} while (0)
-
-#define	WITNESS_RESTORE(m, n) 						\
-do {									\
-	if ((m)->mtx_witness != NULL)					\
-	    witness_restore(m, __CONCAT(n, __wf), __CONCAT(n, __wl));	\
-} while (0)
-
-void	witness_init(struct mtx *, int flag);
-void	witness_destroy(struct mtx *);
-void	witness_enter(struct mtx *, int, const char *, int);
-void	witness_try_enter(struct mtx *, int, const char *, int);
-void	witness_exit(struct mtx *, int, const char *, int);
-void	witness_display(void(*)(const char *fmt, ...));
-void	witness_list(struct proc *);
-int	witness_sleep(int, struct mtx *, const char *, int);
-void	witness_save(struct mtx *, const char **, int *);
-void	witness_restore(struct mtx *, const char *, int);
-#else	/* WITNESS */
-#define WITNESS_ENTER(m, t, f, l)
-#define WITNESS_EXIT(m, t, f, l)
-#define	WITNESS_SLEEP(check, m)
-#define	WITNESS_SAVE_DECL(n)
-#define	WITNESS_SAVE(m, n)
-#define	WITNESS_RESTORE(m, n)
-
-/*
- * flag++ is slezoid way of shutting up unused parameter warning
- * in mtx_init()
- */
-#define witness_init(m, flag) flag++
-#define witness_destroy(m)
-#define witness_enter(m, t, f, l)
-#define witness_try_enter(m, t, f, l)
-#define witness_exit(m, t, f, l)
-#endif	/* WITNESS */
+#define ASS_SIEN(mpp)	MPASS2((mpp)->saveintr != ALPHA_PSL_IPL_HIGH, STR_SIEN)
 
 /*
  * Assembly macros (for internal use only)
  *--------------------------------------------------------------------------
  */
 
-/*
- * Get a sleep lock, deal with recursion inline
- */
-
 #define	_V(x)	__STRING(x)
 
-#define	_getlock_sleep(mp, tid, type) do {				     \
-	if (atomic_cmpset_64(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0) {    \
-		if (((mp)->mtx_lock & MTX_FLAGMASK) != (tid))		     \
-			mtx_enter_hard(mp, (type) & MTX_HARDOPTS, 0);	     \
-		else {							     \
-			atomic_set_64(&(mp)->mtx_lock, MTX_RECURSE);	     \
-			(mp)->mtx_recurse++;				     \
-		}							     \
-	} else {							     \
-		alpha_mb();						     \
-	}								     \
-} while (0)
-
 /*
  * Get a spin lock, handle recusion inline (as the less common case)
  */
@@ -334,208 +80,6 @@ void	witness_restore(struct mtx *, const char *, int);
 	}								\
 } while (0)
 
-/*
- * Get a lock without any recursion handling. Calls the hard enter
- * function if we can't get it inline.
- */
-
-#define	_getlock_norecurse(mp, tid, type) do {				\
-	if (atomic_cmpset_64(&(mp)->mtx_lock, MTX_UNOWNED, (tid)) == 0)	\
-		mtx_enter_hard((mp), (type) & MTX_HARDOPTS, 0);		\
-	else								\
-		alpha_mb();						\
-} while (0)
-
-/*
- * Release a sleep lock assuming we haven't recursed on it, recursion is
- * handled in the hard function.
- */
-
-#define	_exitlock_norecurse(mp, tid, type) do {				\
-	alpha_mb();							\
-	if (atomic_cmpset_64(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0)	\
-		mtx_exit_hard((mp), (type) & MTX_HARDOPTS);		\
-} while (0)
-
-/*
- * Release a sleep lock when its likely we recursed (the code to
- * deal with simple recursion is inline).
- */
-
-#define	_exitlock(mp, tid, type) do {					\
-	alpha_mb();							\
-	if (atomic_cmpset_64(&(mp)->mtx_lock, (tid), MTX_UNOWNED) == 0) {\
-		if (((mp)->mtx_lock & MTX_RECURSE) &&			\
-		    (--(mp)->mtx_recurse == 0))				\
-			atomic_clear_64(&(mp)->mtx_lock, MTX_RECURSE);	\
-		else							\
-			mtx_exit_hard((mp), (type) & MTX_HARDOPTS);	\
-	}								\
-} while (0)
-
-/*
- * Release a spin lock (with possible recursion)
- */
-
-#define	_exitlock_spin(mp) do {						\
-	alpha_mb();							\
-	if ((mp)->mtx_recurse == 0) {					\
-		int _ipl = (mp)->mtx_saveipl;				\
-		atomic_cmpset_64(&(mp)->mtx_lock, (mp)->mtx_lock,	\
-		    MTX_UNOWNED);					\
-		alpha_pal_swpipl(_ipl);					\
-	} else {							\
-		(mp)->mtx_recurse--;					\
-	}								\
-} while (0)
-
-/*
- * Externally visible mutex functions
- *------------------------------------------------------------------------
- */
-
-/*
- * Return non-zero if a mutex is already owned by the current thread
- */
-#define	mtx_owned(m)    (((m)->mtx_lock & MTX_FLAGMASK) == CURTHD)
-
-/* Common strings */
-#ifdef MTX_STRS
-char	STR_mtx_enter_fmt[] = "GOT %s [%p] at %s:%d r=%d";
-char	STR_mtx_bad_type[] = "((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0";
-char	STR_mtx_exit_fmt[] = "REL %s [%p] at %s:%d r=%d";
-char	STR_mtx_owned[] = "mtx_owned(mpp)";
-char	STR_mtx_recurse[] = "mpp->mtx_recurse == 0";
-char	STR_mtx_try_enter_fmt[] = "TRY_ENTER %s [%p] at %s:%d result=%d";
-#else	/* MTX_STRS */
-extern	char STR_mtx_enter_fmt[];
-extern	char STR_mtx_bad_type[];
-extern	char STR_mtx_exit_fmt[];
-extern	char STR_mtx_owned[];
-extern	char STR_mtx_recurse[];
-extern	char STR_mtx_try_enter_fmt[];
-#endif	/* MTX_STRS */
-
-#ifndef KLD_MODULE
-/*
- * Get lock 'm', the macro handles the easy (and most common cases) and
- * leaves the slow stuff to the mtx_enter_hard() function.
- *
- * Note: since type is usually a constant much of this code is optimized out
- */
-_MTX_INLINE void
-_mtx_enter(struct mtx *mtxp, int type, const char *file, int line)
-{
-	struct mtx	*mpp = mtxp;
-
-	/* bits only valid on mtx_exit() */
-	MPASS2(((type) & (MTX_NORECURSE | MTX_NOSWITCH)) == 0,
-	    STR_mtx_bad_type);
-
-	if ((type) & MTX_SPIN) {
-		/*
-		 * Easy cases of spin locks:
-		 *
-		 * 1) We already own the lock and will simply recurse on it (if
-		 *    RLIKELY)
-		 *
-		 * 2) The lock is free, we just get it
-		 */
-		if ((type) & MTX_RLIKELY) {
-			/*
-			 * Check for recursion, if we already have this lock we
-			 * just bump the recursion count.
-			 */
-			if (mpp->mtx_lock == CURTHD) {
-				mpp->mtx_recurse++;
-				goto done;
-			}
-		}
-
-		if (((type) & MTX_TOPHALF) == 0) {
-			/*
-			 * If an interrupt thread uses this we must block
-			 * interrupts here.
-			 */
-			_getlock_spin_block(mpp, CURTHD, (type) & MTX_HARDOPTS);
-		} else
-			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
-	} else {
-		/* Sleep locks */
-		if ((type) & MTX_RLIKELY)
-			_getlock_sleep(mpp, CURTHD, (type) & MTX_HARDOPTS);
-		else
-			_getlock_norecurse(mpp, CURTHD, (type) & MTX_HARDOPTS);
-	}
-	done:
-	WITNESS_ENTER(mpp, type, file, line);
-	CTR5(KTR_LOCK, STR_mtx_enter_fmt,
-	    mpp->mtx_description, mpp, file, line,
-	    mpp->mtx_recurse);
-}
-
-/*
- * Attempt to get MTX_DEF lock, return non-zero if lock acquired
- *
- * XXX DOES NOT HANDLE RECURSION
- */
-_MTX_INLINE int
-_mtx_try_enter(struct mtx *mtxp, int type, const char *file, int line)
-{
-	struct mtx	*const mpp = mtxp;
-	int	rval;
-
-	rval = atomic_cmpset_64(&mpp->mtx_lock, MTX_UNOWNED, CURTHD);
-#ifdef SMP_DEBUG
-	if (rval && mpp->mtx_witness != NULL) {
-		ASS(mpp->mtx_recurse == 0);
-		witness_try_enter(mpp, type, file, line);
-	}
-#endif
-	CTR5(KTR_LOCK, STR_mtx_try_enter_fmt,
-	    mpp->mtx_description, mpp, file, line, rval);
-
-	return rval;
-}
-
-/*
- * Release lock m
- */
-_MTX_INLINE void
-_mtx_exit(struct mtx *mtxp, int type, const char *file, int line)
-{
-	struct mtx	*const mpp = mtxp;
-
-	MPASS2(mtx_owned(mpp), STR_mtx_owned);
-	WITNESS_EXIT(mpp, type, file, line);
-	CTR5(KTR_LOCK, STR_mtx_exit_fmt,
-	    mpp->mtx_description, mpp, file, line,
-	    mpp->mtx_recurse);
-	if ((type) & MTX_SPIN) {
-		if ((type) & MTX_NORECURSE) {
-			MPASS2(mpp->mtx_recurse == 0, STR_mtx_recurse);
-			atomic_cmpset_64(&mpp->mtx_lock, mpp->mtx_lock,
-			    MTX_UNOWNED);
-			if (((type) & MTX_TOPHALF) == 0)
-				alpha_pal_swpipl(mpp->mtx_saveipl);
-		} else
-			if ((type) & MTX_TOPHALF) {
-				_exitlock_norecurse(mpp, CURTHD,
-				    (type) & MTX_HARDOPTS);
-			} else
-				_exitlock_spin(mpp);
-	} else {
-		/* Handle sleep locks */
-		if ((type) & MTX_RLIKELY)
-			_exitlock(mpp, CURTHD, (type) & MTX_HARDOPTS);
-		else {
-			_exitlock_norecurse(mpp, CURTHD,
-			    (type) & MTX_HARDOPTS);
-		}
-	}
-}
-
-#endif	/* KLD_MODULE */
 #endif	/* _KERNEL */
 
 #else	/* !LOCORE */