Fix mtx_legal2block. The only time that it is bad to block on a mutex is

if we hold a spin mutex, since we can trivially get into deadlocks if we
start switching out of processes that hold spinlocks.  Checking to see if
interrupts were disabled was a sort of cheap way of doing this since most
of the time interrupts were only disabled when holding a spin lock.  At
least on the i386.  To fix this properly, use a per-process counter
p_spinlocks that counts the number of spin locks currently held, and
instead of checking to see if interrupts are disabled in the witness code,
check to see if we hold any spin locks.  Since child processes always
start up with the sched lock magically held in fork_exit(), we initialize
p_spinlocks to 1 for child processes.  Note that proc0 doesn't go through
fork_exit(), so it starts with no spin locks held.

Consulting from:	cp

10 files changed, 29 insertions, 13 deletions

diff --git a/sys/alpha/include/mutex.h b/sys/alpha/include/mutex.h
index 9b04128..a6e295d 100644
--- a/sys/alpha/include/mutex.h
+++ b/sys/alpha/include/mutex.h
@@ -36,8 +36,6 @@
 
 #ifdef _KERNEL
 
-#define	mtx_legal2block()						\
-	((alpha_pal_rdps() & ALPHA_PSL_IPL_MASK) == ALPHA_PSL_IPL_0)
 #define	mtx_intr_enable(mutex)	(mutex)->mtx_saveintr = ALPHA_PSL_IPL_0
 
 /*
diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h
index a5f88de..4184cde 100644
--- a/sys/amd64/include/mutex.h
+++ b/sys/amd64/include/mutex.h
@@ -40,7 +40,6 @@
 /* Global locks */
 extern struct mtx	clock_lock;
 
-#define	mtx_legal2block()	(read_eflags() & PSL_I)
 #define	mtx_intr_enable(mutex)	(mutex)->mtx_saveintr |= PSL_I
 
 /*
diff --git a/sys/i386/include/mutex.h b/sys/i386/include/mutex.h
index a5f88de..4184cde 100644
--- a/sys/i386/include/mutex.h
+++ b/sys/i386/include/mutex.h
@@ -40,7 +40,6 @@
 /* Global locks */
 extern struct mtx	clock_lock;
 
-#define	mtx_legal2block()	(read_eflags() & PSL_I)
 #define	mtx_intr_enable(mutex)	(mutex)->mtx_saveintr |= PSL_I
 
 /*
diff --git a/sys/ia64/include/mutex.h b/sys/ia64/include/mutex.h
index 809a56e..f325f2b 100644
--- a/sys/ia64/include/mutex.h
+++ b/sys/ia64/include/mutex.h
@@ -38,7 +38,6 @@
 
 #ifdef _KERNEL
 
-#define	mtx_legal2block()	(ia64_get_psr() & IA64_PSR_I)
 #define	mtx_intr_enable(mutex)	(mutex)->mtx_saveintr |= IA64_PSR_I
 
 #endif	/* _KERNEL */
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index c6776de..2f9b3aa 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -406,6 +406,10 @@ again:
 	if (p1->p_sflag & PS_PROFIL)
 		startprofclock(p2);
 	mtx_unlock_spin(&sched_lock);
+	/*
+	 * We start off holding one spinlock after fork: sched_lock.
+	 */
+	p2->p_spinlocks = 1;
 	PROC_UNLOCK(p2);
 	MALLOC(p2->p_cred, struct pcred *, sizeof(struct pcred),
 	    M_SUBPROC, M_WAITOK);
diff --git a/sys/kern/kern_mutex.c b/sys/kern/kern_mutex.c
index 1ca5cca..6540b56 100644
--- a/sys/kern/kern_mutex.c
+++ b/sys/kern/kern_mutex.c
@@ -1094,6 +1094,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 		}
 		PCPU_SET(witness_spin_check, i | w->w_level);
 		mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
+		p->p_spinlocks++;
+		MPASS(p->p_spinlocks > 0);
 		w->w_file = file;
 		w->w_line = line;
 		m->mtx_line = line;
@@ -1116,7 +1118,7 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 	if (cold)
 		goto out;
 
-	if (!mtx_legal2block())
+	if (p->p_spinlocks != 0)
 		panic("blockable mtx_lock() of %s when not legal @ %s:%d",
 			    m->mtx_description, file, line);
 	/*
@@ -1263,10 +1265,12 @@ void
 witness_exit(struct mtx *m, int flags, const char *file, int line)
 {
 	struct witness *w;
+	struct proc *p;
 
 	if (witness_cold || m->mtx_witness == NULL || panicstr)
 		return;
 	w = m->mtx_witness;
+	p = curproc;
 
 	if (flags & MTX_SPIN) {
 		if ((m->mtx_flags & MTX_SPIN) == 0)
@@ -1283,6 +1287,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 		PCPU_SET(witness_spin_check,
 		    PCPU_GET(witness_spin_check) & ~w->w_level);
 		mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
+		MPASS(p->p_spinlocks > 0);
+		p->p_spinlocks--;
 		return;
 	}
 	if ((m->mtx_flags & MTX_SPIN) != 0)
@@ -1297,7 +1303,7 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 		return;
 	}
 
-	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
+	if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold)
 		panic("switchable mtx_unlock() of %s when not legal @ %s:%d",
 			    m->mtx_description, file, line);
 	LIST_REMOVE(m, mtx_held);
diff --git a/sys/kern/subr_turnstile.c b/sys/kern/subr_turnstile.c
index 1ca5cca..6540b56 100644
--- a/sys/kern/subr_turnstile.c
+++ b/sys/kern/subr_turnstile.c
@@ -1094,6 +1094,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 		}
 		PCPU_SET(witness_spin_check, i | w->w_level);
 		mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
+		p->p_spinlocks++;
+		MPASS(p->p_spinlocks > 0);
 		w->w_file = file;
 		w->w_line = line;
 		m->mtx_line = line;
@@ -1116,7 +1118,7 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 	if (cold)
 		goto out;
 
-	if (!mtx_legal2block())
+	if (p->p_spinlocks != 0)
 		panic("blockable mtx_lock() of %s when not legal @ %s:%d",
 			    m->mtx_description, file, line);
 	/*
@@ -1263,10 +1265,12 @@ void
 witness_exit(struct mtx *m, int flags, const char *file, int line)
 {
 	struct witness *w;
+	struct proc *p;
 
 	if (witness_cold || m->mtx_witness == NULL || panicstr)
 		return;
 	w = m->mtx_witness;
+	p = curproc;
 
 	if (flags & MTX_SPIN) {
 		if ((m->mtx_flags & MTX_SPIN) == 0)
@@ -1283,6 +1287,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 		PCPU_SET(witness_spin_check,
 		    PCPU_GET(witness_spin_check) & ~w->w_level);
 		mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
+		MPASS(p->p_spinlocks > 0);
+		p->p_spinlocks--;
 		return;
 	}
 	if ((m->mtx_flags & MTX_SPIN) != 0)
@@ -1297,7 +1303,7 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 		return;
 	}
 
-	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
+	if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold)
 		panic("switchable mtx_unlock() of %s when not legal @ %s:%d",
 			    m->mtx_description, file, line);
 	LIST_REMOVE(m, mtx_held);
diff --git a/sys/kern/subr_witness.c b/sys/kern/subr_witness.c
index 1ca5cca..6540b56 100644
--- a/sys/kern/subr_witness.c
+++ b/sys/kern/subr_witness.c
@@ -1094,6 +1094,8 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 		}
 		PCPU_SET(witness_spin_check, i | w->w_level);
 		mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
+		p->p_spinlocks++;
+		MPASS(p->p_spinlocks > 0);
 		w->w_file = file;
 		w->w_line = line;
 		m->mtx_line = line;
@@ -1116,7 +1118,7 @@ witness_enter(struct mtx *m, int flags, const char *file, int line)
 	if (cold)
 		goto out;
 
-	if (!mtx_legal2block())
+	if (p->p_spinlocks != 0)
 		panic("blockable mtx_lock() of %s when not legal @ %s:%d",
 			    m->mtx_description, file, line);
 	/*
@@ -1263,10 +1265,12 @@ void
 witness_exit(struct mtx *m, int flags, const char *file, int line)
 {
 	struct witness *w;
+	struct proc *p;
 
 	if (witness_cold || m->mtx_witness == NULL || panicstr)
 		return;
 	w = m->mtx_witness;
+	p = curproc;
 
 	if (flags & MTX_SPIN) {
 		if ((m->mtx_flags & MTX_SPIN) == 0)
@@ -1283,6 +1287,8 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 		PCPU_SET(witness_spin_check,
 		    PCPU_GET(witness_spin_check) & ~w->w_level);
 		mtx_unlock_spin_flags(&w_mtx, MTX_QUIET);
+		MPASS(p->p_spinlocks > 0);
+		p->p_spinlocks--;
 		return;
 	}
 	if ((m->mtx_flags & MTX_SPIN) != 0)
@@ -1297,7 +1303,7 @@ witness_exit(struct mtx *m, int flags, const char *file, int line)
 		return;
 	}
 
-	if ((flags & MTX_NOSWITCH) == 0 && !mtx_legal2block() && !cold)
+	if ((flags & MTX_NOSWITCH) == 0 && p->p_spinlocks != 0 && !cold)
 		panic("switchable mtx_unlock() of %s when not legal @ %s:%d",
 			    m->mtx_description, file, line);
 	LIST_REMOVE(m, mtx_held);
diff --git a/sys/powerpc/include/mutex.h b/sys/powerpc/include/mutex.h
index 9b04128..a6e295d 100644
--- a/sys/powerpc/include/mutex.h
+++ b/sys/powerpc/include/mutex.h
@@ -36,8 +36,6 @@
 
 #ifdef _KERNEL
 
-#define	mtx_legal2block()						\
-	((alpha_pal_rdps() & ALPHA_PSL_IPL_MASK) == ALPHA_PSL_IPL_0)
 #define	mtx_intr_enable(mutex)	(mutex)->mtx_saveintr = ALPHA_PSL_IPL_0
 
 /*
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index e96dffa..b77d49e 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -213,6 +213,7 @@ struct	proc {
 	struct	vnode *p_textvp;	/* (b) Vnode of executable. */
 
 	struct	mtx p_mtx;		/* (k) Lock for this struct. */
+	u_int	p_spinlocks;		/* (k) Count of held spin locks. */
 	char	p_lock;		/* (c) Process lock (prevent swap) count. */
 	u_char	p_oncpu;		/* (j) Which cpu we are on. */
 	u_char	p_lastcpu;		/* (j) Last cpu we were on. */