Trapsignal() and postsig() call kern_sigprocmask() with both process

lock and curproc->p_sigacts->ps_mtx. Reschedule_signals may need to have
ps_mtx locked to decide and wakeup a thread, causing recursion on the
mutex.

Inform kern_sigprocmask() and reschedule_signals() about lock state
of the ps_mtx by new flag SIGPROCMASK_PS_LOCKED to avoid recursion.

Reported and tested by:	keramida
MFC after:	1 month

2 files changed, 17 insertions, 20 deletions

diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 7f5cfa3..e174df1 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -220,7 +220,7 @@ static int sigproptbl[NSIG] = {
         SA_KILL|SA_PROC,		/* SIGUSR2 */
 };
 
-static void reschedule_signals(struct proc *p, sigset_t block);
+static void reschedule_signals(struct proc *p, sigset_t block, int flags);
 
 static void
 sigqueue_start(void)
@@ -1024,7 +1024,7 @@ kern_sigprocmask(struct thread *td, int how, sigset_t *set, sigset_t *oset,
 	 * possibly waking it up.
 	 */
 	if (p->p_numthreads != 1)
-		reschedule_signals(p, new_block);
+		reschedule_signals(p, new_block, flags);
 
 	if (!(flags & SIGPROCMASK_PROC_LOCKED))
 		PROC_UNLOCK(p);
@@ -1859,13 +1859,11 @@ trapsignal(struct thread *td, ksiginfo_t *ksi)
 #endif
 		(*p->p_sysent->sv_sendsig)(ps->ps_sigact[_SIG_IDX(sig)], 
 				ksi, &td->td_sigmask);
-		SIGSETOR(td->td_sigmask, ps->ps_catchmask[_SIG_IDX(sig)]);
-		if (!SIGISMEMBER(ps->ps_signodefer, sig)) {
-			SIGEMPTYSET(mask);
+		mask = ps->ps_catchmask[_SIG_IDX(sig)];
+		if (!SIGISMEMBER(ps->ps_signodefer, sig))
 			SIGADDSET(mask, sig);
-			kern_sigprocmask(td, SIG_BLOCK, &mask, NULL,
-			    SIGPROCMASK_PROC_LOCKED);
-		}
+		kern_sigprocmask(td, SIG_BLOCK, &mask, NULL,
+		    SIGPROCMASK_PROC_LOCKED | SIGPROCMASK_PS_LOCKED);
 		if (SIGISMEMBER(ps->ps_sigreset, sig)) {
 			/*
 			 * See kern_sigaction() for origin of this code.
@@ -2401,7 +2399,7 @@ stopme:
 }
 
 static void
-reschedule_signals(struct proc *p, sigset_t block)
+reschedule_signals(struct proc *p, sigset_t block, int flags)
 {
 	struct sigacts *ps;
 	struct thread *td;
@@ -2419,12 +2417,14 @@ reschedule_signals(struct proc *p, sigset_t block)
 
 		td = sigtd(p, i, 0);
 		signotify(td);
-		mtx_lock(&ps->ps_mtx);
+		if (!(flags & SIGPROCMASK_PS_LOCKED))
+			mtx_lock(&ps->ps_mtx);
 		if (p->p_flag & P_TRACED || SIGISMEMBER(ps->ps_sigcatch, i))
 			tdsigwakeup(td, i, SIG_CATCH,
 			    (SIGISMEMBER(ps->ps_sigintr, i) ? EINTR :
 			     ERESTART));
-		mtx_unlock(&ps->ps_mtx);
+		if (!(flags & SIGPROCMASK_PS_LOCKED))
+			mtx_unlock(&ps->ps_mtx);
 	}
 }
 
@@ -2452,7 +2452,7 @@ tdsigcleanup(struct thread *td)
 	SIGFILLSET(unblocked);
 	SIGSETNAND(unblocked, td->td_sigmask);
 	SIGFILLSET(td->td_sigmask);
-	reschedule_signals(p, unblocked);
+	reschedule_signals(p, unblocked, 0);
 
 }
 
@@ -2734,15 +2734,11 @@ postsig(sig)
 		} else
 			returnmask = td->td_sigmask;
 
-		kern_sigprocmask(td, SIG_BLOCK,
-		    &ps->ps_catchmask[_SIG_IDX(sig)], NULL,
-		    SIGPROCMASK_PROC_LOCKED);
-		if (!SIGISMEMBER(ps->ps_signodefer, sig)) {
-			SIGEMPTYSET(mask);
+		mask = ps->ps_catchmask[_SIG_IDX(sig)];
+		if (!SIGISMEMBER(ps->ps_signodefer, sig))
 			SIGADDSET(mask, sig);
-			kern_sigprocmask(td, SIG_BLOCK, &mask, NULL,
-			    SIGPROCMASK_PROC_LOCKED);
-		}
+		kern_sigprocmask(td, SIG_BLOCK, &mask, NULL,
+		    SIGPROCMASK_PROC_LOCKED | SIGPROCMASK_PS_LOCKED);
 
 		if (SIGISMEMBER(ps->ps_sigreset, sig)) {
 			/*
diff --git a/sys/sys/signalvar.h b/sys/sys/signalvar.h
index b9a54f0..c27a128 100644
--- a/sys/sys/signalvar.h
+++ b/sys/sys/signalvar.h
@@ -319,6 +319,7 @@ extern int kern_logsigexit;	/* Sysctl variable kern.logsigexit */
 /* flags for kern_sigprocmask */
 #define	SIGPROCMASK_OLD		0x0001
 #define	SIGPROCMASK_PROC_LOCKED	0x0002
+#define	SIGPROCMASK_PS_LOCKED	0x0004
 
 /*
  * Machine-independent functions: