Add support for queueing SIGCHLD same as other UNIX systems did.

For each child process whose status has been changed, a SIGCHLD instance
is queued, if the signal is stilling pending, and process changed status
several times, signal information is updated to reflect latest process
status. If wait() returns because the status of a child process is
available, pending SIGCHLD signal associated with the child process is
discarded. Any other pending SIGCHLD signals remain pending.

The signal information is allocated at the same time when proc structure
is allocated, if process signal queue is fully filled or there is a memory
shortage, it can still send the signal to process.

There is a booting time tunable kern.sigqueue.queue_sigchild which
can control the behavior, setting it to zero disables the SIGCHLD queueing
feature, the tunable will be removed if the function is proved that it is
stable enough.

Tested on: i386 (SMP and UP)

6 files changed, 136 insertions, 14 deletions

diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c
index 6107a00..191bb7c 100644
--- a/sys/kern/kern_exit.c
+++ b/sys/kern/kern_exit.c
@@ -173,6 +173,11 @@ retry:
 	}
 
 	p->p_flag |= P_WEXIT;
+
+	PROC_LOCK(p->p_pptr);
+	sigqueue_take(p->p_ksi);
+	PROC_UNLOCK(p->p_pptr);
+
 	PROC_UNLOCK(p);
 
 	/* Are we a task leader? */
@@ -480,8 +485,12 @@ retry:
 
 	if (p->p_pptr == initproc)
 		psignal(p->p_pptr, SIGCHLD);
-	else if (p->p_sigparent != 0)
-		psignal(p->p_pptr, p->p_sigparent);
+	else if (p->p_sigparent != 0) {
+		if (p->p_sigparent == SIGCHLD)
+			childproc_exited(p);
+		else	/* LINUX thread */
+			psignal(p->p_pptr, p->p_sigparent);
+	}
 	PROC_UNLOCK(p->p_pptr);
 
 	/*
@@ -659,6 +668,10 @@ loop:
 				calcru(p, &rusage->ru_utime, &rusage->ru_stime);
 			}
 
+			PROC_LOCK(q);
+			sigqueue_take(p->p_ksi);
+			PROC_UNLOCK(q);
+
 			/*
 			 * If we got the child via a ptrace 'attach',
 			 * we need to give it back to the old parent.
@@ -669,7 +682,7 @@ loop:
 				p->p_oppid = 0;
 				proc_reparent(p, t);
 				PROC_UNLOCK(p);
-				psignal(t, SIGCHLD);
+				tdsignal(t, NULL, SIGCHLD, p->p_ksi);
 				wakeup(t);
 				PROC_UNLOCK(t);
 				sx_xunlock(&proctree_lock);
@@ -751,6 +764,11 @@ loop:
 			if (status)
 				*status = W_STOPCODE(p->p_xstat);
 			PROC_UNLOCK(p);
+
+			PROC_LOCK(q);
+			sigqueue_take(p->p_ksi);
+			PROC_UNLOCK(q);
+
 			return (0);
 		}
 		mtx_unlock_spin(&sched_lock);
@@ -760,6 +778,10 @@ loop:
 			p->p_flag &= ~P_CONTINUED;
 			PROC_UNLOCK(p);
 
+			PROC_LOCK(q);
+			sigqueue_take(p->p_ksi);
+			PROC_UNLOCK(q);
+
 			if (status)
 				*status = SIGCONT;
 			return (0);
diff --git a/sys/kern/kern_proc.c b/sys/kern/kern_proc.c
index 5caef58..43f1c0e 100644
--- a/sys/kern/kern_proc.c
+++ b/sys/kern/kern_proc.c
@@ -165,6 +165,8 @@ proc_dtor(void *mem, int size, void *arg)
 	 */
 	if (((p->p_flag & P_KTHREAD) != 0) && (td->td_altkstack != 0))
 		vm_thread_dispose_altkstack(td);
+	if (p->p_ksi != NULL)
+		KASSERT(! KSI_ONQ(p->p_ksi), ("SIGCHLD queue"));
 }
 
 /*
@@ -204,6 +206,8 @@ proc_fini(void *mem, int size)
 	ksegrp_free(FIRST_KSEGRP_IN_PROC(p));
 	thread_free(FIRST_THREAD_IN_PROC(p));
 	mtx_destroy(&p->p_mtx);
+	if (p->p_ksi != NULL)
+		ksiginfo_free(p->p_ksi);
 #else
 	panic("proc reclaimed");
 #endif
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index e422df4..8efd21d 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -214,10 +214,15 @@ sigqueue_start(void)
 }
 
 ksiginfo_t *
-ksiginfo_alloc(void)
+ksiginfo_alloc(int wait)
 {
+	int flags;
+
+	flags = M_ZERO;
+	if (! wait)
+		flags |= M_NOWAIT;
 	if (ksiginfo_zone != NULL)
-		return ((ksiginfo_t *)uma_zalloc(ksiginfo_zone, M_NOWAIT | M_ZERO));
+		return ((ksiginfo_t *)uma_zalloc(ksiginfo_zone, flags));
 	return (NULL);
 }
 
@@ -291,7 +296,7 @@ sigqueue_take(ksiginfo_t *ksi)
 	struct proc	*p;
 	sigqueue_t	*sq;
 
-	if ((sq = ksi->ksi_sigq) == NULL)
+	if (ksi == NULL || (sq = ksi->ksi_sigq) == NULL)
 		return;
 
 	p = sq->sq_proc;
@@ -331,10 +336,10 @@ sigqueue_add(sigqueue_t *sq, int signo, ksiginfo_t *si)
 	if (__predict_false(ksiginfo_zone == NULL))
 		goto out_set_bit;
 	
-	if (p != NULL && p->p_pendingcnt > max_pending_per_proc) {
+	if (p != NULL && p->p_pendingcnt >= max_pending_per_proc) {
 		signal_overflow++;
 		ret = EAGAIN;
-	} else if ((ksi = ksiginfo_alloc()) == NULL) {
+	} else if ((ksi = ksiginfo_alloc(0)) == NULL) {
 		signal_alloc_fail++;
 		ret = EAGAIN;
 	} else {
@@ -2106,7 +2111,12 @@ do_tdsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi)
 			return (ret);
 		}
 		sigqueue_delete_proc(p, SIGCONT);
-		p->p_flag &= ~P_CONTINUED;
+		if (p->p_flag & P_CONTINUED) {
+			p->p_flag &= ~P_CONTINUED;
+			PROC_LOCK(p->p_pptr);
+			sigqueue_take(p->p_ksi);
+			PROC_UNLOCK(p->p_pptr);
+		}
 	}
 
 	ret = sigqueue_add(sigqueue, sig, ksi);
@@ -2174,7 +2184,10 @@ do_tdsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi)
 			 * Otherwise, process goes back to sleep state.
 			 */
 			p->p_flag &= ~P_STOPPED_SIG;
-			p->p_flag |= P_CONTINUED;
+			if (p->p_numthreads == p->p_suspcount) {
+				p->p_flag |= P_CONTINUED;
+				childproc_continued(p);
+			}
 			if (action == SIG_DFL) {
 				sigqueue_delete(sigqueue, sig);
 			} else if (action == SIG_CATCH) {
@@ -2249,12 +2262,19 @@ do_tdsignal(struct proc *p, struct thread *td, int sig, ksiginfo_t *ksi)
 				    (td0->td_flags & TDF_SINTR) &&
 				    !TD_IS_SUSPENDED(td0)) {
 					thread_suspend_one(td0);
-				} else if (td != td0) {
+				} else {
 					td0->td_flags |= TDF_ASTPENDING;
 				}
 			}
-			thread_stopped(p);
 			if (p->p_numthreads == p->p_suspcount) {
+				/*
+				 * only thread sending signal to another
+				 * process can reach here, if thread is sending
+				 * signal to its process, because thread does
+				 * not suspend itself here, p_numthreads
+				 * should never be equal to p_suspcount.
+				 */
+				thread_stopped(p);
 				mtx_unlock_spin(&sched_lock);
 				sigqueue_delete_proc(p, p->p_xstat);
 			} else
@@ -2646,7 +2666,9 @@ thread_stopped(struct proc *p)
 		mtx_lock(&ps->ps_mtx);
 		if ((ps->ps_flag & PS_NOCLDSTOP) == 0) {
 			mtx_unlock(&ps->ps_mtx);
-			psignal(p->p_pptr, SIGCHLD);
+			childproc_stopped(p,
+				(p->p_flag & P_TRACED) ?
+					CLD_TRAPPED : CLD_STOPPED);
 		} else
 			mtx_unlock(&ps->ps_mtx);
 		PROC_UNLOCK(p->p_pptr);
@@ -2826,6 +2848,61 @@ sigexit(td, sig)
 	/* NOTREACHED */
 }
 
+/*
+ * Send queued SIGCHLD to parent when child process is stopped
+ * or exited.
+ */
+void
+childproc_stopped(struct proc *p, int reason)
+{
+	PROC_LOCK_ASSERT(p, MA_OWNED);
+	PROC_LOCK_ASSERT(p->p_pptr, MA_OWNED);
+
+	if (p->p_ksi != NULL) {
+		p->p_ksi->ksi_signo  = SIGCHLD;
+		p->p_ksi->ksi_code   = reason;
+		p->p_ksi->ksi_status = p->p_xstat;
+		p->p_ksi->ksi_pid    = p->p_pid;
+		p->p_ksi->ksi_uid    = p->p_ucred->cr_ruid;
+		if (KSI_ONQ(p->p_ksi))
+			return;
+	}
+	tdsignal(p->p_pptr, NULL, SIGCHLD, p->p_ksi);
+}
+
+void
+childproc_continued(struct proc *p)
+{
+	PROC_LOCK_ASSERT(p, MA_OWNED);
+	PROC_LOCK_ASSERT(p->p_pptr, MA_NOTOWNED);
+
+	PROC_LOCK(p->p_pptr);
+	if (p->p_ksi != NULL) {
+		p->p_ksi->ksi_signo  = SIGCHLD;
+		p->p_ksi->ksi_code   = CLD_CONTINUED; 
+		p->p_ksi->ksi_status = SIGCONT;
+		p->p_ksi->ksi_pid    = p->p_pid;
+		p->p_ksi->ksi_uid    = p->p_ucred->cr_ruid;
+		if (KSI_ONQ(p->p_ksi))
+			return;
+	}
+	tdsignal(p->p_pptr, NULL, SIGCHLD, p->p_ksi);
+	PROC_UNLOCK(p->p_pptr);
+}
+
+void
+childproc_exited(struct proc *p)
+{
+	int reason;
+
+	reason = CLD_EXITED;
+	if (WCOREDUMP(p->p_xstat))
+		reason = CLD_DUMPED;
+	else if (WIFSIGNALED(p->p_xstat))
+		reason = CLD_KILLED;	
+	childproc_stopped(p, reason);
+}
+
 static char corefilename[MAXPATHLEN] = {"%N.core"};
 SYSCTL_STRING(_kern, OID_AUTO, corefile, CTLFLAG_RW, corefilename,
 	      sizeof(corefilename), "process corefile name format string");
diff --git a/sys/kern/kern_thread.c b/sys/kern/kern_thread.c
index 12be17b..a4ff105 100644
--- a/sys/kern/kern_thread.c
+++ b/sys/kern/kern_thread.c
@@ -78,6 +78,12 @@ TAILQ_HEAD(, ksegrp) zombie_ksegrps = TAILQ_HEAD_INITIALIZER(zombie_ksegrps);
 struct mtx kse_zombie_lock;
 MTX_SYSINIT(kse_zombie_lock, &kse_zombie_lock, "kse zombie lock", MTX_SPIN);
 
+static int queue_sigchild = 0;
+SYSCTL_DECL(_kern_sigqueue);
+SYSCTL_INT(_kern_sigqueue, OID_AUTO, queue_sigchild, CTLFLAG_RD,
+    &queue_sigchild, 0, "queue SIGCHILD");
+TUNABLE_INT("kern.sigqueue.queue_sigchild", &queue_sigchild);
+
 static int
 sysctl_kse_virtual_cpu(SYSCTL_HANDLER_ARGS)
 {
@@ -278,6 +284,15 @@ proc_linkup(struct proc *p, struct ksegrp *kg, struct thread *td)
 	TAILQ_INIT(&p->p_threads);	     /* all threads in proc */
 	TAILQ_INIT(&p->p_suspended);	     /* Threads suspended */
 	sigqueue_init(&p->p_sigqueue, p);
+	if (queue_sigchild) {
+		p->p_ksi = ksiginfo_alloc(1);
+		if (p->p_ksi != NULL) {
+			/* p_ksi may be null if ksiginfo zone is not ready */
+			p->p_ksi->ksi_flags = KSI_EXT | KSI_INS;
+		}
+	}
+	else
+		p->p_ksi = NULL;
 	p->p_numksegrps = 0;
 	p->p_numthreads = 0;
 
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index 5a99e5f..661ea70 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -544,6 +544,7 @@ struct proc {
 	LIST_ENTRY(proc) p_sibling;	/* (e) List of sibling processes. */
 	LIST_HEAD(, proc) p_children;	/* (e) Pointer to list of children. */
 	struct mtx	p_mtx;		/* (n) Lock for this struct. */
+	struct ksiginfo *p_ksi;	/* Locked by parent proc lock */
 	sigqueue_t	p_sigqueue;	/* (c) Sigs not delivered to a td. */
 #define p_siglist	p_sigqueue.sq_signals
 
@@ -930,6 +931,9 @@ int	thread_sleep_check(struct thread *td);
 void	thread_stash(struct thread *td);
 int	thread_statclock(int user);
 void	thread_stopped(struct proc *p);
+void	childproc_stopped(struct proc *child, int reason);
+void	childproc_continued(struct proc *child);
+void	childproc_exited(struct proc *child);
 int	thread_suspend_check(int how);
 void	thread_suspend_one(struct thread *td);
 struct thread *thread_switchout(struct thread *td, int flags,
diff --git a/sys/sys/signalvar.h b/sys/sys/signalvar.h
index 0338635..654686d 100644
--- a/sys/sys/signalvar.h
+++ b/sys/sys/signalvar.h
@@ -333,7 +333,7 @@ int	tdsignal(struct proc *p, struct thread *td, int sig,
 	    ksiginfo_t *ksi);
 void	trapsignal(struct thread *td, ksiginfo_t *);
 int	ptracestop(struct thread *td, int sig);
-ksiginfo_t * ksiginfo_alloc(void);
+ksiginfo_t * ksiginfo_alloc(int);
 void	ksiginfo_free(ksiginfo_t *);
 void	sigqueue_init(struct sigqueue *queue, struct proc *p);
 void	sigqueue_flush(struct sigqueue *queue);