MFC 292894,292896: Add ptrace(2) reporting for LWP events.

292894:
Add ptrace(2) reporting for LWP events.

Add two new LWPINFO flags: PL_FLAG_BORN and PL_FLAG_EXITED for reporting
thread creation and destruction. Newly created threads will stop to report
PL_FLAG_BORN before returning to userland and exiting threads will stop to
report PL_FLAG_EXIT before exiting completely. Both of these events are
only enabled and reported if PT_LWP_EVENTS is enabled on a process.

292896:
Document the recently added support for ptrace(2) LWP events.

8 files changed, 301 insertions, 30 deletions

diff --git a/lib/libc/sys/ptrace.2 b/lib/libc/sys/ptrace.2
index a94e314..786f457 100644
--- a/lib/libc/sys/ptrace.2
+++ b/lib/libc/sys/ptrace.2
@@ -2,7 +2,7 @@
 .\"	$NetBSD: ptrace.2,v 1.2 1995/02/27 12:35:37 cgd Exp $
 .\"
 .\" This file is in the public domain.
-.Dd October 20, 2015
+.Dd December 29, 2015
 .Dt PTRACE 2
 .Os
 .Sh NAME
@@ -372,6 +372,20 @@ The flag is set for first event reported from a new child, which is
 automatically attached due to
 .Dv PT_FOLLOW_FORK
 enabled.
+.It PL_FLAG_BORN
+This flag is set for the first event reported from a new LWP when LWP
+events are enabled via
+.Dv PT_LWP_EVENTS .
+It is reported along with
+.Dv PL_FLAG_SCX
+and is always reported if LWP events are enabled.
+.It PL_FLAG_EXITED
+This flag is set for the last event reported by an exiting LWP when
+LWP events are enabled via
+.Dv PT_LWP_EVENTS .
+Note that this event is not reported when the last LWP in a process exits.
+The termination of the last thread is reported via a normal process exit
+event.
 .El
 .It pl_sigmask
 The current signal mask of the LWP
@@ -463,6 +477,29 @@ Child processes do not inherit this property.
 The traced process will set the
 .Dv PL_FLAG_FORKED
 flag upon exit from a system call that creates a new process.
+.It PT_LWP_EVENTS
+This request controls tracing of LWP creation and destruction.
+If
+.Fa data
+is non-zero,
+then LWPs will stop to report creation and destruction events.
+If
+.Fa data
+is zero,
+then LWP creation and destruction events will not be reported.
+By default, tracing is not enabled for LWP events.
+Child processes do not inherit this property.
+New LWPs will stop to report an event with
+.Dv PL_FLAG_BORN
+set before executing their first instruction.
+Exiting LWPs will stop to report an event with
+.Dv PL_FLAG_EXITED
+set before completing their termination.
+.Pp
+Note that new processes do not report an event for the creation of their
+initial thread,
+and exiting processes do not report an event for the termination of the
+last thread.
 .It PT_VM_TIMESTAMP
 This request returns the generation number or timestamp of the memory map of
 the traced process as the return value from
diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index a84b619..3d7bf09 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -1083,7 +1083,7 @@ fork_return(struct thread *td, struct trapframe *frame)
 			cv_broadcast(&p->p_dbgwait);
 		}
 		PROC_UNLOCK(p);
-	} else if (p->p_flag & P_TRACED) {
+	} else if (p->p_flag & P_TRACED || td->td_dbgflags & TDB_BORN) {
  		/*
 		 * This is the start of a new thread in a traced
 		 * process.  Report a system call exit event.
@@ -1091,9 +1091,10 @@ fork_return(struct thread *td, struct trapframe *frame)
 		PROC_LOCK(p);
 		td->td_dbgflags |= TDB_SCX;
 		_STOPEVENT(p, S_SCX, td->td_dbg_sc_code);
-		if ((p->p_stops & S_PT_SCX) != 0)
+		if ((p->p_stops & S_PT_SCX) != 0 ||
+		    (td->td_dbgflags & TDB_BORN) != 0)
 			ptracestop(td, SIGTRAP);
-		td->td_dbgflags &= ~TDB_SCX;
+		td->td_dbgflags &= ~(TDB_SCX | TDB_BORN);
 		PROC_UNLOCK(p);
 	}
 
diff --git a/sys/kern/kern_sig.c b/sys/kern/kern_sig.c
index 29783f8..65675dd 100644
--- a/sys/kern/kern_sig.c
+++ b/sys/kern/kern_sig.c
@@ -2488,7 +2488,12 @@ ptracestop(struct thread *td, int sig)
 	    td->td_tid, p->p_pid, td->td_dbgflags, sig);
 	PROC_SLOCK(p);
 	while ((p->p_flag & P_TRACED) && (td->td_dbgflags & TDB_XSIG)) {
-		if (p->p_flag & P_SINGLE_EXIT) {
+		if (p->p_flag & P_SINGLE_EXIT &&
+		    !(td->td_dbgflags & TDB_EXIT)) {
+			/*
+			 * Ignore ptrace stops except for thread exit
+			 * events when the process exits.
+			 */
 			td->td_dbgflags &= ~TDB_XSIG;
 			PROC_SUNLOCK(p);
 			return (sig);
diff --git a/sys/kern/kern_thr.c b/sys/kern/kern_thr.c
index 98965b1..8b19967 100644
--- a/sys/kern/kern_thr.c
+++ b/sys/kern/kern_thr.c
@@ -252,6 +252,8 @@ thread_create(struct thread *td, struct rtprio *rtp,
 	thread_unlock(td);
 	if (P_SHOULDSTOP(p))
 		newtd->td_flags |= TDF_ASTPENDING | TDF_NEEDSUSPCHK;
+	if (p->p_flag2 & P2_LWP_EVENTS)
+		newtd->td_dbgflags |= TDB_BORN;
 	PROC_UNLOCK(p);
 
 	tidhash_add(newtd);
@@ -314,29 +316,54 @@ kern_thr_exit(struct thread *td)
 
 	p = td->td_proc;
 
-	rw_wlock(&tidhash_lock);
+	/*
+	 * If all of the threads in a process call this routine to
+	 * exit (e.g. all threads call pthread_exit()), exactly one
+	 * thread should return to the caller to terminate the process
+	 * instead of the thread.
+	 *
+	 * Checking p_numthreads alone is not sufficient since threads
+	 * might be committed to terminating while the PROC_LOCK is
+	 * dropped in either ptracestop() or while removing this thread
+	 * from the tidhash.  Instead, the p_pendingexits field holds
+	 * the count of threads in either of those states and a thread
+	 * is considered the "last" thread if all of the other threads
+	 * in a process are already terminating.
+	 */
 	PROC_LOCK(p);
-
-	if (p->p_numthreads != 1) {
-		racct_sub(p, RACCT_NTHR, 1);
-		LIST_REMOVE(td, td_hash);
-		rw_wunlock(&tidhash_lock);
-		tdsigcleanup(td);
-		umtx_thread_exit(td);
-		PROC_SLOCK(p);
-		thread_stopped(p);
-		thread_exit();
-		/* NOTREACHED */
+	if (p->p_numthreads == p->p_pendingexits + 1) {
+		/*
+		 * Ignore attempts to shut down last thread in the
+		 * proc.  This will actually call _exit(2) in the
+		 * usermode trampoline when it returns.
+		 */
+		PROC_UNLOCK(p);
+		return (0);
 	}
 
+	p->p_pendingexits++;
+	td->td_dbgflags |= TDB_EXIT;
+	if (p->p_flag & P_TRACED && p->p_flag2 & P2_LWP_EVENTS)
+		ptracestop(td, SIGTRAP);
+	PROC_UNLOCK(p);
+	tidhash_remove(td);
+	PROC_LOCK(p);
+	p->p_pendingexits--;
+
 	/*
-	 * Ignore attempts to shut down last thread in the proc.  This
-	 * will actually call _exit(2) in the usermode trampoline when
-	 * it returns.
+	 * The check above should prevent all other threads from this
+	 * process from exiting while the PROC_LOCK is dropped, so
+	 * there must be at least one other thread other than the
+	 * current thread.
 	 */
-	PROC_UNLOCK(p);
-	rw_wunlock(&tidhash_lock);
-	return (0);
+	KASSERT(p->p_numthreads > 1, ("too few threads"));
+	racct_sub(p, RACCT_NTHR, 1);
+	tdsigcleanup(td);
+	umtx_thread_exit(td);
+	PROC_SLOCK(p);
+	thread_stopped(p);
+	thread_exit();
+	/* NOTREACHED */
 }
 
 int
diff --git a/sys/kern/sys_process.c b/sys/kern/sys_process.c
index 96e0181..66e9097 100644
--- a/sys/kern/sys_process.c
+++ b/sys/kern/sys_process.c
@@ -666,6 +666,7 @@ kern_ptrace(struct thread *td, int req, pid_t pid, void *addr, int data)
 	case PT_TO_SCX:
 	case PT_SYSCALL:
 	case PT_FOLLOW_FORK:
+	case PT_LWP_EVENTS:
 	case PT_DETACH:
 		sx_xlock(&proctree_lock);
 		proctree_locked = 1;
@@ -905,6 +906,16 @@ kern_ptrace(struct thread *td, int req, pid_t pid, void *addr, int data)
 			p->p_flag &= ~P_FOLLOWFORK;
 		break;
 
+	case PT_LWP_EVENTS:
+		CTR3(KTR_PTRACE, "PT_LWP_EVENTS: pid %d %s -> %s", p->p_pid,
+		    p->p_flag2 & P2_LWP_EVENTS ? "enabled" : "disabled",
+		    data ? "enabled" : "disabled");
+		if (data)
+			p->p_flag2 |= P2_LWP_EVENTS;
+		else
+			p->p_flag2 &= ~P2_LWP_EVENTS;
+		break;
+
 	case PT_STEP:
 	case PT_CONTINUE:
 	case PT_TO_SCE:
@@ -1227,6 +1238,10 @@ kern_ptrace(struct thread *td, int req, pid_t pid, void *addr, int data)
 		}
 		if (td2->td_dbgflags & TDB_CHILD)
 			pl->pl_flags |= PL_FLAG_CHILD;
+		if (td2->td_dbgflags & TDB_BORN)
+			pl->pl_flags |= PL_FLAG_BORN;
+		if (td2->td_dbgflags & TDB_EXIT)
+			pl->pl_flags |= PL_FLAG_EXITED;
 		pl->pl_sigmask = td2->td_sigmask;
 		pl->pl_siglist = td2->td_siglist;
 		strcpy(pl->pl_tdname, td2->td_name);
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index b2de02d..c4b51e7 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -396,6 +396,8 @@ do {									\
 #define	TDB_STOPATFORK	0x00000080 /* Stop at the return from fork (child
 				      only) */
 #define	TDB_CHILD	0x00000100 /* New child indicator for ptrace() */
+#define	TDB_BORN	0x00000200 /* New LWP indicator for ptrace() */
+#define	TDB_EXIT	0x00000400 /* Exiting LWP indicator for ptrace() */
 
 /*
  * "Private" flags kept in td_pflags:
@@ -551,6 +553,7 @@ struct proc {
 	int		p_pendingcnt;	/* how many signals are pending */
 	struct itimers	*p_itimers;	/* (c) POSIX interval timers. */
 	struct procdesc	*p_procdesc;	/* (e) Process descriptor, if any. */
+	int		p_pendingexits; /* (c) Count of pending thread exits. */
 /* End area that is zeroed on creation. */
 #define	p_endzero	p_magic
 
@@ -674,6 +677,7 @@ struct proc {
 #define	P2_NOTRACE	0x00000002	/* No ptrace(2) attach or coredumps. */
 #define	P2_NOTRACE_EXEC 0x00000004	/* Keep P2_NOPTRACE on exec(2). */
 #define	P2_AST_SU	0x00000008	/* Handles SU ast for kthreads. */
+#define	P2_LWP_EVENTS	0x00000010	/* Report LWP events via ptrace(2). */
 
 /* Flags protected by proctree_lock, kept in p_treeflags. */
 #define	P_TREE_ORPHANED		0x00000001	/* Reparented, on orphan list */
diff --git a/sys/sys/ptrace.h b/sys/sys/ptrace.h
index 7135767..a75d911 100644
--- a/sys/sys/ptrace.h
+++ b/sys/sys/ptrace.h
@@ -64,6 +64,7 @@
 #define	PT_SYSCALL	22
 
 #define	PT_FOLLOW_FORK	23
+#define	PT_LWP_EVENTS	24	/* report LWP birth and exit */
 
 #define PT_GETREGS      33	/* get general-purpose registers */
 #define PT_SETREGS      34	/* set general-purpose registers */
@@ -108,6 +109,8 @@ struct ptrace_lwpinfo {
 #define	PL_FLAG_SI	0x20	/* siginfo is valid */
 #define	PL_FLAG_FORKED	0x40	/* new child */
 #define	PL_FLAG_CHILD	0x80	/* I am from child */
+#define	PL_FLAG_BORN	0x100	/* new LWP */
+#define	PL_FLAG_EXITED	0x200	/* exiting LWP */
 	sigset_t	pl_sigmask;	/* LWP signal mask */
 	sigset_t	pl_siglist;	/* LWP pending signal */
 	struct __siginfo pl_siginfo;	/* siginfo for signal */
diff --git a/tests/sys/kern/ptrace_test.c b/tests/sys/kern/ptrace_test.c
index 1be1144..54ef2cd 100644
--- a/tests/sys/kern/ptrace_test.c
+++ b/tests/sys/kern/ptrace_test.c
@@ -1008,6 +1008,16 @@ simple_thread(void *arg __unused)
 	pthread_exit(NULL);
 }
 
+static __dead2 void
+simple_thread_main(void)
+{
+	pthread_t thread;
+
+	CHILD_REQUIRE(pthread_create(&thread, NULL, simple_thread, NULL) == 0);
+	CHILD_REQUIRE(pthread_join(thread, NULL) == 0);
+	exit(1);
+}
+
 /*
  * Verify that pl_syscall_code in struct ptrace_lwpinfo for a new
  * thread reports the correct value.
@@ -1022,14 +1032,8 @@ ATF_TC_BODY(ptrace__new_child_pl_syscall_code_thread, tc)
 
 	ATF_REQUIRE((fpid = fork()) != -1);
 	if (fpid == 0) {
-		pthread_t thread;
-
 		trace_me();
-
-		CHILD_REQUIRE(pthread_create(&thread, NULL, simple_thread,
-			NULL) == 0);
-		CHILD_REQUIRE(pthread_join(thread, NULL) == 0);
-		exit(1);
+		simple_thread_main();
 	}
 
 	/* The first wait() should report the stop from SIGSTOP. */
@@ -1092,6 +1096,179 @@ ATF_TC_BODY(ptrace__new_child_pl_syscall_code_thread, tc)
 	ATF_REQUIRE(errno == ECHILD);
 }
 
+/*
+ * Verify that the expected LWP events are reported for a child thread.
+ */
+ATF_TC_WITHOUT_HEAD(ptrace__lwp_events);
+ATF_TC_BODY(ptrace__lwp_events, tc)
+{
+	struct ptrace_lwpinfo pl;
+	pid_t fpid, wpid;
+	lwpid_t lwps[2];
+	int status;
+
+	ATF_REQUIRE((fpid = fork()) != -1);
+	if (fpid == 0) {
+		trace_me();
+		simple_thread_main();
+	}
+
+	/* The first wait() should report the stop from SIGSTOP. */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(wpid == fpid);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGSTOP);
+
+	ATF_REQUIRE(ptrace(PT_LWPINFO, wpid, (caddr_t)&pl,
+	    sizeof(pl)) != -1);
+	lwps[0] = pl.pl_lwpid;
+
+	ATF_REQUIRE(ptrace(PT_LWP_EVENTS, wpid, NULL, 1) == 0);
+
+	/* Continue the child ignoring the SIGSTOP. */
+	ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+
+	/* The first event should be for the child thread's birth. */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(wpid == fpid);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGTRAP);
+		
+	ATF_REQUIRE(ptrace(PT_LWPINFO, wpid, (caddr_t)&pl, sizeof(pl)) != -1);
+	ATF_REQUIRE((pl.pl_flags & (PL_FLAG_BORN | PL_FLAG_SCX)) ==
+	    (PL_FLAG_BORN | PL_FLAG_SCX));
+	ATF_REQUIRE(pl.pl_lwpid != lwps[0]);
+	lwps[1] = pl.pl_lwpid;
+
+	ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+
+	/* The next event should be for the child thread's death. */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(wpid == fpid);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGTRAP);
+		
+	ATF_REQUIRE(ptrace(PT_LWPINFO, wpid, (caddr_t)&pl, sizeof(pl)) != -1);
+	ATF_REQUIRE((pl.pl_flags & (PL_FLAG_EXITED | PL_FLAG_SCE)) ==
+	    (PL_FLAG_EXITED | PL_FLAG_SCE));
+	ATF_REQUIRE(pl.pl_lwpid == lwps[1]);
+
+	ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+
+	/* The last event should be for the child process's exit. */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(WIFEXITED(status));
+	ATF_REQUIRE(WEXITSTATUS(status) == 1);
+
+	wpid = wait(&status);
+	ATF_REQUIRE(wpid == -1);
+	ATF_REQUIRE(errno == ECHILD);
+}
+
+static void *
+exec_thread(void *arg __unused)
+{
+
+	execl("/usr/bin/true", "true", NULL);
+	exit(127);
+}
+
+static __dead2 void
+exec_thread_main(void)
+{
+	pthread_t thread;
+
+	CHILD_REQUIRE(pthread_create(&thread, NULL, exec_thread, NULL) == 0);
+	for (;;)
+		sleep(60);
+	exit(1);
+}
+
+/*
+ * Verify that the expected LWP events are reported for a multithreaded
+ * process that calls execve(2).
+ */
+ATF_TC_WITHOUT_HEAD(ptrace__lwp_events_exec);
+ATF_TC_BODY(ptrace__lwp_events_exec, tc)
+{
+	struct ptrace_lwpinfo pl;
+	pid_t fpid, wpid;
+	lwpid_t lwps[2];
+	int status;
+
+	ATF_REQUIRE((fpid = fork()) != -1);
+	if (fpid == 0) {
+		trace_me();
+		exec_thread_main();
+	}
+
+	/* The first wait() should report the stop from SIGSTOP. */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(wpid == fpid);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGSTOP);
+
+	ATF_REQUIRE(ptrace(PT_LWPINFO, wpid, (caddr_t)&pl,
+	    sizeof(pl)) != -1);
+	lwps[0] = pl.pl_lwpid;
+
+	ATF_REQUIRE(ptrace(PT_LWP_EVENTS, wpid, NULL, 1) == 0);
+
+	/* Continue the child ignoring the SIGSTOP. */
+	ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+
+	/* The first event should be for the child thread's birth. */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(wpid == fpid);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGTRAP);
+		
+	ATF_REQUIRE(ptrace(PT_LWPINFO, wpid, (caddr_t)&pl, sizeof(pl)) != -1);
+	ATF_REQUIRE((pl.pl_flags & (PL_FLAG_BORN | PL_FLAG_SCX)) ==
+	    (PL_FLAG_BORN | PL_FLAG_SCX));
+	ATF_REQUIRE(pl.pl_lwpid != lwps[0]);
+	lwps[1] = pl.pl_lwpid;
+
+	ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+
+	/*
+	 * The next event should be for the main thread's death due to
+	 * single threading from execve().
+	 */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(wpid == fpid);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGTRAP);
+
+	ATF_REQUIRE(ptrace(PT_LWPINFO, wpid, (caddr_t)&pl, sizeof(pl)) != -1);
+	ATF_REQUIRE((pl.pl_flags & (PL_FLAG_EXITED | PL_FLAG_SCE)) ==
+	    (PL_FLAG_EXITED));
+	ATF_REQUIRE(pl.pl_lwpid == lwps[0]);
+
+	ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+
+	/* The next event should be for the child process's exec. */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGTRAP);
+
+	ATF_REQUIRE(ptrace(PT_LWPINFO, wpid, (caddr_t)&pl, sizeof(pl)) != -1);
+	ATF_REQUIRE((pl.pl_flags & (PL_FLAG_EXEC | PL_FLAG_SCX)) ==
+	    (PL_FLAG_EXEC | PL_FLAG_SCX));
+	ATF_REQUIRE(pl.pl_lwpid == lwps[1]);
+
+	ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+
+	/* The last event should be for the child process's exit. */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(WIFEXITED(status));
+	ATF_REQUIRE(WEXITSTATUS(status) == 0);
+
+	wpid = wait(&status);
+	ATF_REQUIRE(wpid == -1);
+	ATF_REQUIRE(errno == ECHILD);
+}
+
 ATF_TP_ADD_TCS(tp)
 {
 
@@ -1110,6 +1287,8 @@ ATF_TP_ADD_TCS(tp)
 	ATF_TP_ADD_TC(tp, ptrace__new_child_pl_syscall_code_fork);
 	ATF_TP_ADD_TC(tp, ptrace__new_child_pl_syscall_code_vfork);
 	ATF_TP_ADD_TC(tp, ptrace__new_child_pl_syscall_code_thread);
+	ATF_TP_ADD_TC(tp, ptrace__lwp_events);
+	ATF_TP_ADD_TC(tp, ptrace__lwp_events_exec);
 
 	return (atf_no_error());
 }