Fix various edge cases related to system call tracing.

- Always set td_dbg_sc_* when P_TRACED is set on system call entry
  even if the debugger is not tracing system call entries.  This
  ensures the fields are valid when reporting other stops that
  occur at system call boundaries such as for PT_FOLLOW_FORKS or
  when only tracing system call exits.
- Set TDB_SCX when reporting the stop for a new child process in
  fork_return().  This causes the event to be reported as a system
  call exit.
- Report a system call exit event in fork_return() for new threads in
  a traced process.
- Copy td_dbg_sc_* to new threads instead of zeroing.  This ensures
  that td_dbg_sc_code in particular will report the system call that
  created the new thread or process when it reports a system call
  exit event in fork_return().
- Add new ptrace tests to verify that new child processes and threads
  report system call exit events with a valid pl_syscall_code via
  PT_LWPINFO.

Reviewed by:	kib
Differential Revision:	https://reviews.freebsd.org/D3822

5 files changed, 266 insertions, 22 deletions

diff --git a/sys/kern/kern_fork.c b/sys/kern/kern_fork.c
index b2727bc..9256f39 100644
--- a/sys/kern/kern_fork.c
+++ b/sys/kern/kern_fork.c
@@ -57,6 +57,7 @@ __FBSDID("$FreeBSD$");
 #include <sys/proc.h>
 #include <sys/procdesc.h>
 #include <sys/pioctl.h>
+#include <sys/ptrace.h>
 #include <sys/racct.h>
 #include <sys/resourcevar.h>
 #include <sys/sched.h>
@@ -1031,8 +1032,8 @@ fork_return(struct thread *td, struct trapframe *frame)
 {
 	struct proc *p, *dbg;
 
+	p = td->td_proc;
 	if (td->td_dbgflags & TDB_STOPATFORK) {
-		p = td->td_proc;
 		sx_xlock(&proctree_lock);
 		PROC_LOCK(p);
 		if ((p->p_pptr->p_flag & (P_TRACED | P_FOLLOWFORK)) ==
@@ -1049,9 +1050,9 @@ fork_return(struct thread *td, struct trapframe *frame)
 			    p->p_pid, p->p_oppid);
 			proc_reparent(p, dbg);
 			sx_xunlock(&proctree_lock);
-			td->td_dbgflags |= TDB_CHILD;
+			td->td_dbgflags |= TDB_CHILD | TDB_SCX;
 			ptracestop(td, SIGSTOP);
-			td->td_dbgflags &= ~TDB_CHILD;
+			td->td_dbgflags &= ~(TDB_CHILD | TDB_SCX);
 		} else {
 			/*
 			 * ... otherwise clear the request.
@@ -1061,6 +1062,18 @@ fork_return(struct thread *td, struct trapframe *frame)
 			cv_broadcast(&p->p_dbgwait);
 		}
 		PROC_UNLOCK(p);
+	} else if (p->p_flag & P_TRACED) {
+ 		/*
+		 * This is the start of a new thread in a traced
+		 * process.  Report a system call exit event.
+		 */
+		PROC_LOCK(p);
+		td->td_dbgflags |= TDB_SCX;
+		_STOPEVENT(p, S_SCX, td->td_dbg_sc_code);
+		if ((p->p_stops & S_PT_SCX) != 0)
+			ptracestop(td, SIGTRAP);
+		td->td_dbgflags &= ~TDB_SCX;
+		PROC_UNLOCK(p);
 	}
 
 	userret(td, frame);
diff --git a/sys/kern/subr_syscall.c b/sys/kern/subr_syscall.c
index af09613..71f3d59 100644
--- a/sys/kern/subr_syscall.c
+++ b/sys/kern/subr_syscall.c
@@ -83,11 +83,12 @@ syscallenter(struct thread *td, struct syscall_args *sa)
 	if (error == 0) {
 
 		STOPEVENT(p, S_SCE, sa->narg);
-		if (p->p_flag & P_TRACED && p->p_stops & S_PT_SCE) {
+		if (p->p_flag & P_TRACED) {
 			PROC_LOCK(p);
 			td->td_dbg_sc_code = sa->code;
 			td->td_dbg_sc_narg = sa->narg;
-			ptracestop((td), SIGTRAP);
+			if (p->p_stops & S_PT_SCE)
+				ptracestop((td), SIGTRAP);
 			PROC_UNLOCK(p);
 		}
 		if (td->td_dbgflags & TDB_USERWR) {
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index ae01bbf..749c18e 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -283,8 +283,6 @@ struct thread {
 	int		td_no_sleeping;	/* (k) Sleeping disabled count. */
 	int		td_dom_rr_idx;	/* (k) RR Numa domain selection. */
 	void		*td_su;		/* (k) FFS SU private */
-	u_int		td_dbg_sc_code;	/* (c) Syscall code to debugger. */
-	u_int		td_dbg_sc_narg;	/* (c) Syscall arg count to debugger.*/
 #define	td_endzero td_sigmask
 
 /* Copied during fork1() or create_thread(). */
@@ -296,6 +294,8 @@ struct thread {
 	u_char		td_pri_class;	/* (t) Scheduling class. */
 	u_char		td_user_pri;	/* (t) User pri from estcpu and nice. */
 	u_char		td_base_user_pri; /* (t) Base user pri */
+	u_int		td_dbg_sc_code;	/* (c) Syscall code to debugger. */
+	u_int		td_dbg_sc_narg;	/* (c) Syscall arg count to debugger.*/
 #define	td_endcopy td_pcb
 
 /*
diff --git a/tests/sys/kern/Makefile b/tests/sys/kern/Makefile
index bf6aa0d..c345e5d 100644
--- a/tests/sys/kern/Makefile
+++ b/tests/sys/kern/Makefile
@@ -7,6 +7,7 @@ ATF_TESTS_C+=	ptrace_test
 ATF_TESTS_C+=	unix_seqpacket_test
 TEST_METADATA.unix_seqpacket_test+=	timeout="15"
 
+LDADD.ptrace_test+=			-lpthread
 LDADD.unix_seqpacket_test+=		-lpthread
 
 WARNS?=	5
diff --git a/tests/sys/kern/ptrace_test.c b/tests/sys/kern/ptrace_test.c
index 33878a4..3393a07 100644
--- a/tests/sys/kern/ptrace_test.c
+++ b/tests/sys/kern/ptrace_test.c
@@ -29,10 +29,12 @@ __FBSDID("$FreeBSD$");
 
 #include <sys/types.h>
 #include <sys/ptrace.h>
+#include <sys/syscall.h>
 #include <sys/sysctl.h>
 #include <sys/user.h>
 #include <sys/wait.h>
 #include <errno.h>
+#include <pthread.h>
 #include <signal.h>
 #include <stdio.h>
 #include <stdlib.h>
@@ -409,12 +411,15 @@ ATF_TC_BODY(ptrace__parent_sees_exit_after_unrelated_debugger, tc)
  * debugger is attached to it.
  */
 static __dead2 void
-follow_fork_parent(void)
+follow_fork_parent(bool use_vfork)
 {
 	pid_t fpid, wpid;
 	int status;
 
-	CHILD_REQUIRE((fpid = fork()) != -1);
+	if (use_vfork)
+		CHILD_REQUIRE((fpid = vfork()) != -1);
+	else
+		CHILD_REQUIRE((fpid = fork()) != -1);
 
 	if (fpid == 0)
 		/* Child */
@@ -434,7 +439,7 @@ follow_fork_parent(void)
  * child process.
  */
 static pid_t
-handle_fork_events(pid_t parent)
+handle_fork_events(pid_t parent, struct ptrace_lwpinfo *ppl)
 {
 	struct ptrace_lwpinfo pl;
 	bool fork_reported[2];
@@ -469,6 +474,8 @@ handle_fork_events(pid_t parent)
 				child = wpid;
 			else
 				ATF_REQUIRE(child == wpid);
+			if (ppl != NULL)
+				ppl[1] = pl;
 			fork_reported[1] = true;
 		} else {
 			ATF_REQUIRE(wpid == parent);
@@ -478,6 +485,8 @@ handle_fork_events(pid_t parent)
 				child = pl.pl_child_pid;
 			else
 				ATF_REQUIRE(child == pl.pl_child_pid);
+			if (ppl != NULL)
+				ppl[0] = pl;
 			fork_reported[0] = true;
 		}
 	}
@@ -499,7 +508,7 @@ ATF_TC_BODY(ptrace__follow_fork_both_attached, tc)
 	ATF_REQUIRE((fpid = fork()) != -1);
 	if (fpid == 0) {
 		trace_me();
-		follow_fork_parent();
+		follow_fork_parent(false);
 	}
 
 	/* Parent process. */
@@ -516,7 +525,7 @@ ATF_TC_BODY(ptrace__follow_fork_both_attached, tc)
 	/* Continue the child ignoring the SIGSTOP. */
 	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
 
-	children[1] = handle_fork_events(children[0]);
+	children[1] = handle_fork_events(children[0], NULL);
 	ATF_REQUIRE(children[1] > 0);
 
 	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
@@ -555,7 +564,7 @@ ATF_TC_BODY(ptrace__follow_fork_child_detached, tc)
 	ATF_REQUIRE((fpid = fork()) != -1);
 	if (fpid == 0) {
 		trace_me();
-		follow_fork_parent();
+		follow_fork_parent(false);
 	}
 
 	/* Parent process. */
@@ -572,7 +581,7 @@ ATF_TC_BODY(ptrace__follow_fork_child_detached, tc)
 	/* Continue the child ignoring the SIGSTOP. */
 	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
 
-	children[1] = handle_fork_events(children[0]);
+	children[1] = handle_fork_events(children[0], NULL);
 	ATF_REQUIRE(children[1] > 0);
 
 	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
@@ -606,7 +615,7 @@ ATF_TC_BODY(ptrace__follow_fork_parent_detached, tc)
 	ATF_REQUIRE((fpid = fork()) != -1);
 	if (fpid == 0) {
 		trace_me();
-		follow_fork_parent();
+		follow_fork_parent(false);
 	}
 
 	/* Parent process. */
@@ -623,7 +632,7 @@ ATF_TC_BODY(ptrace__follow_fork_parent_detached, tc)
 	/* Continue the child ignoring the SIGSTOP. */
 	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
 
-	children[1] = handle_fork_events(children[0]);
+	children[1] = handle_fork_events(children[0], NULL);
 	ATF_REQUIRE(children[1] > 0);
 
 	ATF_REQUIRE(ptrace(PT_DETACH, children[0], (caddr_t)1, 0) != -1);
@@ -688,7 +697,7 @@ ATF_TC_BODY(ptrace__follow_fork_both_attached_unrelated_debugger, tc)
 	ATF_REQUIRE((fpid = fork()) != -1);
 	if (fpid == 0) {
 		attach_fork_parent(cpipe);
-		follow_fork_parent();
+		follow_fork_parent(false);
 	}
 
 	/* Parent process. */
@@ -715,7 +724,7 @@ ATF_TC_BODY(ptrace__follow_fork_both_attached_unrelated_debugger, tc)
 	/* Signal the fork parent to continue. */
 	close(cpipe[0]);
 
-	children[1] = handle_fork_events(children[0]);
+	children[1] = handle_fork_events(children[0], NULL);
 	ATF_REQUIRE(children[1] > 0);
 
 	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
@@ -756,7 +765,7 @@ ATF_TC_BODY(ptrace__follow_fork_child_detached_unrelated_debugger, tc)
 	ATF_REQUIRE((fpid = fork()) != -1);
 	if (fpid == 0) {
 		attach_fork_parent(cpipe);
-		follow_fork_parent();
+		follow_fork_parent(false);
 	}
 
 	/* Parent process. */
@@ -783,7 +792,7 @@ ATF_TC_BODY(ptrace__follow_fork_child_detached_unrelated_debugger, tc)
 	/* Signal the fork parent to continue. */
 	close(cpipe[0]);
 
-	children[1] = handle_fork_events(children[0]);
+	children[1] = handle_fork_events(children[0], NULL);
 	ATF_REQUIRE(children[1] > 0);
 
 	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
@@ -819,7 +828,7 @@ ATF_TC_BODY(ptrace__follow_fork_parent_detached_unrelated_debugger, tc)
 	ATF_REQUIRE((fpid = fork()) != -1);
 	if (fpid == 0) {
 		attach_fork_parent(cpipe);
-		follow_fork_parent();
+		follow_fork_parent(false);
 	}
 
 	/* Parent process. */
@@ -846,7 +855,7 @@ ATF_TC_BODY(ptrace__follow_fork_parent_detached_unrelated_debugger, tc)
 	/* Signal the fork parent to continue. */
 	close(cpipe[0]);
 
-	children[1] = handle_fork_events(children[0]);
+	children[1] = handle_fork_events(children[0], NULL);
 	ATF_REQUIRE(children[1] > 0);
 
 	ATF_REQUIRE(ptrace(PT_DETACH, children[0], (caddr_t)1, 0) != -1);
@@ -952,6 +961,223 @@ ATF_TC_BODY(ptrace__getppid, tc)
 	ATF_REQUIRE(WEXITSTATUS(status) == 1);
 }
 
+/*
+ * Verify that pl_syscall_code in struct ptrace_lwpinfo for a new
+ * child process created via fork() reports the correct value.
+ */
+ATF_TC_WITHOUT_HEAD(ptrace__new_child_pl_syscall_code_fork);
+ATF_TC_BODY(ptrace__new_child_pl_syscall_code_fork, tc)
+{
+	struct ptrace_lwpinfo pl[2];
+	pid_t children[2], fpid, wpid;
+	int status;
+
+	ATF_REQUIRE((fpid = fork()) != -1);
+	if (fpid == 0) {
+		trace_me();
+		follow_fork_parent(false);
+	}
+
+	/* Parent process. */
+	children[0] = fpid;
+
+	/* The first wait() should report the stop from SIGSTOP. */
+	wpid = waitpid(children[0], &status, 0);
+	ATF_REQUIRE(wpid == children[0]);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGSTOP);
+
+	ATF_REQUIRE(ptrace(PT_FOLLOW_FORK, children[0], NULL, 1) != -1);
+
+	/* Continue the child ignoring the SIGSTOP. */
+	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
+
+	/* Wait for both halves of the fork event to get reported. */
+	children[1] = handle_fork_events(children[0], pl);
+	ATF_REQUIRE(children[1] > 0);
+
+	ATF_REQUIRE((pl[0].pl_flags & PL_FLAG_SCX) != 0);
+	ATF_REQUIRE((pl[1].pl_flags & PL_FLAG_SCX) != 0);
+	ATF_REQUIRE(pl[0].pl_syscall_code == SYS_fork);
+	ATF_REQUIRE(pl[0].pl_syscall_code == pl[1].pl_syscall_code);
+	ATF_REQUIRE(pl[0].pl_syscall_narg == pl[1].pl_syscall_narg);
+
+	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
+	ATF_REQUIRE(ptrace(PT_CONTINUE, children[1], (caddr_t)1, 0) != -1);
+
+	/*
+	 * The child can't exit until the grandchild reports status, so the
+	 * grandchild should report its exit first to the debugger.
+	 */
+	wpid = wait(&status);
+	ATF_REQUIRE(wpid == children[1]);
+	ATF_REQUIRE(WIFEXITED(status));
+	ATF_REQUIRE(WEXITSTATUS(status) == 2);
+
+	wpid = wait(&status);
+	ATF_REQUIRE(wpid == children[0]);
+	ATF_REQUIRE(WIFEXITED(status));
+	ATF_REQUIRE(WEXITSTATUS(status) == 1);
+
+	wpid = wait(&status);
+	ATF_REQUIRE(wpid == -1);
+	ATF_REQUIRE(errno == ECHILD);
+}
+
+/*
+ * Verify that pl_syscall_code in struct ptrace_lwpinfo for a new
+ * child process created via vfork() reports the correct value.
+ */
+ATF_TC_WITHOUT_HEAD(ptrace__new_child_pl_syscall_code_vfork);
+ATF_TC_BODY(ptrace__new_child_pl_syscall_code_vfork, tc)
+{
+	struct ptrace_lwpinfo pl[2];
+	pid_t children[2], fpid, wpid;
+	int status;
+
+	ATF_REQUIRE((fpid = fork()) != -1);
+	if (fpid == 0) {
+		trace_me();
+		follow_fork_parent(true);
+	}
+
+	/* Parent process. */
+	children[0] = fpid;
+
+	/* The first wait() should report the stop from SIGSTOP. */
+	wpid = waitpid(children[0], &status, 0);
+	ATF_REQUIRE(wpid == children[0]);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGSTOP);
+
+	ATF_REQUIRE(ptrace(PT_FOLLOW_FORK, children[0], NULL, 1) != -1);
+
+	/* Continue the child ignoring the SIGSTOP. */
+	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
+
+	/* Wait for both halves of the fork event to get reported. */
+	children[1] = handle_fork_events(children[0], pl);
+	ATF_REQUIRE(children[1] > 0);
+
+	ATF_REQUIRE((pl[0].pl_flags & PL_FLAG_SCX) != 0);
+	ATF_REQUIRE((pl[1].pl_flags & PL_FLAG_SCX) != 0);
+	ATF_REQUIRE(pl[0].pl_syscall_code == SYS_vfork);
+	ATF_REQUIRE(pl[0].pl_syscall_code == pl[1].pl_syscall_code);
+	ATF_REQUIRE(pl[0].pl_syscall_narg == pl[1].pl_syscall_narg);
+
+	ATF_REQUIRE(ptrace(PT_CONTINUE, children[0], (caddr_t)1, 0) != -1);
+	ATF_REQUIRE(ptrace(PT_CONTINUE, children[1], (caddr_t)1, 0) != -1);
+
+	/*
+	 * The child can't exit until the grandchild reports status, so the
+	 * grandchild should report its exit first to the debugger.
+	 */
+	wpid = wait(&status);
+	ATF_REQUIRE(wpid == children[1]);
+	ATF_REQUIRE(WIFEXITED(status));
+	ATF_REQUIRE(WEXITSTATUS(status) == 2);
+
+	wpid = wait(&status);
+	ATF_REQUIRE(wpid == children[0]);
+	ATF_REQUIRE(WIFEXITED(status));
+	ATF_REQUIRE(WEXITSTATUS(status) == 1);
+
+	wpid = wait(&status);
+	ATF_REQUIRE(wpid == -1);
+	ATF_REQUIRE(errno == ECHILD);
+}
+
+static void *
+simple_thread(void *arg __unused)
+{
+
+	pthread_exit(NULL);
+}
+
+/*
+ * Verify that pl_syscall_code in struct ptrace_lwpinfo for a new
+ * thread reports the correct value.
+ */
+ATF_TC_WITHOUT_HEAD(ptrace__new_child_pl_syscall_code_thread);
+ATF_TC_BODY(ptrace__new_child_pl_syscall_code_thread, tc)
+{
+	struct ptrace_lwpinfo pl;
+	pid_t fpid, wpid;
+	lwpid_t main;
+	int status;
+
+	ATF_REQUIRE((fpid = fork()) != -1);
+	if (fpid == 0) {
+		pthread_t thread;
+
+		trace_me();
+
+		CHILD_REQUIRE(pthread_create(&thread, NULL, simple_thread,
+			NULL) == 0);
+		CHILD_REQUIRE(pthread_join(thread, NULL) == 0);
+		exit(1);
+	}
+
+	/* The first wait() should report the stop from SIGSTOP. */
+	wpid = waitpid(fpid, &status, 0);
+	ATF_REQUIRE(wpid == fpid);
+	ATF_REQUIRE(WIFSTOPPED(status));
+	ATF_REQUIRE(WSTOPSIG(status) == SIGSTOP);
+
+	ATF_REQUIRE(ptrace(PT_LWPINFO, wpid, (caddr_t)&pl,
+	    sizeof(pl)) != -1);
+	main = pl.pl_lwpid;
+
+	/*
+	 * Continue the child ignoring the SIGSTOP and tracing all
+	 * system call exits.
+	 */
+	ATF_REQUIRE(ptrace(PT_TO_SCX, fpid, (caddr_t)1, 0) != -1);
+
+	/*
+	 * Wait for the new thread to arrive.  pthread_create() might
+	 * invoke any number of system calls.  For now we just wait
+	 * for the new thread to arrive and make sure it reports a
+	 * valid system call code.  If ptrace grows thread event
+	 * reporting then this test can be made more precise.
+	 */
+	for (;;) {
+		wpid = waitpid(fpid, &status, 0);
+		ATF_REQUIRE(wpid == fpid);
+		ATF_REQUIRE(WIFSTOPPED(status));
+		ATF_REQUIRE(WSTOPSIG(status) == SIGTRAP);
+		
+		ATF_REQUIRE(ptrace(PT_LWPINFO, wpid, (caddr_t)&pl,
+		    sizeof(pl)) != -1);
+		ATF_REQUIRE((pl.pl_flags & PL_FLAG_SCX) != 0);
+		ATF_REQUIRE(pl.pl_syscall_code != 0);
+		if (pl.pl_lwpid != main)
+			/* New thread seen. */
+			break;
+
+		ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+	}
+
+	/* Wait for the child to exit. */
+	ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+	for (;;) {
+		wpid = waitpid(fpid, &status, 0);
+		ATF_REQUIRE(wpid == fpid);
+		if (WIFEXITED(status))
+			break;
+		
+		ATF_REQUIRE(WIFSTOPPED(status));
+		ATF_REQUIRE(WSTOPSIG(status) == SIGTRAP);
+		ATF_REQUIRE(ptrace(PT_CONTINUE, fpid, (caddr_t)1, 0) == 0);
+	}
+		
+	ATF_REQUIRE(WEXITSTATUS(status) == 1);
+
+	wpid = wait(&status);
+	ATF_REQUIRE(wpid == -1);
+	ATF_REQUIRE(errno == ECHILD);
+}
+
 ATF_TP_ADD_TCS(tp)
 {
 
@@ -968,6 +1194,9 @@ ATF_TP_ADD_TCS(tp)
 	ATF_TP_ADD_TC(tp,
 	    ptrace__follow_fork_parent_detached_unrelated_debugger);
 	ATF_TP_ADD_TC(tp, ptrace__getppid);
+	ATF_TP_ADD_TC(tp, ptrace__new_child_pl_syscall_code_fork);
+	ATF_TP_ADD_TC(tp, ptrace__new_child_pl_syscall_code_vfork);
+	ATF_TP_ADD_TC(tp, ptrace__new_child_pl_syscall_code_thread);
 
 	return (atf_no_error());
 }