MFC r264434:

DTrace's pid provider works by inserting breakpoint instructions at probe
sites and installing a hook at the kernel's trap handler. The fasttrap code
will emulate the overwritten instruction in some common cases, but otherwise
copies it out into some scratch space in the traced process' address space
and ensures that it's executed after returning from the trap.

In Solaris and illumos, this (per-thread) scratch space comes from some
reserved space in TLS, accessible via the fs segment register. This
approach is somewhat unappealing on FreeBSD since it would require some
modifications to rtld and jemalloc (for static TLS) to ensure that TLS is
executable, and would thus introduce dependencies on their implementation
details. I think it would also be impossible to safely trace static binaries
compiled without these modifications.

This change implements the functionality in a different way, by having
fasttrap map pages into the target process' address space on demand. Each
page is divided into 64-byte chunks for use by individual threads, and
fasttrap's process descriptor struct has been extended to keep track of
any scratch space allocated for the corresponding process.

With this change it's possible to trace all libc functions in a program,
e.g. with

  pid$target:libc.so.*::entry {@[probefunc] = count();}

Previously this would generally cause the victim process to crash, as
tracing memcpy on amd64 requires the functionality described above.

4 files changed, 251 insertions, 11 deletions

diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c
index 716dabc..7a30b1b 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c
@@ -61,8 +61,13 @@
 #include <sys/mutex.h>
 #include <sys/kernel.h>
 #if !defined(sun)
-#include <sys/user.h>
 #include <sys/dtrace_bsd.h>
+#include <sys/eventhandler.h>
+#include <sys/user.h>
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/vm_map.h>
+#include <vm/vm_param.h>
 #include <sys/u8_textprep.h>
 #include <cddl/dev/dtrace/dtrace_cddl.h>
 #endif
@@ -207,6 +212,10 @@ static void fasttrap_provider_free(fasttrap_provider_t *);
 static fasttrap_proc_t *fasttrap_proc_lookup(pid_t);
 static void fasttrap_proc_release(fasttrap_proc_t *);
 
+#if !defined(sun)
+static void fasttrap_thread_dtor(void *, struct thread *);
+#endif
+
 #define	FASTTRAP_PROVS_INDEX(pid, name) \
 	((fasttrap_hash_str(name) + (pid)) & fasttrap_provs.fth_mask)
 
@@ -214,6 +223,7 @@ static void fasttrap_proc_release(fasttrap_proc_t *);
 
 #if !defined(sun)
 static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
+static eventhandler_tag fasttrap_thread_dtor_tag;
 #endif
 
 static int
@@ -290,6 +300,118 @@ fasttrap_sigtrap(proc_t *p, kthread_t *t, uintptr_t pc)
 #endif
 }
 
+#if !defined(sun)
+/*
+ * Obtain a chunk of scratch space in the address space of the target process.
+ */
+fasttrap_scrspace_t *
+fasttrap_scraddr(struct thread *td, fasttrap_proc_t *fprc)
+{
+	fasttrap_scrblock_t *scrblk;
+	fasttrap_scrspace_t *scrspc;
+	struct proc *p;
+	vm_offset_t addr;
+	int error, i;
+
+	scrspc = NULL;
+	if (td->t_dtrace_sscr != NULL) {
+		/* If the thread already has scratch space, we're done. */
+		scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
+		return (scrspc);
+	}
+
+	p = td->td_proc;
+
+	mutex_enter(&fprc->ftpc_mtx);
+	if (LIST_EMPTY(&fprc->ftpc_fscr)) {
+		/*
+		 * No scratch space is available, so we'll map a new scratch
+		 * space block into the traced process' address space.
+		 */
+		addr = 0;
+		error = vm_map_find(&p->p_vmspace->vm_map, NULL, 0, &addr,
+		    FASTTRAP_SCRBLOCK_SIZE, 0, VMFS_ANY_SPACE, VM_PROT_ALL,
+		    VM_PROT_ALL, 0);
+		if (error != KERN_SUCCESS)
+			goto done;
+
+		scrblk = malloc(sizeof(*scrblk), M_SOLARIS, M_WAITOK);
+		scrblk->ftsb_addr = addr;
+		LIST_INSERT_HEAD(&fprc->ftpc_scrblks, scrblk, ftsb_next);
+
+		/*
+		 * Carve the block up into chunks and put them on the free list.
+		 */
+		for (i = 0;
+		    i < FASTTRAP_SCRBLOCK_SIZE / FASTTRAP_SCRSPACE_SIZE; i++) {
+			scrspc = malloc(sizeof(*scrspc), M_SOLARIS, M_WAITOK);
+			scrspc->ftss_addr = addr +
+			    i * FASTTRAP_SCRSPACE_SIZE;
+			LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc,
+			    ftss_next);
+		}
+	}
+
+	/*
+	 * Take the first scratch chunk off the free list, put it on the
+	 * allocated list, and return its address.
+	 */
+	scrspc = LIST_FIRST(&fprc->ftpc_fscr);
+	LIST_REMOVE(scrspc, ftss_next);
+	LIST_INSERT_HEAD(&fprc->ftpc_ascr, scrspc, ftss_next);
+
+	/*
+	 * This scratch space is reserved for use by td until the thread exits.
+	 */
+	td->t_dtrace_sscr = scrspc;
+
+done:
+	mutex_exit(&fprc->ftpc_mtx);
+
+	return (scrspc);
+}
+
+/*
+ * Return any allocated per-thread scratch space chunks back to the process'
+ * free list.
+ */
+static void
+fasttrap_thread_dtor(void *arg __unused, struct thread *td)
+{
+	fasttrap_bucket_t *bucket;
+	fasttrap_proc_t *fprc;
+	fasttrap_scrspace_t *scrspc;
+	pid_t pid;
+
+	if (td->t_dtrace_sscr == NULL)
+		return;
+
+	pid = td->td_proc->p_pid;
+	bucket = &fasttrap_procs.fth_table[FASTTRAP_PROCS_INDEX(pid)];
+	fprc = NULL;
+
+	/* Look up the fasttrap process handle for this process. */
+	mutex_enter(&bucket->ftb_mtx);
+	for (fprc = bucket->ftb_data; fprc != NULL; fprc = fprc->ftpc_next) {
+		if (fprc->ftpc_pid == pid) {
+			mutex_enter(&fprc->ftpc_mtx);
+			mutex_exit(&bucket->ftb_mtx);
+			break;
+		}
+	}
+	if (fprc == NULL) {
+		mutex_exit(&bucket->ftb_mtx);
+		return;
+	}
+
+	scrspc = (fasttrap_scrspace_t *)td->t_dtrace_sscr;
+	LIST_REMOVE(scrspc, ftss_next);
+	LIST_INSERT_HEAD(&fprc->ftpc_fscr, scrspc, ftss_next);
+
+	mutex_exit(&fprc->ftpc_mtx);
+}
+#endif
+
 /*
  * This function ensures that no threads are actively using the memory
  * associated with probes that were formerly live.
@@ -450,6 +572,10 @@ fasttrap_pid_cleanup(void)
 static void
 fasttrap_fork(proc_t *p, proc_t *cp)
 {
+#if !defined(sun)
+	fasttrap_scrblock_t *scrblk;
+	fasttrap_proc_t *fprc = NULL;
+#endif
 	pid_t ppid = p->p_pid;
 	int i;
 
@@ -535,9 +661,28 @@ fasttrap_fork(proc_t *p, proc_t *cp)
 				 * mid-fork.
 				 */
 				ASSERT(tp->ftt_proc->ftpc_acount != 0);
+#if !defined(sun)
+				fprc = tp->ftt_proc;
+#endif
 			}
 		}
 		mutex_exit(&bucket->ftb_mtx);
+
+#if !defined(sun)
+		/*
+		 * Unmap any scratch space inherited from the parent's address
+		 * space.
+		 */
+		if (fprc != NULL) {
+			mutex_enter(&fprc->ftpc_mtx);
+			LIST_FOREACH(scrblk, &fprc->ftpc_scrblks, ftsb_next) {
+				vm_map_remove(&cp->p_vmspace->vm_map,
+				    scrblk->ftsb_addr,
+				    scrblk->ftsb_addr + FASTTRAP_SCRBLOCK_SIZE);
+			}
+			mutex_exit(&fprc->ftpc_mtx);
+		}
+#endif
 	}
 
 #if defined(sun)
@@ -558,12 +703,24 @@ fasttrap_fork(proc_t *p, proc_t *cp)
 static void
 fasttrap_exec_exit(proc_t *p)
 {
+#if !defined(sun)
+	struct thread *td;
+#endif
+
 #if defined(sun)
 	ASSERT(p == curproc);
-#endif
+#else
 	PROC_LOCK_ASSERT(p, MA_OWNED);
 	_PHOLD(p);
+	/*
+	 * Since struct threads may be recycled, we cannot rely on t_dtrace_sscr
+	 * fields to be zeroed by kdtrace_thread_ctor. Thus we must zero it
+	 * ourselves when a process exits.
+	 */
+	FOREACH_THREAD_IN_PROC(p, td)
+		td->t_dtrace_sscr = NULL;
 	PROC_UNLOCK(p);
+#endif
 
 	/*
 	 * We clean up the pid provider for this process here; user-land
@@ -573,9 +730,9 @@ fasttrap_exec_exit(proc_t *p)
 #if !defined(sun)
 	if (p->p_dtrace_helpers)
 		dtrace_helpers_destroy(p);
-#endif
 	PROC_LOCK(p);
 	_PRELE(p);
+#endif
 }
 
 
@@ -1368,6 +1525,12 @@ fasttrap_proc_release(fasttrap_proc_t *proc)
 	fasttrap_bucket_t *bucket;
 	fasttrap_proc_t *fprc, **fprcp;
 	pid_t pid = proc->ftpc_pid;
+#if !defined(sun)
+	fasttrap_scrblock_t *scrblk, *scrblktmp;
+	fasttrap_scrspace_t *scrspc, *scrspctmp;
+	struct proc *p;
+	struct thread *td;
+#endif
 
 	mutex_enter(&proc->ftpc_mtx);
 
@@ -1379,6 +1542,31 @@ fasttrap_proc_release(fasttrap_proc_t *proc)
 		return;
 	}
 
+#if !defined(sun)
+	/*
+	 * Free all structures used to manage per-thread scratch space.
+	 */
+	LIST_FOREACH_SAFE(scrblk, &proc->ftpc_scrblks, ftsb_next,
+	    scrblktmp) {
+		LIST_REMOVE(scrblk, ftsb_next);
+		free(scrblk, M_SOLARIS);
+	}
+	LIST_FOREACH_SAFE(scrspc, &proc->ftpc_fscr, ftss_next, scrspctmp) {
+		LIST_REMOVE(scrspc, ftss_next);
+		free(scrspc, M_SOLARIS);
+	}
+	LIST_FOREACH_SAFE(scrspc, &proc->ftpc_ascr, ftss_next, scrspctmp) {
+		LIST_REMOVE(scrspc, ftss_next);
+		free(scrspc, M_SOLARIS);
+	}
+
+	if ((p = pfind(pid)) != NULL) {
+		FOREACH_THREAD_IN_PROC(p, td)
+			td->t_dtrace_sscr = NULL;
+		PROC_UNLOCK(p);
+	}
+#endif
+
 	mutex_exit(&proc->ftpc_mtx);
 
 	/*
@@ -2362,6 +2550,13 @@ fasttrap_load(void)
 		mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
 		    MUTEX_DEFAULT, NULL);
 	}
+
+	/*
+	 * This event handler must run before kdtrace_thread_dtor() since it
+	 * accesses the thread's struct kdtrace_thread.
+	 */
+	fasttrap_thread_dtor_tag = EVENTHANDLER_REGISTER(thread_dtor,
+	    fasttrap_thread_dtor, NULL, EVENTHANDLER_PRI_FIRST);
 #endif
 
 	/*
@@ -2463,6 +2658,8 @@ fasttrap_unload(void)
 #endif
 
 #if !defined(sun)
+	EVENTHANDLER_DEREGISTER(thread_dtor, fasttrap_thread_dtor_tag);
+
 	for (i = 0; i < fasttrap_tpoints.fth_nent; i++)
 		mutex_destroy(&fasttrap_tpoints.fth_table[i].ftb_mtx);
 	for (i = 0; i < fasttrap_provs.fth_nent; i++)
diff --git a/sys/cddl/contrib/opensolaris/uts/common/sys/fasttrap_impl.h b/sys/cddl/contrib/opensolaris/uts/common/sys/fasttrap_impl.h
index 6512da7..6f591fa 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/sys/fasttrap_impl.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/sys/fasttrap_impl.h
@@ -32,6 +32,7 @@
 #include <sys/types.h>
 #include <sys/dtrace.h>
 #include <sys/proc.h>
+#include <sys/queue.h>
 #include <sys/fasttrap.h>
 #include <sys/fasttrap_isa.h>
 
@@ -68,14 +69,41 @@ extern "C" {
  * then disabled, ownership of that tracepoint may be exchanged for an
  * unused tracepoint belonging to another probe that was attached to the
  * enabled tracepoint.
+ *
+ * On FreeBSD, fasttrap providers also maintain per-thread scratch space for use
+ * by the ISA-specific fasttrap code. The fasttrap_scrblock_t type stores the
+ * virtual address of a page-sized memory block that is mapped into a process'
+ * address space. Each block is carved up into chunks (fasttrap_scrspace_t) for
+ * use by individual threads, which keep the address of their scratch space
+ * chunk in their struct kdtrace_thread. A thread's scratch space isn't released
+ * until it exits.
  */
 
+#if !defined(sun)
+typedef struct fasttrap_scrblock {
+	vm_offset_t ftsb_addr;			/* address of a scratch block */
+	LIST_ENTRY(fasttrap_scrblock) ftsb_next;/* next block in list */
+} fasttrap_scrblock_t;
+#define	FASTTRAP_SCRBLOCK_SIZE	PAGE_SIZE
+
+typedef struct fasttrap_scrspace {
+	uintptr_t ftss_addr;			/* scratch space address */
+	LIST_ENTRY(fasttrap_scrspace) ftss_next;/* next in list */
+} fasttrap_scrspace_t;
+#define	FASTTRAP_SCRSPACE_SIZE	64
+#endif
+
 typedef struct fasttrap_proc {
 	pid_t ftpc_pid;				/* process ID for this proc */
 	uint64_t ftpc_acount;			/* count of active providers */
 	uint64_t ftpc_rcount;			/* count of extant providers */
 	kmutex_t ftpc_mtx;			/* lock on all but acount */
 	struct fasttrap_proc *ftpc_next;	/* next proc in hash chain */
+#if !defined(sun)
+	LIST_HEAD(, fasttrap_scrblock) ftpc_scrblks; /* mapped scratch blocks */
+	LIST_HEAD(, fasttrap_scrspace) ftpc_fscr; /* free scratch space */
+	LIST_HEAD(, fasttrap_scrspace) ftpc_ascr; /* used scratch space */
+#endif
 } fasttrap_proc_t;
 
 typedef struct fasttrap_provider {
@@ -170,6 +198,10 @@ typedef struct fasttrap_hash {
 #endif
 
 extern void fasttrap_sigtrap(proc_t *, kthread_t *, uintptr_t);
+#if !defined(sun)
+extern fasttrap_scrspace_t *fasttrap_scraddr(struct thread *,
+    fasttrap_proc_t *);
+#endif
 
 extern dtrace_id_t 		fasttrap_probe_id;
 extern fasttrap_hash_t		fasttrap_tpoints;
diff --git a/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c b/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c
index d65f1fd..22dbda4 100644
--- a/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c
+++ b/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c
@@ -1526,7 +1526,6 @@ fasttrap_pid_probe(struct reg *rp)
 		uint_t i = 0;
 #if defined(sun)
 		klwp_t *lwp = ttolwp(curthread);
-#endif
 
 		/*
 		 * Compute the address of the ulwp_t and step over the
@@ -1534,7 +1533,6 @@ fasttrap_pid_probe(struct reg *rp)
 		 * thread pointer is very different on 32- and 64-bit
 		 * kernels.
 		 */
-#if defined(sun)
 #if defined(__amd64)
 		if (p->p_model == DATAMODEL_LP64) {
 			addr = lwp->lwp_pcb.pcb_fsbase;
@@ -1547,13 +1545,23 @@ fasttrap_pid_probe(struct reg *rp)
 		addr = USD_GETBASE(&lwp->lwp_pcb.pcb_gsdesc);
 		addr += sizeof (void *);
 #endif
-#endif /* sun */
-#ifdef __i386__
-		addr = USD_GETBASE(&curthread->td_pcb->pcb_gsd);
 #else
-		addr = curthread->td_pcb->pcb_gsbase;
-#endif
-		addr += sizeof (void *);
+		fasttrap_scrspace_t *scrspace;
+		scrspace = fasttrap_scraddr(curthread, tp->ftt_proc);
+		if (scrspace == NULL) {
+			/*
+			 * We failed to allocate scratch space for this thread.
+			 * Try to write the original instruction back out and
+			 * reset the pc.
+			 */
+			if (fasttrap_copyout(tp->ftt_instr, (void *)pc,
+			    tp->ftt_size))
+				fasttrap_sigtrap(p, curthread, pc);
+			new_pc = pc;
+			break;
+		}
+		addr = scrspace->ftss_addr;
+#endif /* sun */
 
 		/*
 		 * Generic Instruction Tracing
diff --git a/sys/cddl/dev/dtrace/dtrace_cddl.h b/sys/cddl/dev/dtrace/dtrace_cddl.h
index 6caf3cf..eed32a8 100644
--- a/sys/cddl/dev/dtrace/dtrace_cddl.h
+++ b/sys/cddl/dev/dtrace/dtrace_cddl.h
@@ -84,6 +84,7 @@ typedef struct kdtrace_thread {
 #endif
 	u_int64_t	td_hrtime;	/* Last time on cpu. */
 	int		td_errno;	/* Syscall return value. */
+	void		*td_dtrace_sscr; /* Saved scratch space location. */
 } kdtrace_thread_t;
 
 /*
@@ -108,10 +109,12 @@ typedef struct kdtrace_thread {
 #define	t_dtrace_scrpc	td_dtrace->td_dtrace_scrpc
 #define	t_dtrace_astpc	td_dtrace->td_dtrace_astpc
 #define	t_dtrace_regv	td_dtrace->td_dtrace_regv
+#define	t_dtrace_sscr	td_dtrace->td_dtrace_sscr
 #define	p_dtrace_helpers	p_dtrace->p_dtrace_helpers
 #define	p_dtrace_count	p_dtrace->p_dtrace_count
 #define	p_dtrace_probes	p_dtrace->p_dtrace_probes
 #define	p_model		p_dtrace->p_dtrace_model
+
 #define	DATAMODEL_NATIVE	0
 #ifdef __amd64__
 #define	DATAMODEL_LP64		0