Fix fasttrap tracepoint locking.

Upstream, tracepoints are protected by per-CPU mutexes. An unlinked
tracepoint may be freed once all the tracepoint mutexes have been acquired
and released - this is done in fasttrap_mod_barrier(). This mechanism was
not properly ported: in some places, the proc lock is used in place of a
tracepoint lock, and in others the locking is omitted entirely. This change
implements tracepoint locking with an rmlock, where the read lock is used
in fasttrap probe context. As a side effect, this fixes a recursion on the
proc lock when the raise action is used from a userland probe.

MFC after:	1 month

4 files changed, 37 insertions, 20 deletions

diff --git a/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c b/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c
index 758ce0b..51db19b 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c
+++ b/sys/cddl/contrib/opensolaris/uts/common/dtrace/fasttrap.c
@@ -63,13 +63,16 @@
 #ifndef illumos
 #include <sys/dtrace_bsd.h>
 #include <sys/eventhandler.h>
+#include <sys/rmlock.h>
 #include <sys/sysctl.h>
 #include <sys/u8_textprep.h>
 #include <sys/user.h>
+
 #include <vm/vm.h>
 #include <vm/pmap.h>
 #include <vm/vm_map.h>
 #include <vm/vm_param.h>
+
 #include <cddl/dev/dtrace/dtrace_cddl.h>
 #endif
 
@@ -224,7 +227,7 @@ static void fasttrap_thread_dtor(void *, struct thread *);
 #define	FASTTRAP_PROCS_INDEX(pid) ((pid) & fasttrap_procs.fth_mask)
 
 #ifndef illumos
-static kmutex_t fasttrap_cpuc_pid_lock[MAXCPU];
+struct rmlock fasttrap_tp_lock;
 static eventhandler_tag fasttrap_thread_dtor_tag;
 #endif
 
@@ -439,10 +442,15 @@ fasttrap_mod_barrier(uint64_t gen)
 
 	fasttrap_mod_gen++;
 
+#ifdef illumos
 	CPU_FOREACH(i) {
 		mutex_enter(&fasttrap_cpuc_pid_lock[i]);
 		mutex_exit(&fasttrap_cpuc_pid_lock[i]);
 	}
+#else
+	rm_wlock(&fasttrap_tp_lock);
+	rm_wunlock(&fasttrap_tp_lock);
+#endif
 }
 
 /*
@@ -2574,10 +2582,7 @@ fasttrap_load(void)
 		mutex_init(&fasttrap_procs.fth_table[i].ftb_mtx,
 		    "processes bucket mtx", MUTEX_DEFAULT, NULL);
 
-	CPU_FOREACH(i) {
-		mutex_init(&fasttrap_cpuc_pid_lock[i], "fasttrap barrier",
-		    MUTEX_DEFAULT, NULL);
-	}
+	rm_init(&fasttrap_tp_lock, "fasttrap tracepoint");
 
 	/*
 	 * This event handler must run before kdtrace_thread_dtor() since it
@@ -2710,9 +2715,7 @@ fasttrap_unload(void)
 #ifndef illumos
 	destroy_dev(fasttrap_cdev);
 	mutex_destroy(&fasttrap_count_mtx);
-	CPU_FOREACH(i) {
-		mutex_destroy(&fasttrap_cpuc_pid_lock[i]);
-	}
+	rm_destroy(&fasttrap_tp_lock);
 #endif
 
 	return (0);
diff --git a/sys/cddl/contrib/opensolaris/uts/common/sys/fasttrap_impl.h b/sys/cddl/contrib/opensolaris/uts/common/sys/fasttrap_impl.h
index cae9193..c104e79 100644
--- a/sys/cddl/contrib/opensolaris/uts/common/sys/fasttrap_impl.h
+++ b/sys/cddl/contrib/opensolaris/uts/common/sys/fasttrap_impl.h
@@ -206,6 +206,10 @@ extern fasttrap_scrspace_t *fasttrap_scraddr(struct thread *,
 extern dtrace_id_t 		fasttrap_probe_id;
 extern fasttrap_hash_t		fasttrap_tpoints;
 
+#ifndef illumos
+extern struct rmlock		fasttrap_tp_lock;
+#endif
+
 #define	FASTTRAP_TPOINTS_INDEX(pid, pc) \
 	(((pc) / sizeof (fasttrap_instr_t) + (pid)) & fasttrap_tpoints.fth_mask)
 
diff --git a/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c b/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c
index 4c9fc0f..5b3f485 100644
--- a/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c
+++ b/sys/cddl/contrib/opensolaris/uts/intel/dtrace/fasttrap_isa.c
@@ -46,6 +46,7 @@
 #include <cddl/dev/dtrace/dtrace_cddl.h>
 #include <sys/types.h>
 #include <sys/proc.h>
+#include <sys/rmlock.h>
 #include <sys/dtrace_bsd.h>
 #include <cddl/dev/dtrace/x86/regset.h>
 #include <machine/segments.h>
@@ -737,11 +738,13 @@ fasttrap_return_common(struct reg *rp, uintptr_t pc, pid_t pid,
 	fasttrap_id_t *id;
 #ifdef illumos
 	kmutex_t *pid_mtx;
-#endif
 
-#ifdef illumos
 	pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock;
 	mutex_enter(pid_mtx);
+#else
+	struct rm_priotracker tracker;
+
+	rm_rlock(&fasttrap_tp_lock, &tracker);
 #endif
 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
 
@@ -759,6 +762,8 @@ fasttrap_return_common(struct reg *rp, uintptr_t pc, pid_t pid,
 	if (tp == NULL) {
 #ifdef illumos
 		mutex_exit(pid_mtx);
+#else
+		rm_runlock(&fasttrap_tp_lock, &tracker);
 #endif
 		return;
 	}
@@ -782,6 +787,8 @@ fasttrap_return_common(struct reg *rp, uintptr_t pc, pid_t pid,
 
 #ifdef illumos
 	mutex_exit(pid_mtx);
+#else
+	rm_runlock(&fasttrap_tp_lock, &tracker);
 #endif
 }
 
@@ -990,6 +997,7 @@ fasttrap_pid_probe(struct reg *rp)
 {
 	proc_t *p = curproc;
 #ifndef illumos
+	struct rm_priotracker tracker;
 	proc_t *pp;
 #endif
 	uintptr_t pc = rp->r_rip - 1;
@@ -1049,8 +1057,7 @@ fasttrap_pid_probe(struct reg *rp)
 	sx_sunlock(&proctree_lock);
 	pp = NULL;
 
-	PROC_LOCK(p);
-	_PHOLD(p);
+	rm_rlock(&fasttrap_tp_lock, &tracker);
 #endif
 
 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
@@ -1073,8 +1080,7 @@ fasttrap_pid_probe(struct reg *rp)
 #ifdef illumos
 		mutex_exit(pid_mtx);
 #else
-		_PRELE(p);
-		PROC_UNLOCK(p);
+		rm_runlock(&fasttrap_tp_lock, &tracker);
 #endif
 		return (-1);
 	}
@@ -1200,7 +1206,7 @@ fasttrap_pid_probe(struct reg *rp)
 #ifdef illumos
 	mutex_exit(pid_mtx);
 #else
-	PROC_UNLOCK(p);
+	rm_runlock(&fasttrap_tp_lock, &tracker);
 #endif
 	tp = &tp_local;
 
@@ -1813,7 +1819,6 @@ done:
 #ifndef illumos
 	PROC_LOCK(p);
 	proc_write_regs(curthread, rp);
-	_PRELE(p);
 	PROC_UNLOCK(p);
 #endif
 
diff --git a/sys/cddl/contrib/opensolaris/uts/powerpc/dtrace/fasttrap_isa.c b/sys/cddl/contrib/opensolaris/uts/powerpc/dtrace/fasttrap_isa.c
index 8e4c647..e8dd684 100644
--- a/sys/cddl/contrib/opensolaris/uts/powerpc/dtrace/fasttrap_isa.c
+++ b/sys/cddl/contrib/opensolaris/uts/powerpc/dtrace/fasttrap_isa.c
@@ -33,6 +33,7 @@
 #include <sys/types.h>
 #include <sys/uio.h>
 #include <sys/ptrace.h>
+#include <sys/rmlock.h>
 #include <sys/sysent.h>
 
 #define OP(x)	((x) >> 26)
@@ -288,10 +289,12 @@ static void
 fasttrap_return_common(struct reg *rp, uintptr_t pc, pid_t pid,
     uintptr_t new_pc)
 {
+	struct rm_priotracker tracker;
 	fasttrap_tracepoint_t *tp;
 	fasttrap_bucket_t *bucket;
 	fasttrap_id_t *id;
 
+	rm_rlock(&fasttrap_tp_lock, &tracker);
 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
 
 	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
@@ -306,6 +309,7 @@ fasttrap_return_common(struct reg *rp, uintptr_t pc, pid_t pid,
 	 * is not essential to the correct execution of the process.
 	 */
 	if (tp == NULL) {
+		rm_runlock(&fasttrap_tp_lock, &tracker);
 		return;
 	}
 
@@ -323,6 +327,7 @@ fasttrap_return_common(struct reg *rp, uintptr_t pc, pid_t pid,
 		    pc - id->fti_probe->ftp_faddr,
 		    rp->fixreg[3], rp->fixreg[4], 0, 0);
 	}
+	rm_runlock(&fasttrap_tp_lock, &tracker);
 }
 
 
@@ -351,6 +356,7 @@ fasttrap_branch_taken(int bo, int bi, struct reg *regs)
 int
 fasttrap_pid_probe(struct reg *rp)
 {
+	struct rm_priotracker tracker;
 	proc_t *p = curproc;
 	uintptr_t pc = rp->pc;
 	uintptr_t new_pc = 0;
@@ -381,8 +387,7 @@ fasttrap_pid_probe(struct reg *rp)
 	curthread->t_dtrace_scrpc = 0;
 	curthread->t_dtrace_astpc = 0;
 
-
-	PROC_LOCK(p);
+	rm_rlock(&fasttrap_tp_lock, &tracker);
 	pid = p->p_pid;
 	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
 
@@ -401,7 +406,7 @@ fasttrap_pid_probe(struct reg *rp)
 	 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
 	 */
 	if (tp == NULL) {
-		PROC_UNLOCK(p);
+		rm_runlock(&fasttrap_tp_lock, &tracker);
 		return (-1);
 	}
 
@@ -455,7 +460,7 @@ fasttrap_pid_probe(struct reg *rp)
 	 * tracepoint again later if we need to light up any return probes.
 	 */
 	tp_local = *tp;
-	PROC_UNLOCK(p);
+	rm_runlock(&fasttrap_tp_lock, &tracker);
 	tp = &tp_local;
 
 	/*