MFC r256543,r259245,r259421,r259668,r259674

r256543:

Add fasttrap for PowerPC.  This is the last piece of the DTrace/ppc puzzle.
It's incomplete, it doesn't contain full instruction emulation, but it should be
sufficient for most cases.

r259245,r259421: (FBT)

FBT now does work fully on PowerPC.

Save r3 before using it for the trap check, else we end up saving the new r3,
containing the trap instruction encoding (0x7c810808), and restoring it back
with the frame on return.  This caused it to panic on my ppc32 machine.

r259668,r259674:
Fix a typo in the FBT code.

10 files changed, 848 insertions, 32 deletions

diff --git a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_link.c b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_link.c
index 2d0428a..ee68479 100644
--- a/cddl/contrib/opensolaris/lib/libdtrace/common/dt_link.c
+++ b/cddl/contrib/opensolaris/lib/libdtrace/common/dt_link.c
@@ -242,8 +242,14 @@ printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 /* XXX */
 printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 #elif defined(__powerpc__)
-/* XXX */
-printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
+			/*
+			 * Add 4 bytes to hit the low half of this 64-bit
+			 * big-endian address.
+			 */
+			rel->r_offset = s->dofs_offset +
+			    dofr[j].dofr_offset + 4;
+			rel->r_info = ELF32_R_INFO(count + dep->de_global,
+			    R_PPC_REL32);
 #elif defined(__sparc)
 			/*
 			 * Add 4 bytes to hit the low half of this 64-bit
@@ -423,7 +429,10 @@ prepare_elf64(dtrace_hdl_t *dtp, const dof_hdr_t *dof, dof_elf64_t *dep)
 #elif defined(__mips__)
 /* XXX */
 #elif defined(__powerpc__)
-/* XXX */
+			rel->r_offset = s->dofs_offset +
+			    dofr[j].dofr_offset;
+			rel->r_info = ELF64_R_INFO(count + dep->de_global,
+			    R_PPC64_REL64);
 #elif defined(__i386) || defined(__amd64)
 			rel->r_offset = s->dofs_offset +
 			    dofr[j].dofr_offset;
@@ -824,12 +833,84 @@ printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
 	return (0);
 }
 #elif defined(__powerpc__)
+/* The sentinel is 'xor r3,r3,r3'. */
+#define DT_OP_XOR_R3	0x7c631a78
+
+#define DT_OP_NOP		0x60000000
+#define DT_OP_BLR		0x4e800020
+
+/* This captures all forms of branching to address. */
+#define DT_IS_BRANCH(inst)	((inst & 0xfc000000) == 0x48000000)
+#define DT_IS_BL(inst)	(DT_IS_BRANCH(inst) && (inst & 0x01))
+
 /* XXX */
 static int
 dt_modtext(dtrace_hdl_t *dtp, char *p, int isenabled, GElf_Rela *rela,
     uint32_t *off)
 {
-printf("%s:%s(%d): DOODAD\n",__FUNCTION__,__FILE__,__LINE__);
+	uint32_t *ip;
+
+	if ((rela->r_offset & (sizeof (uint32_t) - 1)) != 0)
+		return (-1);
+
+	/*LINTED*/
+	ip = (uint32_t *)(p + rela->r_offset);
+
+	/*
+	 * We only know about some specific relocation types.
+	 */
+	if (GELF_R_TYPE(rela->r_info) != R_PPC_REL24 &&
+	    GELF_R_TYPE(rela->r_info) != R_PPC_PLTREL24)
+		return (-1);
+
+	/*
+	 * We may have already processed this object file in an earlier linker
+	 * invocation. Check to see if the present instruction sequence matches
+	 * the one we would install below.
+	 */
+	if (isenabled) {
+		if (ip[0] == DT_OP_XOR_R3) {
+			(*off) += sizeof (ip[0]);
+			return (0);
+		}
+	} else {
+		if (ip[0] == DT_OP_NOP) {
+			(*off) += sizeof (ip[0]);
+			return (0);
+		}
+	}
+
+	/*
+	 * We only expect branch to address instructions.
+	 */
+	if (!DT_IS_BRANCH(ip[0])) {
+		dt_dprintf("found %x instead of a branch instruction at %llx\n",
+		    ip[0], (u_longlong_t)rela->r_offset);
+		return (-1);
+	}
+
+	if (isenabled) {
+		/*
+		 * It would necessarily indicate incorrect usage if an is-
+		 * enabled probe were tail-called so flag that as an error.
+		 * It's also potentially (very) tricky to handle gracefully,
+		 * but could be done if this were a desired use scenario.
+		 */
+		if (!DT_IS_BL(ip[0])) {
+			dt_dprintf("tail call to is-enabled probe at %llx\n",
+			    (u_longlong_t)rela->r_offset);
+			return (-1);
+		}
+
+		ip[0] = DT_OP_XOR_R3;
+		(*off) += sizeof (ip[0]);
+	} else {
+		if (DT_IS_BL(ip[0]))
+			ip[0] = DT_OP_NOP;
+		else
+			ip[0] = DT_OP_BLR;
+	}
+
 	return (0);
 }
 
diff --git a/cddl/contrib/opensolaris/lib/libdtrace/powerpc/dt_isadep.c b/cddl/contrib/opensolaris/lib/libdtrace/powerpc/dt_isadep.c
index 1aeb95f..f4b02c9 100644
--- a/cddl/contrib/opensolaris/lib/libdtrace/powerpc/dt_isadep.c
+++ b/cddl/contrib/opensolaris/lib/libdtrace/powerpc/dt_isadep.c
@@ -35,14 +35,26 @@
 #include <dt_impl.h>
 #include <dt_pid.h>
 
+#include <libproc_compat.h>
+
 /*ARGSUSED*/
 int
 dt_pid_create_entry_probe(struct ps_prochandle *P, dtrace_hdl_t *dtp,
     fasttrap_probe_spec_t *ftp, const GElf_Sym *symp)
 {
+	ftp->ftps_type = DTFTP_ENTRY;
+	ftp->ftps_pc = (uintptr_t)symp->st_value;
+	ftp->ftps_size = (size_t)symp->st_size;
+	ftp->ftps_noffs = 1;
+	ftp->ftps_offs[0] = 0;
+
+	if (ioctl(dtp->dt_ftfd, FASTTRAPIOC_MAKEPROBE, ftp) != 0) {
+		dt_dprintf("fasttrap probe creation ioctl failed: %s\n",
+		    strerror(errno));
+		return (dt_set_errno(dtp, errno));
+	}
 
-	dt_dprintf("%s: unimplemented\n", __func__);
-	return (DT_PROC_ERR);
+	return (1);
 }
 
 int
@@ -50,8 +62,74 @@ dt_pid_create_return_probe(struct ps_prochandle *P, dtrace_hdl_t *dtp,
     fasttrap_probe_spec_t *ftp, const GElf_Sym *symp, uint64_t *stret)
 {
 
-	dt_dprintf("%s: unimplemented\n", __func__);
-	return (DT_PROC_ERR);
+	uintptr_t temp;
+	uint32_t *text;
+	int i;
+	int srdepth = 0;
+
+	if ((text = malloc(symp->st_size + 4)) == NULL) {
+		dt_dprintf("mr sparkle: malloc() failed\n");
+		return (DT_PROC_ERR);
+	}
+
+	if (Pread(P, text, symp->st_size, symp->st_value) != symp->st_size) {
+		dt_dprintf("mr sparkle: Pread() failed\n");
+		free(text);
+		return (DT_PROC_ERR);
+	}
+
+	/*
+	 * Leave a dummy instruction in the last slot to simplify edge
+	 * conditions.
+	 */
+	text[symp->st_size / 4] = 0;
+
+	ftp->ftps_type = DTFTP_RETURN;
+	ftp->ftps_pc = symp->st_value;
+	ftp->ftps_size = symp->st_size;
+	ftp->ftps_noffs = 0;
+
+	for (i = 0; i < symp->st_size / 4; i++) {
+
+		if ((text[i] & 0xfc000001) != 0x48000000 &&
+		    text[i] != 0x4e800020)
+			continue;
+
+		/*
+		 * Check for a jump within this function.  If it's outside this
+		 * function then it's a tail-call, so a return point.
+		 */
+		if ((text[i] & 0xfc000000) == 0x48000000) {
+			temp = (text[i] & 0x03fffffc);
+			/* Bit 30 denotes an absolute address. */
+			if (!(text[i] & 0x02)) {
+				temp += symp->st_value + i * 4;
+			}
+			else {
+				/* Sign extend the absolute address. */
+				if (temp & 0x02000000) {
+					temp |= (UINTPTR_MAX - 0x03ffffff);
+				}
+			}
+			if (temp >= symp->st_value &&
+			    temp <= (symp->st_value + symp->st_size))
+				continue;
+		}
+		dt_dprintf("return at offset %x\n", i * 4);
+		ftp->ftps_offs[ftp->ftps_noffs++] = i * 4;
+	}
+
+	free(text);
+	if (ftp->ftps_noffs > 0) {
+		if (ioctl(dtp->dt_ftfd, FASTTRAPIOC_MAKEPROBE, ftp) != 0) {
+			dt_dprintf("fasttrap probe creation ioctl failed: %s\n",
+			    strerror(errno));
+			return (dt_set_errno(dtp, errno));
+		}
+	}
+
+
+	return (ftp->ftps_noffs);
 }
 
 /*ARGSUSED*/
@@ -59,9 +137,22 @@ int
 dt_pid_create_offset_probe(struct ps_prochandle *P, dtrace_hdl_t *dtp,
     fasttrap_probe_spec_t *ftp, const GElf_Sym *symp, ulong_t off)
 {
+	if (off & 0x3)
+		return (DT_PROC_ALIGN);
+
+	ftp->ftps_type = DTFTP_OFFSETS;
+	ftp->ftps_pc = (uintptr_t)symp->st_value;
+	ftp->ftps_size = (size_t)symp->st_size;
+	ftp->ftps_noffs = 1;
+	ftp->ftps_offs[0] = off;
+
+	if (ioctl(dtp->dt_ftfd, FASTTRAPIOC_MAKEPROBE, ftp) != 0) {
+		dt_dprintf("fasttrap probe creation ioctl failed: %s\n",
+		    strerror(errno));
+		return (dt_set_errno(dtp, errno));
+	}
 
-	dt_dprintf("%s: unimplemented\n", __func__);
-	return (DT_PROC_ERR);
+	return (1);
 }
 
 /*ARGSUSED*/
@@ -69,7 +160,38 @@ int
 dt_pid_create_glob_offset_probes(struct ps_prochandle *P, dtrace_hdl_t *dtp,
     fasttrap_probe_spec_t *ftp, const GElf_Sym *symp, const char *pattern)
 {
+	ulong_t i;
+
+	ftp->ftps_type = DTFTP_OFFSETS;
+	ftp->ftps_pc = (uintptr_t)symp->st_value;
+	ftp->ftps_size = (size_t)symp->st_size;
+	ftp->ftps_noffs = 0;
+
+	/*
+	 * If we're matching against everything, just iterate through each
+	 * instruction in the function, otherwise look for matching offset
+	 * names by constructing the string and comparing it against the
+	 * pattern.
+	 */
+	if (strcmp("*", pattern) == 0) {
+		for (i = 0; i < symp->st_size; i += 4) {
+			ftp->ftps_offs[ftp->ftps_noffs++] = i;
+		}
+	} else {
+		char name[sizeof (i) * 2 + 1];
+
+		for (i = 0; i < symp->st_size; i += 4) {
+			(void) sprintf(name, "%lx", i);
+			if (gmatch(name, pattern))
+				ftp->ftps_offs[ftp->ftps_noffs++] = i;
+		}
+	}
+
+	if (ioctl(dtp->dt_ftfd, FASTTRAPIOC_MAKEPROBE, ftp) != 0) {
+		dt_dprintf("fasttrap probe creation ioctl failed: %s\n",
+		    strerror(errno));
+		return (dt_set_errno(dtp, errno));
+	}
 
-	dt_dprintf("%s: unimplemented\n", __func__);
-	return (DT_PROC_ERR);
+	return (ftp->ftps_noffs);
 }
diff --git a/sys/cddl/contrib/opensolaris/uts/powerpc/dtrace/fasttrap_isa.c b/sys/cddl/contrib/opensolaris/uts/powerpc/dtrace/fasttrap_isa.c
index 36115ea..c035c99 100644
--- a/sys/cddl/contrib/opensolaris/uts/powerpc/dtrace/fasttrap_isa.c
+++ b/sys/cddl/contrib/opensolaris/uts/powerpc/dtrace/fasttrap_isa.c
@@ -18,13 +18,560 @@
  *
  * CDDL HEADER END
  */
-
+/* Portions Copyright 2013 Justin Hibbits */
 /*
  * Copyright 2007 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
  */
 
+#include <sys/fasttrap_isa.h>
+#include <sys/fasttrap_impl.h>
+#include <sys/dtrace.h>
+#include <sys/dtrace_impl.h>
+#include <cddl/dev/dtrace/dtrace_cddl.h>
+#include <sys/proc.h>
+#include <sys/types.h>
+#include <sys/uio.h>
+#include <sys/ptrace.h>
+#include <sys/sysent.h>
+
+#define OP(x)	((x) >> 26)
+#define OPX(x)	(((x) >> 2) & 0x3FF)
+#define OP_BO(x) (((x) & 0x03E00000) >> 21)
+#define OP_BI(x) (((x) & 0x001F0000) >> 16)
+#define OP_RS(x) (((x) & 0x03E00000) >> 21)
+#define OP_RA(x) (((x) & 0x001F0000) >> 16)
+#define OP_RB(x) (((x) & 0x0000F100) >> 11)
+
+
+static int
+proc_ops(int op, proc_t *p, void *kaddr, off_t uaddr, size_t len)
+{
+	struct iovec iov;
+	struct uio uio;
+
+	iov.iov_base = kaddr;
+	iov.iov_len = len;
+	uio.uio_offset = uaddr;
+	uio.uio_iov = &iov;
+	uio.uio_resid = len;
+	uio.uio_iovcnt = 1;
+	uio.uio_segflg = UIO_SYSSPACE;
+	uio.uio_td = curthread;
+	uio.uio_rw = op;
+	PHOLD(p);
+	if (proc_rwmem(p, &uio) < 0) {
+		PRELE(p);
+		return (-1);
+	}
+	PRELE(p);
+
+	return (0);
+}
+
+static int
+uread(proc_t *p, void *kaddr, size_t len, uintptr_t uaddr)
+{
+
+	return (proc_ops(UIO_READ, p, kaddr, uaddr, len));
+}
+
+static int
+uwrite(proc_t *p, void *kaddr, size_t len, uintptr_t uaddr)
+{
+
+	return (proc_ops(UIO_WRITE, p, kaddr, uaddr, len));
+}
+
+int
+fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp)
+{
+	fasttrap_instr_t instr = FASTTRAP_INSTR;
+
+	if (uwrite(p, &instr, 4, tp->ftt_pc) != 0)
+		return (-1);
+
+	return (0);
+}
+
+int
+fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp)
+{
+	uint32_t instr;
+
+	/*
+	 * Distinguish between read or write failures and a changed
+	 * instruction.
+	 */
+	if (uread(p, &instr, 4, tp->ftt_pc) != 0)
+		return (0);
+	if (instr != FASTTRAP_INSTR)
+		return (0);
+	if (uwrite(p, &tp->ftt_instr, 4, tp->ftt_pc) != 0)
+		return (-1);
+
+	return (0);
+}
+
+int
+fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, uintptr_t pc,
+    fasttrap_probe_type_t type)
+{
+	uint32_t instr;
+	//int32_t disp;
+
+	/*
+	 * Read the instruction at the given address out of the process's
+	 * address space. We don't have to worry about a debugger
+	 * changing this instruction before we overwrite it with our trap
+	 * instruction since P_PR_LOCK is set.
+	 */
+	if (uread(p, &instr, 4, pc) != 0)
+		return (-1);
+
+	/*
+	 * Decode the instruction to fill in the probe flags. We can have
+	 * the process execute most instructions on its own using a pc/npc
+	 * trick, but pc-relative control transfer present a problem since
+	 * we're relocating the instruction. We emulate these instructions
+	 * in the kernel. We assume a default type and over-write that as
+	 * needed.
+	 *
+	 * pc-relative instructions must be emulated for correctness;
+	 * other instructions (which represent a large set of commonly traced
+	 * instructions) are emulated or otherwise optimized for performance.
+	 */
+	tp->ftt_type = FASTTRAP_T_COMMON;
+	tp->ftt_instr = instr;
+
+	switch (OP(instr)) {
+	/* The following are invalid for trapping (invalid opcodes, tw/twi). */
+	case 0:
+	case 1:
+	case 2:
+	case 4:
+	case 5:
+	case 6:
+	case 30:
+	case 39:
+	case 58:
+	case 62:
+	case 3:	/* twi */
+		return (-1);
+	case 31:	/* tw */
+		if (OPX(instr) == 4)
+			return (-1);
+		else if (OPX(instr) == 444 && OP_RS(instr) == OP_RA(instr) &&
+		    OP_RS(instr) == OP_RB(instr))
+			tp->ftt_type = FASTTRAP_T_NOP;
+		break;
+	case 16:
+		tp->ftt_type = FASTTRAP_T_BC;
+		tp->ftt_dest = instr & 0x0000FFFC; /* Extract target address */
+		if (instr & 0x00008000)
+			tp->ftt_dest |= 0xFFFF0000;
+		/* Use as offset if not absolute address. */
+		if (!(instr & 0x02))
+			tp->ftt_dest += pc;
+		tp->ftt_bo = OP_BO(instr);
+		tp->ftt_bi = OP_BI(instr);
+		break;
+	case 18:
+		tp->ftt_type = FASTTRAP_T_B;
+		tp->ftt_dest = instr & 0x03FFFFFC; /* Extract target address */
+		if (instr & 0x02000000)
+			tp->ftt_dest |= 0xFC000000;
+		/* Use as offset if not absolute address. */
+		if (!(instr & 0x02))
+			tp->ftt_dest += pc;
+		break;
+	case 19:
+		switch (OPX(instr)) {
+		case 528:	/* bcctr */
+			tp->ftt_type = FASTTRAP_T_BCTR;
+			tp->ftt_bo = OP_BO(instr);
+			tp->ftt_bi = OP_BI(instr);
+			break;
+		case 16:	/* bclr */
+			tp->ftt_type = FASTTRAP_T_BCTR;
+			tp->ftt_bo = OP_BO(instr);
+			tp->ftt_bi = OP_BI(instr);
+			break;
+		};
+		break;
+	case 24:
+		if (OP_RS(instr) == OP_RA(instr) &&
+		    (instr & 0x0000FFFF) == 0)
+			tp->ftt_type = FASTTRAP_T_NOP;
+		break;
+	};
+
+	/*
+	 * We don't know how this tracepoint is going to be used, but in case
+	 * it's used as part of a function return probe, we need to indicate
+	 * whether it's always a return site or only potentially a return
+	 * site. If it's part of a return probe, it's always going to be a
+	 * return from that function if it's a restore instruction or if
+	 * the previous instruction was a return. If we could reliably
+	 * distinguish jump tables from return sites, this wouldn't be
+	 * necessary.
+	 */
+#if 0
+	if (tp->ftt_type != FASTTRAP_T_RESTORE &&
+	    (uread(p, &instr, 4, pc - sizeof (instr)) != 0 ||
+	    !(OP(instr) == 2 && OP3(instr) == OP3_RETURN)))
+		tp->ftt_flags |= FASTTRAP_F_RETMAYBE;
+#endif
+
+	return (0);
+}
+
+static uint64_t
+fasttrap_anarg(struct reg *rp, int argno)
+{
+	uint64_t value;
+	proc_t  *p = curproc;
+
+	/* The first 8 arguments are in registers. */
+	if (argno < 8)
+		return rp->fixreg[argno + 3];
+
+	/* Arguments on stack start after SP+LR (2 register slots). */
+	if (SV_PROC_FLAG(p, SV_ILP32)) {
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+		value = dtrace_fuword32((void *)(rp->fixreg[1] + 8 +
+		    ((argno - 8) * sizeof(uint32_t))));
+		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR);
+	} else {
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+		value = dtrace_fuword64((void *)(rp->fixreg[1] + 16 +
+		    ((argno - 8) * sizeof(uint32_t))));
+		DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR);
+	}
+	return value;
+}
+
+uint64_t
+fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
+    int aframes)
+{
+	struct reg r;
+
+	fill_regs(curthread, &r);
+
+	return (fasttrap_anarg(&r, argno));
+}
+
+uint64_t
+fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno,
+    int aframes)
+{
+	struct reg r;
+
+	fill_regs(curthread, &r);
+
+	return (fasttrap_anarg(&r, argno));
+}
+
+static void
+fasttrap_usdt_args(fasttrap_probe_t *probe, struct reg *rp, int argc,
+    uintptr_t *argv)
+{
+	int i, x, cap = MIN(argc, probe->ftp_nargs);
+
+	for (i = 0; i < cap; i++) {
+		x = probe->ftp_argmap[i];
+
+		if (x < 8)
+			argv[i] = rp->fixreg[x];
+		else
+			if (SV_PROC_FLAG(curproc, SV_ILP32))
+				argv[i] = fuword32((void *)(rp->fixreg[1] + 8 +
+				    (x * sizeof(uint32_t))));
+			else
+				argv[i] = fuword32((void *)(rp->fixreg[1] + 16 +
+				    (x * sizeof(uint64_t))));
+	}
+
+	for (; i < argc; i++) {
+		argv[i] = 0;
+	}
+}
+
+static void
+fasttrap_return_common(struct reg *rp, uintptr_t pc, pid_t pid,
+    uintptr_t new_pc)
+{
+	fasttrap_tracepoint_t *tp;
+	fasttrap_bucket_t *bucket;
+	fasttrap_id_t *id;
+
+	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+
+	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
+		    tp->ftt_proc->ftpc_acount != 0)
+			break;
+	}
+
+	/*
+	 * Don't sweat it if we can't find the tracepoint again; unlike
+	 * when we're in fasttrap_pid_probe(), finding the tracepoint here
+	 * is not essential to the correct execution of the process.
+	 */
+	if (tp == NULL) {
+		return;
+	}
+
+	for (id = tp->ftt_retids; id != NULL; id = id->fti_next) {
+		/*
+		 * If there's a branch that could act as a return site, we
+		 * need to trace it, and check here if the program counter is
+		 * external to the function.
+		 */
+		/* Skip function-local branches. */
+		if ((new_pc - id->fti_probe->ftp_faddr) < id->fti_probe->ftp_fsize)
+			continue;
+
+		dtrace_probe(id->fti_probe->ftp_id,
+		    pc - id->fti_probe->ftp_faddr,
+		    rp->fixreg[3], rp->fixreg[4], 0, 0);
+	}
+}
+
+
+static int
+fasttrap_branch_taken(int bo, int bi, struct reg *regs)
+{
+	int crzero = 0;
+
+	/* Branch always? */
+	if ((bo & 0x14) == 0x14)
+		return 1;
+
+	/* Handle decrementing ctr */
+	if (!(bo & 0x04)) {
+		--regs->ctr;
+		crzero = (regs->ctr == 0);
+		if (bo & 0x10) {
+			return (!(crzero ^ (bo >> 1)));
+		}
+	}
+
+	return (crzero | (((regs->cr >> (31 - bi)) ^ (bo >> 3)) ^ 1));
+}
+
+
+int
+fasttrap_pid_probe(struct reg *rp)
+{
+	proc_t *p = curproc;
+	uintptr_t pc = rp->pc;
+	uintptr_t new_pc = 0;
+	fasttrap_bucket_t *bucket;
+	fasttrap_tracepoint_t *tp, tp_local;
+	pid_t pid;
+	dtrace_icookie_t cookie;
+	uint_t is_enabled = 0;
+
+	/*
+	 * It's possible that a user (in a veritable orgy of bad planning)
+	 * could redirect this thread's flow of control before it reached the
+	 * return probe fasttrap. In this case we need to kill the process
+	 * since it's in a unrecoverable state.
+	 */
+	if (curthread->t_dtrace_step) {
+		ASSERT(curthread->t_dtrace_on);
+		fasttrap_sigtrap(p, curthread, pc);
+		return (0);
+	}
+
+	/*
+	 * Clear all user tracing flags.
+	 */
+	curthread->t_dtrace_ft = 0;
+	curthread->t_dtrace_pc = 0;
+	curthread->t_dtrace_npc = 0;
+	curthread->t_dtrace_scrpc = 0;
+	curthread->t_dtrace_astpc = 0;
+
+
+	PROC_LOCK(p);
+	pid = p->p_pid;
+	bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)];
+
+	/*
+	 * Lookup the tracepoint that the process just hit.
+	 */
+	for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) {
+		if (pid == tp->ftt_pid && pc == tp->ftt_pc &&
+		    tp->ftt_proc->ftpc_acount != 0)
+			break;
+	}
+
+	/*
+	 * If we couldn't find a matching tracepoint, either a tracepoint has
+	 * been inserted without using the pid<pid> ioctl interface (see
+	 * fasttrap_ioctl), or somehow we have mislaid this tracepoint.
+	 */
+	if (tp == NULL) {
+		PROC_UNLOCK(p);
+		return (-1);
+	}
+
+	if (tp->ftt_ids != NULL) {
+		fasttrap_id_t *id;
+
+		for (id = tp->ftt_ids; id != NULL; id = id->fti_next) {
+			fasttrap_probe_t *probe = id->fti_probe;
+
+			if (id->fti_ptype == DTFTP_ENTRY) {
+				/*
+				 * We note that this was an entry
+				 * probe to help ustack() find the
+				 * first caller.
+				 */
+				cookie = dtrace_interrupt_disable();
+				DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY);
+				dtrace_probe(probe->ftp_id, rp->fixreg[3],
+						rp->fixreg[4], rp->fixreg[5], rp->fixreg[6],
+						rp->fixreg[7]);
+				DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY);
+				dtrace_interrupt_enable(cookie);
+			} else if (id->fti_ptype == DTFTP_IS_ENABLED) {
+				/*
+				 * Note that in this case, we don't
+				 * call dtrace_probe() since it's only
+				 * an artificial probe meant to change
+				 * the flow of control so that it
+				 * encounters the true probe.
+				 */
+				is_enabled = 1;
+			} else if (probe->ftp_argmap == NULL) {
+				dtrace_probe(probe->ftp_id, rp->fixreg[3],
+				    rp->fixreg[4], rp->fixreg[5], rp->fixreg[6],
+				    rp->fixreg[7]);
+			} else {
+				uintptr_t t[5];
+
+				fasttrap_usdt_args(probe, rp,
+				    sizeof (t) / sizeof (t[0]), t);
+
+				dtrace_probe(probe->ftp_id, t[0], t[1],
+				    t[2], t[3], t[4]);
+			}
+		}
+	}
+
+	/*
+	 * We're about to do a bunch of work so we cache a local copy of
+	 * the tracepoint to emulate the instruction, and then find the
+	 * tracepoint again later if we need to light up any return probes.
+	 */
+	tp_local = *tp;
+	PROC_UNLOCK(p);
+	tp = &tp_local;
+
+	/*
+	 * If there's an is-enabled probe connected to this tracepoint it
+	 * means that there was a 'xor r3, r3, r3'
+	 * instruction that was placed there by DTrace when the binary was
+	 * linked. As this probe is, in fact, enabled, we need to stuff 1
+	 * into R3. Accordingly, we can bypass all the instruction
+	 * emulation logic since we know the inevitable result. It's possible
+	 * that a user could construct a scenario where the 'is-enabled'
+	 * probe was on some other instruction, but that would be a rather
+	 * exotic way to shoot oneself in the foot.
+	 */
+	if (is_enabled) {
+		rp->fixreg[3] = 1;
+		new_pc = rp->pc + 4;
+		goto done;
+	}
+
+
+	switch (tp->ftt_type) {
+	case FASTTRAP_T_NOP:
+		new_pc = rp->pc + 4;
+		break;
+	case FASTTRAP_T_BC:
+		if (!fasttrap_branch_taken(tp->ftt_bo, tp->ftt_bi, rp))
+			break;
+		/* FALLTHROUGH */
+	case FASTTRAP_T_B:
+		if (tp->ftt_instr & 0x01)
+			rp->lr = rp->pc + 4;
+		new_pc = tp->ftt_dest;
+		break;
+	case FASTTRAP_T_BLR:
+	case FASTTRAP_T_BCTR:
+		if (!fasttrap_branch_taken(tp->ftt_bo, tp->ftt_bi, rp))
+			break;
+		/* FALLTHROUGH */
+		if (tp->ftt_type == FASTTRAP_T_BCTR)
+			new_pc = rp->ctr;
+		else
+			new_pc = rp->lr;
+		if (tp->ftt_instr & 0x01)
+			rp->lr = rp->pc + 4;
+		break;
+	case FASTTRAP_T_COMMON:
+		break;
+	};
+done:
+	/*
+	 * If there were no return probes when we first found the tracepoint,
+	 * we should feel no obligation to honor any return probes that were
+	 * subsequently enabled -- they'll just have to wait until the next
+	 * time around.
+	 */
+	if (tp->ftt_retids != NULL) {
+		/*
+		 * We need to wait until the results of the instruction are
+		 * apparent before invoking any return probes. If this
+		 * instruction was emulated we can just call
+		 * fasttrap_return_common(); if it needs to be executed, we
+		 * need to wait until the user thread returns to the kernel.
+		 */
+		if (tp->ftt_type != FASTTRAP_T_COMMON) {
+			fasttrap_return_common(rp, pc, pid, new_pc);
+		} else {
+			ASSERT(curthread->t_dtrace_ret != 0);
+			ASSERT(curthread->t_dtrace_pc == pc);
+			ASSERT(curthread->t_dtrace_scrpc != 0);
+			ASSERT(new_pc == curthread->t_dtrace_astpc);
+		}
+	}
+
+	rp->pc = new_pc;
+	set_regs(curthread, rp);
+
+	return (0);
+}
+
+int
+fasttrap_return_probe(struct reg *rp)
+{
+	proc_t *p = curproc;
+	uintptr_t pc = curthread->t_dtrace_pc;
+	uintptr_t npc = curthread->t_dtrace_npc;
+
+	curthread->t_dtrace_pc = 0;
+	curthread->t_dtrace_npc = 0;
+	curthread->t_dtrace_scrpc = 0;
+	curthread->t_dtrace_astpc = 0;
+
+	/*
+	 * We set rp->pc to the address of the traced instruction so
+	 * that it appears to dtrace_probe() that we're on the original
+	 * instruction, and so that the user can't easily detect our
+	 * complex web of lies. dtrace_return_probe() (our caller)
+	 * will correctly set %pc after we return.
+	 */
+	rp->pc = pc;
+
+	fasttrap_return_common(rp, pc, p->p_pid, npc);
+
+	return (0);
+}
 
-/*
- *	XXX: Placeholder for PowerPC fasttrap code
- */
diff --git a/sys/cddl/contrib/opensolaris/uts/powerpc/sys/fasttrap_isa.h b/sys/cddl/contrib/opensolaris/uts/powerpc/sys/fasttrap_isa.h
index 0f21899..98d5d26 100644
--- a/sys/cddl/contrib/opensolaris/uts/powerpc/sys/fasttrap_isa.h
+++ b/sys/cddl/contrib/opensolaris/uts/powerpc/sys/fasttrap_isa.h
@@ -19,6 +19,7 @@
  *
  * CDDL HEADER END
  */
+/* Portions Copyright 2013 Justin Hibbits */
 /*
  * Copyright 2005 Sun Microsystems, Inc.  All rights reserved.
  * Use is subject to license terms.
@@ -34,13 +35,39 @@
 #ifdef	__cplusplus
 extern "C" {
 #endif
-/*
- *  XXXDTRACE: placehodler for PowerPC fasttrap stuff
- */
 
-typedef	uint32_t	fasttrap_instr_t;
 #define	FASTTRAP_SUNWDTRACE_SIZE	64
-#define FASTTRAP_INSTR 0x0FFFDDDD
+#define FASTTRAP_INSTR			0x0FFFDDDD
+
+typedef	uint32_t	fasttrap_instr_t;
+
+typedef struct fasttrap_machtp_t {
+	fasttrap_instr_t	ftmt_instr;	/* original instruction */
+	uintptr_t		ftmt_dest;	/* branch target */
+	uint8_t			ftmt_type;	/* emulation type */
+	uint8_t			ftmt_flags;	/* emulation flags */
+	uint8_t			ftmt_bo;	/* BO field */
+	uint8_t			ftmt_bi;	/* BI field (CR bit) */
+} fasttrap_machtp_t;
+
+#define	ftt_instr	ftt_mtp.ftmt_instr
+#define	ftt_dest	ftt_mtp.ftmt_dest
+#define	ftt_type	ftt_mtp.ftmt_type
+#define	ftt_flags	ftt_mtp.ftmt_flags
+#define	ftt_bo		ftt_mtp.ftmt_bo
+#define	ftt_bi		ftt_mtp.ftmt_bi
+
+#define FASTTRAP_T_COMMON	0x00
+#define FASTTRAP_T_B		0x01
+#define FASTTRAP_T_BC		0x02
+#define FASTTRAP_T_BLR		0x03
+#define FASTTRAP_T_BCTR		0x04
+#define FASTTRAP_T_NOP		0x05
+
+#define	FASTTRAP_AFRAMES		3
+#define	FASTTRAP_RETURN_AFRAMES		4
+#define	FASTTRAP_ENTRY_AFRAMES		3
+#define	FASTTRAP_OFFSET_AFRAMES		3
 
 #ifdef	__cplusplus
 }
diff --git a/sys/cddl/dev/fbt/fbt_powerpc.c b/sys/cddl/dev/fbt/fbt_powerpc.c
index cdaa06a..88b74dd 100644
--- a/sys/cddl/dev/fbt/fbt_powerpc.c
+++ b/sys/cddl/dev/fbt/fbt_powerpc.c
@@ -219,7 +219,7 @@ fbt_provide_module_function(linker_file_t lf, int symindx,
 		return (0);
 
 	instr = (u_int32_t *) symval->value;
-	limit = (u_int32_t *) symval->value + symval->size;
+	limit = (u_int32_t *) (symval->value + symval->size);
 
 	for (; instr < limit; instr++)
 		if (*instr == FBT_MFLR_R0)
@@ -278,7 +278,7 @@ again:
 	instr++;
 
 	for (j = 0; j < 12 && instr < limit; j++, instr++) {
-		if ((*instr == FBT_BCTR) || (*instr == FBT_BLR) |
+		if ((*instr == FBT_BCTR) || (*instr == FBT_BLR) ||
 		    FBT_IS_JUMP(*instr))
 			break;
 	}
diff --git a/sys/modules/dtrace/Makefile b/sys/modules/dtrace/Makefile
index 2fa7ecd..8b31a17 100644
--- a/sys/modules/dtrace/Makefile
+++ b/sys/modules/dtrace/Makefile
@@ -20,7 +20,7 @@ SUBDIR=		dtmalloc	\
 SUBDIR+=	fasttrap fbt systrace_linux32
 .endif
 .if ${MACHINE_CPUARCH} == "powerpc"
-SUBDIR+=	fbt
+SUBDIR+=	fbt fasttrap
 .endif
 .if ${MACHINE_CPUARCH} == "amd64" || ${MACHINE_ARCH} == "powerpc64"
 SUBDIR+=	systrace_freebsd32
diff --git a/sys/modules/dtrace/fasttrap/Makefile b/sys/modules/dtrace/fasttrap/Makefile
index 6008498..50b5c58 100644
--- a/sys/modules/dtrace/fasttrap/Makefile
+++ b/sys/modules/dtrace/fasttrap/Makefile
@@ -13,6 +13,9 @@ CFLAGS+=	-I${.CURDIR}/../../../cddl/compat/opensolaris \
 .if ${MACHINE_CPUARCH} == "amd64" ||  ${MACHINE_CPUARCH} == "i386"
 CFLAGS+=	-I${.CURDIR}/../../../cddl/contrib/opensolaris/uts/intel
 .PATH:		${.CURDIR}/../../../cddl/contrib/opensolaris/uts/intel/dtrace
+.elif ${MACHINE_CPUARCH} == "powerpc"
+CFLAGS+=	-I${.CURDIR}/../../../cddl/contrib/opensolaris/uts/powerpc
+.PATH:		${.CURDIR}/../../../cddl/contrib/opensolaris/uts/powerpc/dtrace
 .endif
 
 CFLAGS+=	-DSMP
diff --git a/sys/powerpc/aim/trap.c b/sys/powerpc/aim/trap.c
index d31e47b..41e1a79 100644
--- a/sys/powerpc/aim/trap.c
+++ b/sys/powerpc/aim/trap.c
@@ -175,6 +175,9 @@ trap(struct trapframe *frame)
 {
 	struct thread	*td;
 	struct proc	*p;
+#ifdef KDTRACE_HOOKS
+	uint32_t inst;
+#endif
 	int		sig, type, user;
 	u_int		ucode;
 	ksiginfo_t	ksi;
@@ -279,9 +282,18 @@ trap(struct trapframe *frame)
 
 		case EXC_PGM:
 			/* Identify the trap reason */
-			if (frame->srr1 & EXC_PGM_TRAP)
+			if (frame->srr1 & EXC_PGM_TRAP) {
+#ifdef KDTRACE_HOOKS
+				inst = fuword32((const void *)frame->srr0);
+				if (inst == 0x0FFFDDDD && dtrace_pid_probe_ptr != NULL) {
+					struct reg regs;
+					fill_regs(td, &regs);
+					(*dtrace_pid_probe_ptr)(&regs);
+					break;
+				}
+#endif
  				sig = SIGTRAP;
-			else if (ppc_instr_emulate(frame) == 0)
+			} else if (ppc_instr_emulate(frame) == 0)
 				frame->srr0 += 4;
 			else
 				sig = SIGILL;
@@ -299,7 +311,7 @@ trap(struct trapframe *frame)
 #ifdef KDTRACE_HOOKS
 		case EXC_PGM:
 			if (frame->srr1 & EXC_PGM_TRAP) {
-				if (*(uintptr_t *)frame->srr0 == 0x7c810808) {
+				if (*(uint32_t *)frame->srr0 == 0x7c810808) {
 					if (dtrace_invop_jump_addr != NULL) {
 						dtrace_invop_jump_addr(frame);
 						return;
diff --git a/sys/powerpc/aim/trap_subr32.S b/sys/powerpc/aim/trap_subr32.S
index d137fd4..80fa893 100644
--- a/sys/powerpc/aim/trap_subr32.S
+++ b/sys/powerpc/aim/trap_subr32.S
@@ -882,8 +882,8 @@ CNAME(dblow):
 	mfcr	%r29			/* save CR in r29 */
 	mfsrr1	%r1
 	mtcr	%r1
-	bf	17,1f			/* branch if privileged */
-
+	bf	17,2f			/* branch if privileged */
+1:
 	/* Unprivileged case */
 	mtcr	%r29			/* put the condition register back */
         mfsprg2	%r29			/* ... and r29 */
@@ -892,7 +892,19 @@ CNAME(dblow):
 	li	%r1, 0	 		/* How to get the vector from LR */
 
         bla     generictrap		/* and we look like a generic trap */
-1:
+2:
+#ifdef KDTRACE_HOOKS
+	/* Privileged, so drop to KDB */
+	mfsrr0	%r1
+	mtsprg3	%r3
+	lwz	%r1,0(%r1)
+	/* Check if it's a DTrace trap. */
+	li	%r3,0x0808
+	addis	%r3,%r3,0x7c81
+	cmplw	%cr0,%r3,%r1
+	mfsprg3	%r3
+	beq	%cr0,1b
+#endif
 	/* Privileged, so drop to KDB */
 	GET_CPUINFO(%r1)
 	stw	%r28,(PC_DBSAVE+CPUSAVE_R28)(%r1)	/* free r28 */
diff --git a/sys/powerpc/aim/trap_subr64.S b/sys/powerpc/aim/trap_subr64.S
index 8550227..16185f6 100644
--- a/sys/powerpc/aim/trap_subr64.S
+++ b/sys/powerpc/aim/trap_subr64.S
@@ -788,8 +788,9 @@ CNAME(dblow):
 	mfcr	%r29			/* save CR in r29 */
 	mfsrr1	%r1
 	mtcr	%r1
-	bf	17,1f			/* branch if privileged */
+	bf	17,2f			/* branch if privileged */
 
+1:
 	/* Unprivileged case */
 	mtcr	%r29			/* put the condition register back */
         mfsprg2	%r29			/* ... and r29 */
@@ -798,8 +799,19 @@ CNAME(dblow):
 	li	%r1, 0	 		/* How to get the vector from LR */
 
         bla     generictrap		/* and we look like a generic trap */
-1:
+2:
+#ifdef KDTRACE_HOOKS
 	/* Privileged, so drop to KDB */
+	mfsrr0	%r1
+	mtsprg3	%r3
+	lwz	%r1,0(%r1)
+	/* Check if it's a DTrace trap. */
+	li	%r3,0x0808
+	addis	%r3,%r3,0x7c81
+	cmplw	%cr0,%r3,%r1
+	mfsprg3	%r3
+	beq	%cr0,1b
+#endif
 	GET_CPUINFO(%r1)
 	std	%r27,(PC_DBSAVE+CPUSAVE_R27)(%r1)	/* free r27 */
 	std	%r28,(PC_DBSAVE+CPUSAVE_R28)(%r1)	/* free r28 */