Fixes for DTrace on PowerPC:

- Implement dtrace_getarg()
- Sync fbt with x86, and fix a typo.
- Pull in the time synchronization code from amd64.

3 files changed, 261 insertions, 112 deletions

diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c
index 3793adf..9582c97 100644
--- a/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c
+++ b/sys/cddl/dev/dtrace/powerpc/dtrace_isa.c
@@ -349,50 +349,84 @@ zero:
 uint64_t
 dtrace_getarg(int arg, int aframes)
 {
-	return (0);
-}
-
-#ifdef notyet
-{
-	int depth = 0;
-	register_t sp;
-	vm_offset_t callpc;
-	pc_t caller = (pc_t) solaris_cpu[curcpu].cpu_dtrace_caller;
-
-	if (intrpc != 0)
-		pcstack[depth++] = (pc_t) intrpc;
-
-	aframes++;
-
-	sp = dtrace_getfp();
+	uintptr_t val;
+	uintptr_t *fp = (uintptr_t *)dtrace_getfp();
+	uintptr_t *stack;
+	int i;
 
-	while (depth < pcstack_limit) {
-		if (!INKERNEL((long) frame))
-			break;
+	/*
+	 * A total of 8 arguments are passed via registers; any argument with
+	 * index of 7 or lower is therefore in a register.
+	 */
+	int inreg = 7;
 
-		callpc = *(void **)(sp + RETURN_OFFSET);
+	for (i = 1; i <= aframes; i++) {
+		fp = (uintptr_t *)*fp;
 
-		if (!INKERNEL(callpc))
-			break;
+		/*
+		 * On ppc32 AIM, and booke, trapexit() is the immediately following
+		 * label.  On ppc64 AIM trapexit() follows a nop.
+		 */
+		if (((long)(fp[1]) == (long)trapexit) ||
+				(((long)(fp[1]) + 4 == (long)trapexit))) {
+			/*
+			 * In the case of powerpc, we will use the pointer to the regs
+			 * structure that was pushed when we took the trap.  To get this
+			 * structure, we must increment beyond the frame structure.  If the
+			 * argument that we're seeking is passed on the stack, we'll pull
+			 * the true stack pointer out of the saved registers and decrement
+			 * our argument by the number of arguments passed in registers; if
+			 * the argument we're seeking is passed in regsiters, we can just
+			 * load it directly.
+			 */
+#ifdef __powerpc64__
+			struct reg *rp = (struct reg *)((uintptr_t)fp[0] + 48);
+#else
+			struct reg *rp = (struct reg *)((uintptr_t)fp[0] + 8);
+#endif
 
-		if (aframes > 0) {
-			aframes--;
-			if ((aframes == 0) && (caller != 0)) {
-				pcstack[depth++] = caller;
+			if (arg <= inreg) {
+				stack = &rp->fixreg[3];
+			} else {
+				stack = (uintptr_t *)(rp->fixreg[1]);
+				arg -= inreg;
 			}
-		}
-		else {
-			pcstack[depth++] = callpc;
+			goto load;
 		}
 
-		sp = *(void **)sp;
 	}
 
-	for (; depth < pcstack_limit; depth++) {
-		pcstack[depth] = 0;
+	/*
+	 * We know that we did not come through a trap to get into
+	 * dtrace_probe() -- the provider simply called dtrace_probe()
+	 * directly.  As this is the case, we need to shift the argument
+	 * that we're looking for:  the probe ID is the first argument to
+	 * dtrace_probe(), so the argument n will actually be found where
+	 * one would expect to find argument (n + 1).
+	 */
+	arg++;
+
+	if (arg <= inreg) {
+		/*
+		 * This shouldn't happen.  If the argument is passed in a
+		 * register then it should have been, well, passed in a
+		 * register...
+		 */
+		DTRACE_CPUFLAG_SET(CPU_DTRACE_ILLOP);
+		return (0);
 	}
+
+	arg -= (inreg + 1);
+	stack = fp + 2;
+
+load:
+	DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT);
+	val = stack[arg];
+	DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT);
+
+	return (val);
+	return (0);
 }
-#endif
 
 int
 dtrace_getstackdepth(int aframes)
diff --git a/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c b/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c
index e6f1ec0..d22f207 100644
--- a/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c
+++ b/sys/cddl/dev/dtrace/powerpc/dtrace_subr.c
@@ -51,6 +51,8 @@ extern int		dtrace_in_probe;
 extern dtrace_id_t	dtrace_probeid_error;
 extern int (*dtrace_invop_jump_addr)(struct trapframe *);
 
+extern void dtrace_getnanotime(struct timespec *tsp);
+
 int dtrace_invop(uintptr_t, uintptr_t *, uintptr_t);
 void dtrace_invop_init(void);
 void dtrace_invop_uninit(void);
@@ -63,13 +65,13 @@ typedef struct dtrace_invop_hdlr {
 dtrace_invop_hdlr_t *dtrace_invop_hdlr;
 
 int
-dtrace_invop(uintptr_t addr, uintptr_t *stack, uintptr_t eax)
+dtrace_invop(uintptr_t addr, uintptr_t *stack, uintptr_t arg0)
 {
 	dtrace_invop_hdlr_t *hdlr;
 	int rval;
 
 	for (hdlr = dtrace_invop_hdlr; hdlr != NULL; hdlr = hdlr->dtih_next)
-		if ((rval = hdlr->dtih_func(addr, stack, eax)) != 0)
+		if ((rval = hdlr->dtih_func(addr, stack, arg0)) != 0)
 			return (rval);
 
 	return (0);
@@ -134,7 +136,7 @@ dtrace_xcall(processorid_t cpu, dtrace_xcall_t func, void *arg)
 		CPU_SETOF(cpu, &cpus);
 
 	smp_rendezvous_cpus(cpus, smp_no_rendevous_barrier, func,
-	    smp_no_rendevous_barrier, arg);
+			smp_no_rendevous_barrier, arg);
 }
 
 static void
@@ -145,9 +147,82 @@ dtrace_sync_func(void)
 void
 dtrace_sync(void)
 {
-        dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
+	dtrace_xcall(DTRACE_CPUALL, (dtrace_xcall_t)dtrace_sync_func, NULL);
 }
 
+static int64_t	tgt_cpu_tsc;
+static int64_t	hst_cpu_tsc;
+static int64_t	timebase_skew[MAXCPU];
+static uint64_t	nsec_scale;
+
+/* See below for the explanation of this macro. */
+/* This is taken from the amd64 dtrace_subr, to provide a synchronized timer
+ * between multiple processors in dtrace.  Since PowerPC Timebases can be much
+ * lower than x86, the scale shift is 26 instead of 28, allowing for a 15.63MHz
+ * timebase.
+ */
+#define SCALE_SHIFT	26
+
+static void
+dtrace_gethrtime_init_cpu(void *arg)
+{
+	uintptr_t cpu = (uintptr_t) arg;
+
+	if (cpu == curcpu)
+		tgt_cpu_tsc = mftb();
+	else
+		hst_cpu_tsc = mftb();
+}
+
+static void
+dtrace_gethrtime_init(void *arg)
+{
+	struct pcpu *pc;
+	uint64_t tb_f;
+	cpuset_t map;
+	int i;
+
+	tb_f = cpu_tickrate();
+
+	/*
+	 * The following line checks that nsec_scale calculated below
+	 * doesn't overflow 32-bit unsigned integer, so that it can multiply
+	 * another 32-bit integer without overflowing 64-bit.
+	 * Thus minimum supported Timebase frequency is 15.63MHz.
+	 */
+	KASSERT(tb_f > (NANOSEC >> (32 - SCALE_SHIFT)), ("Timebase frequency is too low"));
+
+	/*
+	 * We scale up NANOSEC/tb_f ratio to preserve as much precision
+	 * as possible.
+	 * 2^26 factor was chosen quite arbitrarily from practical
+	 * considerations:
+	 * - it supports TSC frequencies as low as 15.63MHz (see above);
+	 */
+	nsec_scale = ((uint64_t)NANOSEC << SCALE_SHIFT) / tb_f;
+
+	/* The current CPU is the reference one. */
+	sched_pin();
+	timebase_skew[curcpu] = 0;
+	CPU_FOREACH(i) {
+		if (i == curcpu)
+			continue;
+
+		pc = pcpu_find(i);
+		CPU_SETOF(PCPU_GET(cpuid), &map);
+		CPU_SET(pc->pc_cpuid, &map);
+
+		smp_rendezvous_cpus(map, NULL,
+				dtrace_gethrtime_init_cpu,
+				smp_no_rendevous_barrier, (void *)(uintptr_t) i);
+
+		timebase_skew[i] = tgt_cpu_tsc - hst_cpu_tsc;
+	}
+	sched_unpin();
+}
+
+SYSINIT(dtrace_gethrtime_init, SI_SUB_SMP, SI_ORDER_ANY, dtrace_gethrtime_init, NULL);
+
 /*
  * DTrace needs a high resolution time function which can
  * be called from a probe context and guaranteed not to have
@@ -158,12 +233,21 @@ dtrace_sync(void)
 uint64_t
 dtrace_gethrtime()
 {
-	struct      timespec curtime;
-
-	nanouptime(&curtime);
-
-	return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec);
+	uint64_t timebase;
+	uint32_t lo;
+	uint32_t hi;
 
+	/*
+	 * We split timebase value into lower and higher 32-bit halves and separately
+	 * scale them with nsec_scale, then we scale them down by 2^28
+	 * (see nsec_scale calculations) taking into account 32-bit shift of
+	 * the higher half and finally add.
+	 */
+	timebase = mftb() - timebase_skew[curcpu];
+	lo = timebase;
+	hi = timebase >> 32;
+	return (((lo * nsec_scale) >> SCALE_SHIFT) +
+		((hi * nsec_scale) << (32 - SCALE_SHIFT)));
 }
 
 uint64_t
@@ -171,12 +255,12 @@ dtrace_gethrestime(void)
 {
 	struct      timespec curtime;
 
-	getnanotime(&curtime);
+	dtrace_getnanotime(&curtime);
 
 	return (curtime.tv_sec * 1000000000UL + curtime.tv_nsec);
 }
 
-/* Function to handle DTrace traps during probes. See amd64/amd64/trap.c */
+/* Function to handle DTrace traps during probes. See powerpc/powerpc/trap.c */
 int
 dtrace_trap(struct trapframe *frame, u_int type)
 {
@@ -196,34 +280,34 @@ dtrace_trap(struct trapframe *frame, u_int type)
 		 * All the rest will be handled in the usual way.
 		 */
 		switch (type) {
-		/* Page fault. */
-		case EXC_DSI:
-		case EXC_DSE:
-			/* Flag a bad address. */
-			cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
-			cpu_core[curcpu].cpuc_dtrace_illval = frame->cpu.aim.dar;
-
-			/*
-			 * Offset the instruction pointer to the instruction
-			 * following the one causing the fault.
-			 */
-			frame->srr0 += sizeof(int);
-			return (1);
-		case EXC_ISI:
-		case EXC_ISE:
-			/* Flag a bad address. */
-			cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
-			cpu_core[curcpu].cpuc_dtrace_illval = frame->srr0;
-
-			/*
-			 * Offset the instruction pointer to the instruction
-			 * following the one causing the fault.
-			 */
-			frame->srr0 += sizeof(int);
-			return (1);
-		default:
-			/* Handle all other traps in the usual way. */
-			break;
+			/* Page fault. */
+			case EXC_DSI:
+			case EXC_DSE:
+				/* Flag a bad address. */
+				cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
+				cpu_core[curcpu].cpuc_dtrace_illval = frame->cpu.aim.dar;
+
+				/*
+				 * Offset the instruction pointer to the instruction
+				 * following the one causing the fault.
+				 */
+				frame->srr0 += sizeof(int);
+				return (1);
+			case EXC_ISI:
+			case EXC_ISE:
+				/* Flag a bad address. */
+				cpu_core[curcpu].cpuc_dtrace_flags |= CPU_DTRACE_BADADDR;
+				cpu_core[curcpu].cpuc_dtrace_illval = frame->srr0;
+
+				/*
+				 * Offset the instruction pointer to the instruction
+				 * following the one causing the fault.
+				 */
+				frame->srr0 += sizeof(int);
+				return (1);
+			default:
+				/* Handle all other traps in the usual way. */
+				break;
 		}
 	}
 
@@ -237,28 +321,29 @@ dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
 {
 
 	dtrace_probe(dtrace_probeid_error, (uint64_t)(uintptr_t)state,
-	    (uintptr_t)epid,
-	    (uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs);
+			(uintptr_t)epid,
+			(uintptr_t)which, (uintptr_t)fault, (uintptr_t)fltoffs);
 }
 
 static int
 dtrace_invop_start(struct trapframe *frame)
 {
 	switch (dtrace_invop(frame->srr0, (uintptr_t *)frame, frame->fixreg[3])) {
-	case DTRACE_INVOP_JUMP:
-		break;
-	case DTRACE_INVOP_BCTR:
-		frame->srr0 = frame->ctr;
-		break;
-	case DTRACE_INVOP_BLR:
-		frame->srr0 = frame->lr;
-		break;
-	case DTRACE_INVOP_MFLR_R0:
-		frame->fixreg[0] = frame->lr ;
-		break;
-	default:
-		return (-1);
-		break;
+		case DTRACE_INVOP_JUMP:
+			break;
+		case DTRACE_INVOP_BCTR:
+			frame->srr0 = frame->ctr;
+			break;
+		case DTRACE_INVOP_BLR:
+			frame->srr0 = frame->lr;
+			break;
+		case DTRACE_INVOP_MFLR_R0:
+			frame->fixreg[0] = frame->lr;
+			frame->srr0 = frame->srr0 + 4;
+			break;
+		default:
+			return (-1);
+			break;
 	}
 
 	return (0);
diff --git a/sys/cddl/dev/fbt/fbt_powerpc.c b/sys/cddl/dev/fbt/fbt_powerpc.c
index bee3dc7..cdaa06a 100644
--- a/sys/cddl/dev/fbt/fbt_powerpc.c
+++ b/sys/cddl/dev/fbt/fbt_powerpc.c
@@ -57,6 +57,7 @@
 #include <sys/sysproto.h>
 #include <sys/uio.h>
 #include <sys/unistd.h>
+#include <machine/md_var.h>
 #include <machine/stdarg.h>
 
 #include <sys/dtrace.h>
@@ -172,7 +173,11 @@ fbt_invop(uintptr_t addr, uintptr_t *stack, uintptr_t rval)
 					tmp = fbt->fbtp_savedval & FBT_BR_MASK;
 					/* Sign extend. */
 					if (tmp & 0x02000000)
-						tmp |= 0xFC000000;
+#ifdef __powerpc64__
+						tmp |= 0xfffffffffc000000ULL;
+#else
+						tmp |= 0xfc000000UL;
+#endif
 					frame->srr0 += tmp;
 				}
 				cpu->cpu_dtrace_caller = 0;
@@ -193,9 +198,12 @@ fbt_provide_module_function(linker_file_t lf, int symindx,
 	const char *name = symval->name;
 	fbt_probe_t *fbt, *retfbt;
 	int j;
-	int size;
 	u_int32_t *instr, *limit;
 
+	/* PowerPC64 uses '.' prefixes on symbol names, ignore it. */
+	if (name[0] == '.')
+		name++;
+
 	if (strncmp(name, "dtrace_", 7) == 0 &&
 	    strncmp(name, "dtrace_safe_", 12) != 0) {
 		/*
@@ -210,8 +218,6 @@ fbt_provide_module_function(linker_file_t lf, int symindx,
 	if (name[0] == '_' && name[1] == '_')
 		return (0);
 
-	size = symval->size;
-
 	instr = (u_int32_t *) symval->value;
 	limit = (u_int32_t *) symval->value + symval->size;
 
@@ -219,7 +225,7 @@ fbt_provide_module_function(linker_file_t lf, int symindx,
 		if (*instr == FBT_MFLR_R0)
 			break;
 
-	if (*instr != FBT_MFLR_R0);
+	if (*instr != FBT_MFLR_R0)
 		return (0);
 
 	fbt = malloc(sizeof (fbt_probe_t), M_FBT, M_WAITOK | M_ZERO);
@@ -264,9 +270,6 @@ again:
 		}
 	}
 
-	if (*instr == FBT_MFLR_R0)
-		return (0);
-
 	if (*instr != FBT_MTLR_R0) {
 		instr++;
 		goto again;
@@ -291,7 +294,7 @@ again:
 
 	if (retfbt == NULL) {
 		fbt->fbtp_id = dtrace_probe_create(fbt_id, modname,
-		    name, FBT_RETURN, 3, fbt);
+		    name, FBT_RETURN, 5, fbt);
 	} else {
 		retfbt->fbtp_next = fbt;
 		fbt->fbtp_id = retfbt->fbtp_id;
@@ -317,7 +320,7 @@ again:
 
 	lf->fbt_nentries++;
 
-	instr += size;
+	instr += 4;
 	goto again;
 }
 
@@ -434,6 +437,7 @@ fbt_enable(void *arg, dtrace_id_t id, void *parg)
 
 	for (; fbt != NULL; fbt = fbt->fbtp_next) {
 		*fbt->fbtp_patchpoint = fbt->fbtp_patchval;
+		__syncicache(fbt->fbtp_patchpoint, 4);
 	}
 }
 
@@ -449,8 +453,10 @@ fbt_disable(void *arg, dtrace_id_t id, void *parg)
 	if ((ctl->loadcnt != fbt->fbtp_loadcnt))
 		return;
 
-	for (; fbt != NULL; fbt = fbt->fbtp_next)
+	for (; fbt != NULL; fbt = fbt->fbtp_next) {
 		*fbt->fbtp_patchpoint = fbt->fbtp_savedval;
+		__syncicache(fbt->fbtp_patchpoint, 4);
+	}
 }
 
 static void
@@ -464,8 +470,10 @@ fbt_suspend(void *arg, dtrace_id_t id, void *parg)
 	if ((ctl->loadcnt != fbt->fbtp_loadcnt))
 		return;
 
-	for (; fbt != NULL; fbt = fbt->fbtp_next)
+	for (; fbt != NULL; fbt = fbt->fbtp_next) {
 		*fbt->fbtp_patchpoint = fbt->fbtp_savedval;
+		__syncicache(fbt->fbtp_patchpoint, 4);
+	}
 }
 
 static void
@@ -479,15 +487,16 @@ fbt_resume(void *arg, dtrace_id_t id, void *parg)
 	if ((ctl->loadcnt != fbt->fbtp_loadcnt))
 		return;
 
-	for (; fbt != NULL; fbt = fbt->fbtp_next)
+	for (; fbt != NULL; fbt = fbt->fbtp_next) {
 		*fbt->fbtp_patchpoint = fbt->fbtp_patchval;
+		__syncicache(fbt->fbtp_patchpoint, 4);
+	}
 }
 
 static int
 fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc)
 {
 	const Elf_Sym *symp = lc->symtab;;
-	const char *name;
 	const ctf_header_t *hp = (const ctf_header_t *) lc->ctftab;
 	const uint8_t *ctfdata = lc->ctftab + sizeof(ctf_header_t);
 	int i;
@@ -519,11 +528,6 @@ fbt_ctfoff_init(modctl_t *lf, linker_ctf_t *lc)
 			continue;
 		}
 
-		if (symp->st_name < lc->strcnt)
-			name = lc->strtab + symp->st_name;
-		else
-			name = "(?)";
-
 		switch (ELF_ST_TYPE(symp->st_info)) {
 		case STT_OBJECT:
 			if (objtoff >= hp->cth_funcoff ||
@@ -690,6 +694,8 @@ fbt_typoff_init(linker_ctf_t *lc)
 		pop[kind]++;
 	}
 
+	/* account for a sentinel value below */
+	ctf_typemax++;
 	*lc->typlenp = ctf_typemax;
 
 	if ((xp = malloc(sizeof(uint32_t) * ctf_typemax, M_LINKER, M_ZERO | M_WAITOK)) == NULL)
@@ -1171,6 +1177,11 @@ fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_a
 	uint32_t offset;
 	ushort_t info, kind, n;
 
+	if (fbt->fbtp_roffset != 0 && desc->dtargd_ndx == 0) {
+		(void) strcpy(desc->dtargd_native, "int");
+		return;
+	}
+
 	desc->dtargd_ndx = DTRACE_ARGNONE;
 
 	/* Get a pointer to the CTF data and it's length. */
@@ -1221,12 +1232,19 @@ fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_a
 		return;
 	}
 
-	/* Check if the requested argument doesn't exist. */
-	if (ndx >= n)
-		return;
+	if (fbt->fbtp_roffset != 0) {
+		/* Only return type is available for args[1] in return probe. */
+		if (ndx > 1)
+			return;
+		ASSERT(ndx == 1);
+	} else {
+		/* Check if the requested argument doesn't exist. */
+		if (ndx >= n)
+			return;
 
-	/* Skip the return type and arguments up to the one requested. */
-	dp += ndx + 1;
+		/* Skip the return type and arguments up to the one requested. */
+		dp += ndx + 1;
+	}
 
 	if (fbt_type_name(&lc, *dp, desc->dtargd_native, sizeof(desc->dtargd_native)) > 0)
 		desc->dtargd_ndx = ndx;
@@ -1234,6 +1252,15 @@ fbt_getargdesc(void *arg __unused, dtrace_id_t id __unused, void *parg, dtrace_a
 	return;
 }
 
+static int
+fbt_linker_file_cb(linker_file_t lf, void *arg)
+{
+
+	fbt_provide_module(arg, lf);
+
+	return (0);
+}
+
 static void
 fbt_load(void *dummy)
 {
@@ -1257,6 +1284,9 @@ fbt_load(void *dummy)
 	if (dtrace_register("fbt", &fbt_attr, DTRACE_PRIV_USER,
 	    NULL, &fbt_pops, NULL, &fbt_id) != 0)
 		return;
+
+	/* Create probes for the kernel and already-loaded modules. */
+	linker_file_foreach(fbt_linker_file_cb, NULL);
 }