diff options
Diffstat (limited to 'sys/cddl/contrib/opensolaris/uts/sparc/dtrace/fasttrap_isa.c')
| -rw-r--r-- | sys/cddl/contrib/opensolaris/uts/sparc/dtrace/fasttrap_isa.c | 1595 |
1 files changed, 0 insertions, 1595 deletions
diff --git a/sys/cddl/contrib/opensolaris/uts/sparc/dtrace/fasttrap_isa.c b/sys/cddl/contrib/opensolaris/uts/sparc/dtrace/fasttrap_isa.c deleted file mode 100644 index 43315a0..0000000 --- a/sys/cddl/contrib/opensolaris/uts/sparc/dtrace/fasttrap_isa.c +++ /dev/null @@ -1,1595 +0,0 @@ -/* - * CDDL HEADER START - * - * The contents of this file are subject to the terms of the - * Common Development and Distribution License (the "License"). - * You may not use this file except in compliance with the License. - * - * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE - * or http://www.opensolaris.org/os/licensing. - * See the License for the specific language governing permissions - * and limitations under the License. - * - * When distributing Covered Code, include this CDDL HEADER in each - * file and include the License file at usr/src/OPENSOLARIS.LICENSE. - * If applicable, add the following below this CDDL HEADER, with the - * fields enclosed by brackets "[]" replaced with your own identifying - * information: Portions Copyright [yyyy] [name of copyright owner] - * - * CDDL HEADER END - */ - -/* - * Copyright 2007 Sun Microsystems, Inc. All rights reserved. - * Use is subject to license terms. - */ - -#include <sys/fasttrap_isa.h> -#include <sys/fasttrap_impl.h> -#include <sys/dtrace.h> -#include <sys/dtrace_impl.h> -#include <sys/cmn_err.h> -#include <sys/frame.h> -#include <sys/stack.h> -#include <sys/sysmacros.h> -#include <sys/trap.h> - -#include <v9/sys/machpcb.h> -#include <v9/sys/privregs.h> - -/* - * Lossless User-Land Tracing on SPARC - * ----------------------------------- - * - * The Basic Idea - * - * The most important design constraint is, of course, correct execution of - * the user thread above all else. The next most important goal is rapid - * execution. We combine execution of instructions in user-land with - * emulation of certain instructions in the kernel to aim for complete - * correctness and maximal performance. - * - * We take advantage of the split PC/NPC architecture to speed up logical - * single-stepping; when we copy an instruction out to the scratch space in - * the ulwp_t structure (held in the %g7 register on SPARC), we can - * effectively single step by setting the PC to our scratch space and leaving - * the NPC alone. This executes the replaced instruction and then continues - * on without having to reenter the kernel as with single- stepping. The - * obvious caveat is for instructions whose execution is PC dependant -- - * branches, call and link instructions (call and jmpl), and the rdpc - * instruction. These instructions cannot be executed in the manner described - * so they must be emulated in the kernel. - * - * Emulation for this small set of instructions if fairly simple; the most - * difficult part being emulating branch conditions. - * - * - * A Cache Heavy Portfolio - * - * It's important to note at this time that copying an instruction out to the - * ulwp_t scratch space in user-land is rather complicated. SPARC has - * separate data and instruction caches so any writes to the D$ (using a - * store instruction for example) aren't necessarily reflected in the I$. - * The flush instruction can be used to synchronize the two and must be used - * for any self-modifying code, but the flush instruction only applies to the - * primary address space (the absence of a flusha analogue to the flush - * instruction that accepts an ASI argument is an obvious omission from SPARC - * v9 where the notion of the alternate address space was introduced on - * SPARC). To correctly copy out the instruction we must use a block store - * that doesn't allocate in the D$ and ensures synchronization with the I$; - * see dtrace_blksuword32() for the implementation (this function uses - * ASI_BLK_COMMIT_S to write a block through the secondary ASI in the manner - * described). Refer to the UltraSPARC I/II manual for details on the - * ASI_BLK_COMMIT_S ASI. - * - * - * Return Subtleties - * - * When we're firing a return probe we need to expose the value returned by - * the function being traced. Since the function can set the return value - * in its last instruction, we need to fire the return probe only _after_ - * the effects of the instruction are apparent. For instructions that we - * emulate, we can call dtrace_probe() after we've performed the emulation; - * for instructions that we execute after we return to user-land, we set - * %pc to the instruction we copied out (as described above) and set %npc - * to a trap instruction stashed in the ulwp_t structure. After the traced - * instruction is executed, the trap instruction returns control to the - * kernel where we can fire the return probe. - * - * This need for a second trap in cases where we execute the traced - * instruction makes it all the more important to emulate the most common - * instructions to avoid the second trip in and out of the kernel. - * - * - * Making it Fast - * - * Since copying out an instruction is neither simple nor inexpensive for the - * CPU, we should attempt to avoid doing it in as many cases as possible. - * Since function entry and return are usually the most interesting probe - * sites, we attempt to tune the performance of the fasttrap provider around - * instructions typically in those places. - * - * Looking at a bunch of functions in libraries and executables reveals that - * most functions begin with either a save or a sethi (to setup a larger - * argument to the save) and end with a restore or an or (in the case of leaf - * functions). To try to improve performance, we emulate all of these - * instructions in the kernel. - * - * The save and restore instructions are a little tricky since they perform - * register window maniplulation. Rather than trying to tinker with the - * register windows from the kernel, we emulate the implicit add that takes - * place as part of those instructions and set the %pc to point to a simple - * save or restore we've hidden in the ulwp_t structure. If we're in a return - * probe so want to make it seem as though the tracepoint has been completely - * executed we need to remember that we've pulled this trick with restore and - * pull registers from the previous window (the one that we'll switch to once - * the simple store instruction is executed) rather than the current one. This - * is why in the case of emulating a restore we set the DTrace CPU flag - * CPU_DTRACE_FAKERESTORE before calling dtrace_probe() for the return probes - * (see fasttrap_return_common()). - */ - -#define OP(x) ((x) >> 30) -#define OP2(x) (((x) >> 22) & 0x07) -#define OP3(x) (((x) >> 19) & 0x3f) -#define RCOND(x) (((x) >> 25) & 0x07) -#define COND(x) (((x) >> 25) & 0x0f) -#define A(x) (((x) >> 29) & 0x01) -#define I(x) (((x) >> 13) & 0x01) -#define RD(x) (((x) >> 25) & 0x1f) -#define RS1(x) (((x) >> 14) & 0x1f) -#define RS2(x) (((x) >> 0) & 0x1f) -#define CC(x) (((x) >> 20) & 0x03) -#define DISP16(x) ((((x) >> 6) & 0xc000) | ((x) & 0x3fff)) -#define DISP22(x) ((x) & 0x3fffff) -#define DISP19(x) ((x) & 0x7ffff) -#define DISP30(x) ((x) & 0x3fffffff) -#define SW_TRAP(x) ((x) & 0x7f) - -#define OP3_OR 0x02 -#define OP3_RD 0x28 -#define OP3_JMPL 0x38 -#define OP3_RETURN 0x39 -#define OP3_TCC 0x3a -#define OP3_SAVE 0x3c -#define OP3_RESTORE 0x3d - -#define OP3_PREFETCH 0x2d -#define OP3_CASA 0x3c -#define OP3_PREFETCHA 0x3d -#define OP3_CASXA 0x3e - -#define OP2_ILLTRAP 0x0 -#define OP2_BPcc 0x1 -#define OP2_Bicc 0x2 -#define OP2_BPr 0x3 -#define OP2_SETHI 0x4 -#define OP2_FBPfcc 0x5 -#define OP2_FBfcc 0x6 - -#define R_G0 0 -#define R_O0 8 -#define R_SP 14 -#define R_I0 24 -#define R_I1 25 -#define R_I2 26 -#define R_I3 27 -#define R_I4 28 - -/* - * Check the comment in fasttrap.h when changing these offsets or adding - * new instructions. - */ -#define FASTTRAP_OFF_SAVE 64 -#define FASTTRAP_OFF_RESTORE 68 -#define FASTTRAP_OFF_FTRET 72 -#define FASTTRAP_OFF_RETURN 76 - -#define BREAKPOINT_INSTR 0x91d02001 /* ta 1 */ - -/* - * Tunable to let users turn off the fancy save instruction optimization. - * If a program is non-ABI compliant, there's a possibility that the save - * instruction optimization could cause an error. - */ -int fasttrap_optimize_save = 1; - -static uint64_t -fasttrap_anarg(struct regs *rp, int argno) -{ - uint64_t value; - - if (argno < 6) - return ((&rp->r_o0)[argno]); - - if (curproc->p_model == DATAMODEL_NATIVE) { - struct frame *fr = (struct frame *)(rp->r_sp + STACK_BIAS); - - DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); - value = dtrace_fulword(&fr->fr_argd[argno]); - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR | - CPU_DTRACE_BADALIGN); - } else { - struct frame32 *fr = (struct frame32 *)rp->r_sp; - - DTRACE_CPUFLAG_SET(CPU_DTRACE_NOFAULT); - value = dtrace_fuword32(&fr->fr_argd[argno]); - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_NOFAULT | CPU_DTRACE_BADADDR | - CPU_DTRACE_BADALIGN); - } - - return (value); -} - -static ulong_t fasttrap_getreg(struct regs *, uint_t); -static void fasttrap_putreg(struct regs *, uint_t, ulong_t); - -static void -fasttrap_usdt_args(fasttrap_probe_t *probe, struct regs *rp, - uint_t fake_restore, int argc, uintptr_t *argv) -{ - int i, x, cap = MIN(argc, probe->ftp_nargs); - int inc = (fake_restore ? 16 : 0); - - /* - * The only way we'll hit the fake_restore case is if a USDT probe is - * invoked as a tail-call. While it wouldn't be incorrect, we can - * avoid a call to fasttrap_getreg(), and safely use rp->r_sp - * directly since a tail-call can't be made if the invoked function - * would use the argument dump space (i.e. if there were more than - * 6 arguments). We take this shortcut because unconditionally rooting - * around for R_FP (R_SP + 16) would be unnecessarily painful. - */ - - if (curproc->p_model == DATAMODEL_NATIVE) { - struct frame *fr = (struct frame *)(rp->r_sp + STACK_BIAS); - uintptr_t v; - - for (i = 0; i < cap; i++) { - x = probe->ftp_argmap[i]; - - if (x < 6) - argv[i] = fasttrap_getreg(rp, R_O0 + x + inc); - else if (fasttrap_fulword(&fr->fr_argd[x], &v) != 0) - argv[i] = 0; - } - - } else { - struct frame32 *fr = (struct frame32 *)rp->r_sp; - uint32_t v; - - for (i = 0; i < cap; i++) { - x = probe->ftp_argmap[i]; - - if (x < 6) - argv[i] = fasttrap_getreg(rp, R_O0 + x + inc); - else if (fasttrap_fuword32(&fr->fr_argd[x], &v) != 0) - argv[i] = 0; - } - } - - for (; i < argc; i++) { - argv[i] = 0; - } -} - -static void -fasttrap_return_common(struct regs *rp, uintptr_t pc, pid_t pid, - uint_t fake_restore) -{ - fasttrap_tracepoint_t *tp; - fasttrap_bucket_t *bucket; - fasttrap_id_t *id; - kmutex_t *pid_mtx; - dtrace_icookie_t cookie; - - pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; - mutex_enter(pid_mtx); - bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; - - for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { - if (pid == tp->ftt_pid && pc == tp->ftt_pc && - tp->ftt_proc->ftpc_acount != 0) - break; - } - - /* - * Don't sweat it if we can't find the tracepoint again; unlike - * when we're in fasttrap_pid_probe(), finding the tracepoint here - * is not essential to the correct execution of the process. - */ - if (tp == NULL || tp->ftt_retids == NULL) { - mutex_exit(pid_mtx); - return; - } - - for (id = tp->ftt_retids; id != NULL; id = id->fti_next) { - fasttrap_probe_t *probe = id->fti_probe; - - if (id->fti_ptype == DTFTP_POST_OFFSETS) { - if (probe->ftp_argmap != NULL && fake_restore) { - uintptr_t t[5]; - - fasttrap_usdt_args(probe, rp, fake_restore, - sizeof (t) / sizeof (t[0]), t); - - cookie = dtrace_interrupt_disable(); - DTRACE_CPUFLAG_SET(CPU_DTRACE_FAKERESTORE); - dtrace_probe(probe->ftp_id, t[0], t[1], - t[2], t[3], t[4]); - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_FAKERESTORE); - dtrace_interrupt_enable(cookie); - - } else if (probe->ftp_argmap != NULL) { - uintptr_t t[5]; - - fasttrap_usdt_args(probe, rp, fake_restore, - sizeof (t) / sizeof (t[0]), t); - - dtrace_probe(probe->ftp_id, t[0], t[1], - t[2], t[3], t[4]); - - } else if (fake_restore) { - uintptr_t arg0 = fasttrap_getreg(rp, R_I0); - uintptr_t arg1 = fasttrap_getreg(rp, R_I1); - uintptr_t arg2 = fasttrap_getreg(rp, R_I2); - uintptr_t arg3 = fasttrap_getreg(rp, R_I3); - uintptr_t arg4 = fasttrap_getreg(rp, R_I4); - - cookie = dtrace_interrupt_disable(); - DTRACE_CPUFLAG_SET(CPU_DTRACE_FAKERESTORE); - dtrace_probe(probe->ftp_id, arg0, arg1, - arg2, arg3, arg4); - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_FAKERESTORE); - dtrace_interrupt_enable(cookie); - - } else { - dtrace_probe(probe->ftp_id, rp->r_o0, rp->r_o1, - rp->r_o2, rp->r_o3, rp->r_o4); - } - - continue; - } - - /* - * If this is only a possible return point, we must - * be looking at a potential tail call in leaf context. - * If the %npc is still within this function, then we - * must have misidentified a jmpl as a tail-call when it - * is, in fact, part of a jump table. It would be nice to - * remove this tracepoint, but this is neither the time - * nor the place. - */ - if ((tp->ftt_flags & FASTTRAP_F_RETMAYBE) && - rp->r_npc - probe->ftp_faddr < probe->ftp_fsize) - continue; - - /* - * It's possible for a function to branch to the delay slot - * of an instruction that we've identified as a return site. - * We can dectect this spurious return probe activation by - * observing that in this case %npc will be %pc + 4 and %npc - * will be inside the current function (unless the user is - * doing _crazy_ instruction picking in which case there's - * very little we can do). The second check is important - * in case the last instructions of a function make a tail- - * call to the function located immediately subsequent. - */ - if (rp->r_npc == rp->r_pc + 4 && - rp->r_npc - probe->ftp_faddr < probe->ftp_fsize) - continue; - - /* - * The first argument is the offset of return tracepoint - * in the function; the remaining arguments are the return - * values. - * - * If fake_restore is set, we need to pull the return values - * out of the %i's rather than the %o's -- a little trickier. - */ - if (!fake_restore) { - dtrace_probe(probe->ftp_id, pc - probe->ftp_faddr, - rp->r_o0, rp->r_o1, rp->r_o2, rp->r_o3); - } else { - uintptr_t arg0 = fasttrap_getreg(rp, R_I0); - uintptr_t arg1 = fasttrap_getreg(rp, R_I1); - uintptr_t arg2 = fasttrap_getreg(rp, R_I2); - uintptr_t arg3 = fasttrap_getreg(rp, R_I3); - - cookie = dtrace_interrupt_disable(); - DTRACE_CPUFLAG_SET(CPU_DTRACE_FAKERESTORE); - dtrace_probe(probe->ftp_id, pc - probe->ftp_faddr, - arg0, arg1, arg2, arg3); - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_FAKERESTORE); - dtrace_interrupt_enable(cookie); - } - } - - mutex_exit(pid_mtx); -} - -int -fasttrap_pid_probe(struct regs *rp) -{ - proc_t *p = curproc; - fasttrap_tracepoint_t *tp, tp_local; - fasttrap_id_t *id; - pid_t pid; - uintptr_t pc = rp->r_pc; - uintptr_t npc = rp->r_npc; - uintptr_t orig_pc = pc; - fasttrap_bucket_t *bucket; - kmutex_t *pid_mtx; - uint_t fake_restore = 0, is_enabled = 0; - dtrace_icookie_t cookie; - - /* - * It's possible that a user (in a veritable orgy of bad planning) - * could redirect this thread's flow of control before it reached the - * return probe fasttrap. In this case we need to kill the process - * since it's in a unrecoverable state. - */ - if (curthread->t_dtrace_step) { - ASSERT(curthread->t_dtrace_on); - fasttrap_sigtrap(p, curthread, pc); - return (0); - } - - /* - * Clear all user tracing flags. - */ - curthread->t_dtrace_ft = 0; - curthread->t_dtrace_pc = 0; - curthread->t_dtrace_npc = 0; - curthread->t_dtrace_scrpc = 0; - curthread->t_dtrace_astpc = 0; - - /* - * Treat a child created by a call to vfork(2) as if it were its - * parent. We know that there's only one thread of control in such a - * process: this one. - */ - while (p->p_flag & SVFORK) { - p = p->p_parent; - } - - pid = p->p_pid; - pid_mtx = &cpu_core[CPU->cpu_id].cpuc_pid_lock; - mutex_enter(pid_mtx); - bucket = &fasttrap_tpoints.fth_table[FASTTRAP_TPOINTS_INDEX(pid, pc)]; - - /* - * Lookup the tracepoint that the process just hit. - */ - for (tp = bucket->ftb_data; tp != NULL; tp = tp->ftt_next) { - if (pid == tp->ftt_pid && pc == tp->ftt_pc && - tp->ftt_proc->ftpc_acount != 0) - break; - } - - /* - * If we couldn't find a matching tracepoint, either a tracepoint has - * been inserted without using the pid<pid> ioctl interface (see - * fasttrap_ioctl), or somehow we have mislaid this tracepoint. - */ - if (tp == NULL) { - mutex_exit(pid_mtx); - return (-1); - } - - for (id = tp->ftt_ids; id != NULL; id = id->fti_next) { - fasttrap_probe_t *probe = id->fti_probe; - int isentry = (id->fti_ptype == DTFTP_ENTRY); - - if (id->fti_ptype == DTFTP_IS_ENABLED) { - is_enabled = 1; - continue; - } - - /* - * We note that this was an entry probe to help ustack() find - * the first caller. - */ - if (isentry) { - cookie = dtrace_interrupt_disable(); - DTRACE_CPUFLAG_SET(CPU_DTRACE_ENTRY); - } - dtrace_probe(probe->ftp_id, rp->r_o0, rp->r_o1, rp->r_o2, - rp->r_o3, rp->r_o4); - if (isentry) { - DTRACE_CPUFLAG_CLEAR(CPU_DTRACE_ENTRY); - dtrace_interrupt_enable(cookie); - } - } - - /* - * We're about to do a bunch of work so we cache a local copy of - * the tracepoint to emulate the instruction, and then find the - * tracepoint again later if we need to light up any return probes. - */ - tp_local = *tp; - mutex_exit(pid_mtx); - tp = &tp_local; - - /* - * If there's an is-enabled probe conntected to this tracepoint it - * means that there was a 'mov %g0, %o0' instruction that was placed - * there by DTrace when the binary was linked. As this probe is, in - * fact, enabled, we need to stuff 1 into %o0. Accordingly, we can - * bypass all the instruction emulation logic since we know the - * inevitable result. It's possible that a user could construct a - * scenario where the 'is-enabled' probe was on some other - * instruction, but that would be a rather exotic way to shoot oneself - * in the foot. - */ - if (is_enabled) { - rp->r_o0 = 1; - pc = rp->r_npc; - npc = pc + 4; - goto done; - } - - /* - * We emulate certain types of instructions to ensure correctness - * (in the case of position dependent instructions) or optimize - * common cases. The rest we have the thread execute back in user- - * land. - */ - switch (tp->ftt_type) { - case FASTTRAP_T_SAVE: - { - int32_t imm; - - /* - * This an optimization to let us handle function entry - * probes more efficiently. Many functions begin with a save - * instruction that follows the pattern: - * save %sp, <imm>, %sp - * - * Meanwhile, we've stashed the instruction: - * save %g1, %g0, %sp - * - * off of %g7, so all we have to do is stick the right value - * into %g1 and reset %pc to point to the instruction we've - * cleverly hidden (%npc should not be touched). - */ - - imm = tp->ftt_instr << 19; - imm >>= 19; - rp->r_g1 = rp->r_sp + imm; - pc = rp->r_g7 + FASTTRAP_OFF_SAVE; - break; - } - - case FASTTRAP_T_RESTORE: - { - ulong_t value; - uint_t rd; - - /* - * This is an optimization to let us handle function - * return probes more efficiently. Most non-leaf functions - * end with the sequence: - * ret - * restore <reg>, <reg_or_imm>, %oX - * - * We've stashed the instruction: - * restore %g0, %g0, %g0 - * - * off of %g7 so we just need to place the correct value - * in the right %i register (since after our fake-o - * restore, the %i's will become the %o's) and set the %pc - * to point to our hidden restore. We also set fake_restore to - * let fasttrap_return_common() know that it will find the - * return values in the %i's rather than the %o's. - */ - - if (I(tp->ftt_instr)) { - int32_t imm; - - imm = tp->ftt_instr << 19; - imm >>= 19; - value = fasttrap_getreg(rp, RS1(tp->ftt_instr)) + imm; - } else { - value = fasttrap_getreg(rp, RS1(tp->ftt_instr)) + - fasttrap_getreg(rp, RS2(tp->ftt_instr)); - } - - /* - * Convert %o's to %i's; leave %g's as they are. - */ - rd = RD(tp->ftt_instr); - fasttrap_putreg(rp, ((rd & 0x18) == 0x8) ? rd + 16 : rd, value); - - pc = rp->r_g7 + FASTTRAP_OFF_RESTORE; - fake_restore = 1; - break; - } - - case FASTTRAP_T_RETURN: - { - uintptr_t target; - - /* - * A return instruction is like a jmpl (without the link - * part) that executes an implicit restore. We've stashed - * the instruction: - * return %o0 - * - * off of %g7 so we just need to place the target in %o0 - * and set the %pc to point to the stashed return instruction. - * We use %o0 since that register disappears after the return - * executes, erasing any evidence of this tampering. - */ - if (I(tp->ftt_instr)) { - int32_t imm; - - imm = tp->ftt_instr << 19; - imm >>= 19; - target = fasttrap_getreg(rp, RS1(tp->ftt_instr)) + imm; - } else { - target = fasttrap_getreg(rp, RS1(tp->ftt_instr)) + - fasttrap_getreg(rp, RS2(tp->ftt_instr)); - } - - fasttrap_putreg(rp, R_O0, target); - - pc = rp->r_g7 + FASTTRAP_OFF_RETURN; - fake_restore = 1; - break; - } - - case FASTTRAP_T_OR: - { - ulong_t value; - - if (I(tp->ftt_instr)) { - int32_t imm; - - imm = tp->ftt_instr << 19; - imm >>= 19; - value = fasttrap_getreg(rp, RS1(tp->ftt_instr)) | imm; - } else { - value = fasttrap_getreg(rp, RS1(tp->ftt_instr)) | - fasttrap_getreg(rp, RS2(tp->ftt_instr)); - } - - fasttrap_putreg(rp, RD(tp->ftt_instr), value); - pc = rp->r_npc; - npc = pc + 4; - break; - } - - case FASTTRAP_T_SETHI: - if (RD(tp->ftt_instr) != R_G0) { - uint32_t imm32 = tp->ftt_instr << 10; - fasttrap_putreg(rp, RD(tp->ftt_instr), (ulong_t)imm32); - } - pc = rp->r_npc; - npc = pc + 4; - break; - - case FASTTRAP_T_CCR: - { - uint_t c, v, z, n, taken; - uint_t ccr = rp->r_tstate >> TSTATE_CCR_SHIFT; - - if (tp->ftt_cc != 0) - ccr >>= 4; - - c = (ccr >> 0) & 1; - v = (ccr >> 1) & 1; - z = (ccr >> 2) & 1; - n = (ccr >> 3) & 1; - - switch (tp->ftt_code) { - case 0x0: /* BN */ - taken = 0; break; - case 0x1: /* BE */ - taken = z; break; - case 0x2: /* BLE */ - taken = z | (n ^ v); break; - case 0x3: /* BL */ - taken = n ^ v; break; - case 0x4: /* BLEU */ - taken = c | z; break; - case 0x5: /* BCS (BLU) */ - taken = c; break; - case 0x6: /* BNEG */ - taken = n; break; - case 0x7: /* BVS */ - taken = v; break; - case 0x8: /* BA */ - /* - * We handle the BA case differently since the annul - * bit means something slightly different. - */ - panic("fasttrap: mishandled a branch"); - taken = 1; break; - case 0x9: /* BNE */ - taken = ~z; break; - case 0xa: /* BG */ - taken = ~(z | (n ^ v)); break; - case 0xb: /* BGE */ - taken = ~(n ^ v); break; - case 0xc: /* BGU */ - taken = ~(c | z); break; - case 0xd: /* BCC (BGEU) */ - taken = ~c; break; - case 0xe: /* BPOS */ - taken = ~n; break; - case 0xf: /* BVC */ - taken = ~v; break; - } - - if (taken & 1) { - pc = rp->r_npc; - npc = tp->ftt_dest; - } else if (tp->ftt_flags & FASTTRAP_F_ANNUL) { - /* - * Untaken annulled branches don't execute the - * instruction in the delay slot. - */ - pc = rp->r_npc + 4; - npc = pc + 4; - } else { - pc = rp->r_npc; - npc = pc + 4; - } - break; - } - - case FASTTRAP_T_FCC: - { - uint_t fcc; - uint_t taken; - uint64_t fsr; - - dtrace_getfsr(&fsr); - - if (tp->ftt_cc == 0) { - fcc = (fsr >> 10) & 0x3; - } else { - uint_t shift; - ASSERT(tp->ftt_cc <= 3); - shift = 30 + tp->ftt_cc * 2; - fcc = (fsr >> shift) & 0x3; - } - - switch (tp->ftt_code) { - case 0x0: /* FBN */ - taken = (1 << fcc) & (0|0|0|0); break; - case 0x1: /* FBNE */ - taken = (1 << fcc) & (8|4|2|0); break; - case 0x2: /* FBLG */ - taken = (1 << fcc) & (0|4|2|0); break; - case 0x3: /* FBUL */ - taken = (1 << fcc) & (8|0|2|0); break; - case 0x4: /* FBL */ - taken = (1 << fcc) & (0|0|2|0); break; - case 0x5: /* FBUG */ - taken = (1 << fcc) & (8|4|0|0); break; - case 0x6: /* FBG */ - taken = (1 << fcc) & (0|4|0|0); break; - case 0x7: /* FBU */ - taken = (1 << fcc) & (8|0|0|0); break; - case 0x8: /* FBA */ - /* - * We handle the FBA case differently since the annul - * bit means something slightly different. - */ - panic("fasttrap: mishandled a branch"); - taken = (1 << fcc) & (8|4|2|1); break; - case 0x9: /* FBE */ - taken = (1 << fcc) & (0|0|0|1); break; - case 0xa: /* FBUE */ - taken = (1 << fcc) & (8|0|0|1); break; - case 0xb: /* FBGE */ - taken = (1 << fcc) & (0|4|0|1); break; - case 0xc: /* FBUGE */ - taken = (1 << fcc) & (8|4|0|1); break; - case 0xd: /* FBLE */ - taken = (1 << fcc) & (0|0|2|1); break; - case 0xe: /* FBULE */ - taken = (1 << fcc) & (8|0|2|1); break; - case 0xf: /* FBO */ - taken = (1 << fcc) & (0|4|2|1); break; - } - - if (taken) { - pc = rp->r_npc; - npc = tp->ftt_dest; - } else if (tp->ftt_flags & FASTTRAP_F_ANNUL) { - /* - * Untaken annulled branches don't execute the - * instruction in the delay slot. - */ - pc = rp->r_npc + 4; - npc = pc + 4; - } else { - pc = rp->r_npc; - npc = pc + 4; - } - break; - } - - case FASTTRAP_T_REG: - { - int64_t value; - uint_t taken; - uint_t reg = RS1(tp->ftt_instr); - - /* - * An ILP32 process shouldn't be using a branch predicated on - * an %i or an %l since it would violate the ABI. It's a - * violation of the ABI because we can't ensure deterministic - * behavior. We should have identified this case when we - * enabled the probe. - */ - ASSERT(p->p_model == DATAMODEL_LP64 || reg < 16); - - value = (int64_t)fasttrap_getreg(rp, reg); - - switch (tp->ftt_code) { - case 0x1: /* BRZ */ - taken = (value == 0); break; - case 0x2: /* BRLEZ */ - taken = (value <= 0); break; - case 0x3: /* BRLZ */ - taken = (value < 0); break; - case 0x5: /* BRNZ */ - taken = (value != 0); break; - case 0x6: /* BRGZ */ - taken = (value > 0); break; - case 0x7: /* BRGEZ */ - taken = (value >= 0); break; - default: - case 0x0: - case 0x4: - panic("fasttrap: mishandled a branch"); - } - - if (taken) { - pc = rp->r_npc; - npc = tp->ftt_dest; - } else if (tp->ftt_flags & FASTTRAP_F_ANNUL) { - /* - * Untaken annulled branches don't execute the - * instruction in the delay slot. - */ - pc = rp->r_npc + 4; - npc = pc + 4; - } else { - pc = rp->r_npc; - npc = pc + 4; - } - break; - } - - case FASTTRAP_T_ALWAYS: - /* - * BAs, BA,As... - */ - - if (tp->ftt_flags & FASTTRAP_F_ANNUL) { - /* - * Annulled branch always instructions never execute - * the instruction in the delay slot. - */ - pc = tp->ftt_dest; - npc = tp->ftt_dest + 4; - } else { - pc = rp->r_npc; - npc = tp->ftt_dest; - } - break; - - case FASTTRAP_T_RDPC: - fasttrap_putreg(rp, RD(tp->ftt_instr), rp->r_pc); - pc = rp->r_npc; - npc = pc + 4; - break; - - case FASTTRAP_T_CALL: - /* - * It's a call _and_ link remember... - */ - rp->r_o7 = rp->r_pc; - pc = rp->r_npc; - npc = tp->ftt_dest; - break; - - case FASTTRAP_T_JMPL: - pc = rp->r_npc; - - if (I(tp->ftt_instr)) { - uint_t rs1 = RS1(tp->ftt_instr); - int32_t imm; - - imm = tp->ftt_instr << 19; - imm >>= 19; - npc = fasttrap_getreg(rp, rs1) + imm; - } else { - uint_t rs1 = RS1(tp->ftt_instr); - uint_t rs2 = RS2(tp->ftt_instr); - - npc = fasttrap_getreg(rp, rs1) + - fasttrap_getreg(rp, rs2); - } - - /* - * Do the link part of the jump-and-link instruction. - */ - fasttrap_putreg(rp, RD(tp->ftt_instr), rp->r_pc); - - break; - - case FASTTRAP_T_COMMON: - { - curthread->t_dtrace_scrpc = rp->r_g7; - curthread->t_dtrace_astpc = rp->r_g7 + FASTTRAP_OFF_FTRET; - - /* - * Copy the instruction to a reserved location in the - * user-land thread structure, then set the PC to that - * location and leave the NPC alone. We take pains to ensure - * consistency in the instruction stream (See SPARC - * Architecture Manual Version 9, sections 8.4.7, A.20, and - * H.1.6; UltraSPARC I/II User's Manual, sections 3.1.1.1, - * and 13.6.4) by using the ASI ASI_BLK_COMMIT_S to copy the - * instruction into the user's address space without - * bypassing the I$. There's no AS_USER version of this ASI - * (as exist for other ASIs) so we use the lofault - * mechanism to catch faults. - */ - if (dtrace_blksuword32(rp->r_g7, &tp->ftt_instr, 1) == -1) { - /* - * If the copyout fails, then the process's state - * is not consistent (the effects of the traced - * instruction will never be seen). This process - * cannot be allowed to continue execution. - */ - fasttrap_sigtrap(curproc, curthread, pc); - return (0); - } - - curthread->t_dtrace_pc = pc; - curthread->t_dtrace_npc = npc; - curthread->t_dtrace_on = 1; - - pc = curthread->t_dtrace_scrpc; - - if (tp->ftt_retids != NULL) { - curthread->t_dtrace_step = 1; - curthread->t_dtrace_ret = 1; - npc = curthread->t_dtrace_astpc; - } - break; - } - - default: - panic("fasttrap: mishandled an instruction"); - } - - /* - * This bit me in the ass a couple of times, so lets toss this - * in as a cursory sanity check. - */ - ASSERT(pc != rp->r_g7 + 4); - ASSERT(pc != rp->r_g7 + 8); - -done: - /* - * If there were no return probes when we first found the tracepoint, - * we should feel no obligation to honor any return probes that were - * subsequently enabled -- they'll just have to wait until the next - * time around. - */ - if (tp->ftt_retids != NULL) { - /* - * We need to wait until the results of the instruction are - * apparent before invoking any return probes. If this - * instruction was emulated we can just call - * fasttrap_return_common(); if it needs to be executed, we - * need to wait until we return to the kernel. - */ - if (tp->ftt_type != FASTTRAP_T_COMMON) { - fasttrap_return_common(rp, orig_pc, pid, fake_restore); - } else { - ASSERT(curthread->t_dtrace_ret != 0); - ASSERT(curthread->t_dtrace_pc == orig_pc); - ASSERT(curthread->t_dtrace_scrpc == rp->r_g7); - ASSERT(npc == curthread->t_dtrace_astpc); - } - } - - ASSERT(pc != 0); - rp->r_pc = pc; - rp->r_npc = npc; - - return (0); -} - -int -fasttrap_return_probe(struct regs *rp) -{ - proc_t *p = ttoproc(curthread); - pid_t pid; - uintptr_t pc = curthread->t_dtrace_pc; - uintptr_t npc = curthread->t_dtrace_npc; - - curthread->t_dtrace_pc = 0; - curthread->t_dtrace_npc = 0; - curthread->t_dtrace_scrpc = 0; - curthread->t_dtrace_astpc = 0; - - /* - * Treat a child created by a call to vfork(2) as if it were its - * parent. We know there's only one thread of control in such a - * process: this one. - */ - while (p->p_flag & SVFORK) { - p = p->p_parent; - } - - /* - * We set the %pc and %npc to their values when the traced - * instruction was initially executed so that it appears to - * dtrace_probe() that we're on the original instruction, and so that - * the user can't easily detect our complex web of lies. - * dtrace_return_probe() (our caller) will correctly set %pc and %npc - * after we return. - */ - rp->r_pc = pc; - rp->r_npc = npc; - - pid = p->p_pid; - fasttrap_return_common(rp, pc, pid, 0); - - return (0); -} - -int -fasttrap_tracepoint_install(proc_t *p, fasttrap_tracepoint_t *tp) -{ - fasttrap_instr_t instr = FASTTRAP_INSTR; - - if (uwrite(p, &instr, 4, tp->ftt_pc) != 0) - return (-1); - - return (0); -} - -int -fasttrap_tracepoint_remove(proc_t *p, fasttrap_tracepoint_t *tp) -{ - fasttrap_instr_t instr; - - /* - * Distinguish between read or write failures and a changed - * instruction. - */ - if (uread(p, &instr, 4, tp->ftt_pc) != 0) - return (0); - if (instr != FASTTRAP_INSTR && instr != BREAKPOINT_INSTR) - return (0); - if (uwrite(p, &tp->ftt_instr, 4, tp->ftt_pc) != 0) - return (-1); - - return (0); -} - -int -fasttrap_tracepoint_init(proc_t *p, fasttrap_tracepoint_t *tp, uintptr_t pc, - fasttrap_probe_type_t type) -{ - uint32_t instr; - int32_t disp; - - /* - * Read the instruction at the given address out of the process's - * address space. We don't have to worry about a debugger - * changing this instruction before we overwrite it with our trap - * instruction since P_PR_LOCK is set. - */ - if (uread(p, &instr, 4, pc) != 0) - return (-1); - - /* - * Decode the instruction to fill in the probe flags. We can have - * the process execute most instructions on its own using a pc/npc - * trick, but pc-relative control transfer present a problem since - * we're relocating the instruction. We emulate these instructions - * in the kernel. We assume a default type and over-write that as - * needed. - * - * pc-relative instructions must be emulated for correctness; - * other instructions (which represent a large set of commonly traced - * instructions) are emulated or otherwise optimized for performance. - */ - tp->ftt_type = FASTTRAP_T_COMMON; - if (OP(instr) == 1) { - /* - * Call instructions. - */ - tp->ftt_type = FASTTRAP_T_CALL; - disp = DISP30(instr) << 2; - tp->ftt_dest = pc + (intptr_t)disp; - - } else if (OP(instr) == 0) { - /* - * Branch instructions. - * - * Unconditional branches need careful attention when they're - * annulled: annulled unconditional branches never execute - * the instruction in the delay slot. - */ - switch (OP2(instr)) { - case OP2_ILLTRAP: - case 0x7: - /* - * The compiler may place an illtrap after a call to - * a function that returns a structure. In the case of - * a returned structure, the compiler places an illtrap - * whose const22 field is the size of the returned - * structure immediately following the delay slot of - * the call. To stay out of the way, we refuse to - * place tracepoints on top of illtrap instructions. - * - * This is one of the dumbest architectural decisions - * I've ever had to work around. - * - * We also identify the only illegal op2 value (See - * SPARC Architecture Manual Version 9, E.2 table 31). - */ - return (-1); - - case OP2_BPcc: - if (COND(instr) == 8) { - tp->ftt_type = FASTTRAP_T_ALWAYS; - } else { - /* - * Check for an illegal instruction. - */ - if (CC(instr) & 1) - return (-1); - tp->ftt_type = FASTTRAP_T_CCR; - tp->ftt_cc = CC(instr); - tp->ftt_code = COND(instr); - } - - if (A(instr) != 0) - tp->ftt_flags |= FASTTRAP_F_ANNUL; - - disp = DISP19(instr); - disp <<= 13; - disp >>= 11; - tp->ftt_dest = pc + (intptr_t)disp; - break; - - case OP2_Bicc: - if (COND(instr) == 8) { - tp->ftt_type = FASTTRAP_T_ALWAYS; - } else { - tp->ftt_type = FASTTRAP_T_CCR; - tp->ftt_cc = 0; - tp->ftt_code = COND(instr); - } - - if (A(instr) != 0) - tp->ftt_flags |= FASTTRAP_F_ANNUL; - - disp = DISP22(instr); - disp <<= 10; - disp >>= 8; - tp->ftt_dest = pc + (intptr_t)disp; - break; - - case OP2_BPr: - /* - * Check for an illegal instruction. - */ - if ((RCOND(instr) & 3) == 0) - return (-1); - - /* - * It's a violation of the v8plus ABI to use a - * register-predicated branch in a 32-bit app if - * the register used is an %l or an %i (%gs and %os - * are legit because they're not saved to the stack - * in 32-bit words when we take a trap). - */ - if (p->p_model == DATAMODEL_ILP32 && RS1(instr) >= 16) - return (-1); - - tp->ftt_type = FASTTRAP_T_REG; - if (A(instr) != 0) - tp->ftt_flags |= FASTTRAP_F_ANNUL; - disp = DISP16(instr); - disp <<= 16; - disp >>= 14; - tp->ftt_dest = pc + (intptr_t)disp; - tp->ftt_code = RCOND(instr); - break; - - case OP2_SETHI: - tp->ftt_type = FASTTRAP_T_SETHI; - break; - - case OP2_FBPfcc: - if (COND(instr) == 8) { - tp->ftt_type = FASTTRAP_T_ALWAYS; - } else { - tp->ftt_type = FASTTRAP_T_FCC; - tp->ftt_cc = CC(instr); - tp->ftt_code = COND(instr); - } - - if (A(instr) != 0) - tp->ftt_flags |= FASTTRAP_F_ANNUL; - - disp = DISP19(instr); - disp <<= 13; - disp >>= 11; - tp->ftt_dest = pc + (intptr_t)disp; - break; - - case OP2_FBfcc: - if (COND(instr) == 8) { - tp->ftt_type = FASTTRAP_T_ALWAYS; - } else { - tp->ftt_type = FASTTRAP_T_FCC; - tp->ftt_cc = 0; - tp->ftt_code = COND(instr); - } - - if (A(instr) != 0) - tp->ftt_flags |= FASTTRAP_F_ANNUL; - - disp = DISP22(instr); - disp <<= 10; - disp >>= 8; - tp->ftt_dest = pc + (intptr_t)disp; - break; - } - - } else if (OP(instr) == 2) { - switch (OP3(instr)) { - case OP3_RETURN: - tp->ftt_type = FASTTRAP_T_RETURN; - break; - - case OP3_JMPL: - tp->ftt_type = FASTTRAP_T_JMPL; - break; - - case OP3_RD: - if (RS1(instr) == 5) - tp->ftt_type = FASTTRAP_T_RDPC; - break; - - case OP3_SAVE: - /* - * We optimize for save instructions at function - * entry; see the comment in fasttrap_pid_probe() - * (near FASTTRAP_T_SAVE) for details. - */ - if (fasttrap_optimize_save != 0 && - type == DTFTP_ENTRY && - I(instr) == 1 && RD(instr) == R_SP) - tp->ftt_type = FASTTRAP_T_SAVE; - break; - - case OP3_RESTORE: - /* - * We optimize restore instructions at function - * return; see the comment in fasttrap_pid_probe() - * (near FASTTRAP_T_RESTORE) for details. - * - * rd must be an %o or %g register. - */ - if ((RD(instr) & 0x10) == 0) - tp->ftt_type = FASTTRAP_T_RESTORE; - break; - - case OP3_OR: - /* - * A large proportion of instructions in the delay - * slot of retl instructions are or's so we emulate - * these downstairs as an optimization. - */ - tp->ftt_type = FASTTRAP_T_OR; - break; - - case OP3_TCC: - /* - * Breakpoint instructions are effectively position- - * dependent since the debugger uses the %pc value - * to lookup which breakpoint was executed. As a - * result, we can't actually instrument breakpoints. - */ - if (SW_TRAP(instr) == ST_BREAKPOINT) - return (-1); - break; - - case 0x19: - case 0x1d: - case 0x29: - case 0x33: - case 0x3f: - /* - * Identify illegal instructions (See SPARC - * Architecture Manual Version 9, E.2 table 32). - */ - return (-1); - } - } else if (OP(instr) == 3) { - uint32_t op3 = OP3(instr); - - /* - * Identify illegal instructions (See SPARC Architecture - * Manual Version 9, E.2 table 33). - */ - if ((op3 & 0x28) == 0x28) { - if (op3 != OP3_PREFETCH && op3 != OP3_CASA && - op3 != OP3_PREFETCHA && op3 != OP3_CASXA) - return (-1); - } else { - if ((op3 & 0x0f) == 0x0c || (op3 & 0x3b) == 0x31) - return (-1); - } - } - - tp->ftt_instr = instr; - - /* - * We don't know how this tracepoint is going to be used, but in case - * it's used as part of a function return probe, we need to indicate - * whether it's always a return site or only potentially a return - * site. If it's part of a return probe, it's always going to be a - * return from that function if it's a restore instruction or if - * the previous instruction was a return. If we could reliably - * distinguish jump tables from return sites, this wouldn't be - * necessary. - */ - if (tp->ftt_type != FASTTRAP_T_RESTORE && - (uread(p, &instr, 4, pc - sizeof (instr)) != 0 || - !(OP(instr) == 2 && OP3(instr) == OP3_RETURN))) - tp->ftt_flags |= FASTTRAP_F_RETMAYBE; - - return (0); -} - -/*ARGSUSED*/ -uint64_t -fasttrap_pid_getarg(void *arg, dtrace_id_t id, void *parg, int argno, - int aframes) -{ - return (fasttrap_anarg(ttolwp(curthread)->lwp_regs, argno)); -} - -/*ARGSUSED*/ -uint64_t -fasttrap_usdt_getarg(void *arg, dtrace_id_t id, void *parg, int argno, - int aframes) -{ - return (fasttrap_anarg(ttolwp(curthread)->lwp_regs, argno)); -} - -static uint64_t fasttrap_getreg_fast_cnt; -static uint64_t fasttrap_getreg_mpcb_cnt; -static uint64_t fasttrap_getreg_slow_cnt; - -static ulong_t -fasttrap_getreg(struct regs *rp, uint_t reg) -{ - ulong_t value; - dtrace_icookie_t cookie; - struct machpcb *mpcb; - extern ulong_t dtrace_getreg_win(uint_t, uint_t); - - /* - * We have the %os and %gs in our struct regs, but if we need to - * snag a %l or %i we need to go scrounging around in the process's - * address space. - */ - if (reg == 0) - return (0); - - if (reg < 16) - return ((&rp->r_g1)[reg - 1]); - - /* - * Before we look at the user's stack, we'll check the register - * windows to see if the information we want is in there. - */ - cookie = dtrace_interrupt_disable(); - if (dtrace_getotherwin() > 0) { - value = dtrace_getreg_win(reg, 1); - dtrace_interrupt_enable(cookie); - - atomic_inc_64(&fasttrap_getreg_fast_cnt); - - return (value); - } - dtrace_interrupt_enable(cookie); - - /* - * First check the machpcb structure to see if we've already read - * in the register window we're looking for; if we haven't, (and - * we probably haven't) try to copy in the value of the register. - */ - /* LINTED - alignment */ - mpcb = (struct machpcb *)((caddr_t)rp - REGOFF); - - if (get_udatamodel() == DATAMODEL_NATIVE) { - struct frame *fr = (struct frame *)(rp->r_sp + STACK_BIAS); - - if (mpcb->mpcb_wbcnt > 0) { - struct rwindow *rwin = (void *)mpcb->mpcb_wbuf; - int i = mpcb->mpcb_wbcnt; - do { - i--; - if ((long)mpcb->mpcb_spbuf[i] != rp->r_sp) - continue; - - atomic_inc_64(&fasttrap_getreg_mpcb_cnt); - return (rwin[i].rw_local[reg - 16]); - } while (i > 0); - } - - if (fasttrap_fulword(&fr->fr_local[reg - 16], &value) != 0) - goto err; - } else { - struct frame32 *fr = - (struct frame32 *)(uintptr_t)(caddr32_t)rp->r_sp; - uint32_t *v32 = (uint32_t *)&value; - - if (mpcb->mpcb_wbcnt > 0) { - struct rwindow32 *rwin = (void *)mpcb->mpcb_wbuf; - int i = mpcb->mpcb_wbcnt; - do { - i--; - if ((long)mpcb->mpcb_spbuf[i] != rp->r_sp) - continue; - - atomic_inc_64(&fasttrap_getreg_mpcb_cnt); - return (rwin[i].rw_local[reg - 16]); - } while (i > 0); - } - - if (fasttrap_fuword32(&fr->fr_local[reg - 16], &v32[1]) != 0) - goto err; - - v32[0] = 0; - } - - atomic_inc_64(&fasttrap_getreg_slow_cnt); - return (value); - -err: - /* - * If the copy in failed, the process will be in a irrecoverable - * state, and we have no choice but to kill it. - */ - kern_psignal(ttoproc(curthread), SIGILL); - return (0); -} - -static uint64_t fasttrap_putreg_fast_cnt; -static uint64_t fasttrap_putreg_mpcb_cnt; -static uint64_t fasttrap_putreg_slow_cnt; - -static void -fasttrap_putreg(struct regs *rp, uint_t reg, ulong_t value) -{ - dtrace_icookie_t cookie; - struct machpcb *mpcb; - extern void dtrace_putreg_win(uint_t, ulong_t); - - if (reg == 0) - return; - - if (reg < 16) { - (&rp->r_g1)[reg - 1] = value; - return; - } - - /* - * If the user process is still using some register windows, we - * can just place the value in the correct window. - */ - cookie = dtrace_interrupt_disable(); - if (dtrace_getotherwin() > 0) { - dtrace_putreg_win(reg, value); - dtrace_interrupt_enable(cookie); - atomic_inc_64(&fasttrap_putreg_fast_cnt); - return; - } - dtrace_interrupt_enable(cookie); - - /* - * First see if there's a copy of the register window in the - * machpcb structure that we can modify; if there isn't try to - * copy out the value. If that fails, we try to create a new - * register window in the machpcb structure. While this isn't - * _precisely_ the intended use of the machpcb structure, it - * can't cause any problems since we know at this point in the - * code that all of the user's data have been flushed out of the - * register file (since %otherwin is 0). - */ - /* LINTED - alignment */ - mpcb = (struct machpcb *)((caddr_t)rp - REGOFF); - - if (get_udatamodel() == DATAMODEL_NATIVE) { - struct frame *fr = (struct frame *)(rp->r_sp + STACK_BIAS); - /* LINTED - alignment */ - struct rwindow *rwin = (struct rwindow *)mpcb->mpcb_wbuf; - - if (mpcb->mpcb_wbcnt > 0) { - int i = mpcb->mpcb_wbcnt; - do { - i--; - if ((long)mpcb->mpcb_spbuf[i] != rp->r_sp) - continue; - - rwin[i].rw_local[reg - 16] = value; - atomic_inc_64(&fasttrap_putreg_mpcb_cnt); - return; - } while (i > 0); - } - - if (fasttrap_sulword(&fr->fr_local[reg - 16], value) != 0) { - if (mpcb->mpcb_wbcnt >= MAXWIN || copyin(fr, - &rwin[mpcb->mpcb_wbcnt], sizeof (*rwin)) != 0) - goto err; - - rwin[mpcb->mpcb_wbcnt].rw_local[reg - 16] = value; - mpcb->mpcb_spbuf[mpcb->mpcb_wbcnt] = (caddr_t)rp->r_sp; - mpcb->mpcb_wbcnt++; - atomic_inc_64(&fasttrap_putreg_mpcb_cnt); - return; - } - } else { - struct frame32 *fr = - (struct frame32 *)(uintptr_t)(caddr32_t)rp->r_sp; - /* LINTED - alignment */ - struct rwindow32 *rwin = (struct rwindow32 *)mpcb->mpcb_wbuf; - uint32_t v32 = (uint32_t)value; - - if (mpcb->mpcb_wbcnt > 0) { - int i = mpcb->mpcb_wbcnt; - do { - i--; - if ((long)mpcb->mpcb_spbuf[i] != rp->r_sp) - continue; - - rwin[i].rw_local[reg - 16] = v32; - atomic_inc_64(&fasttrap_putreg_mpcb_cnt); - return; - } while (i > 0); - } - - if (fasttrap_suword32(&fr->fr_local[reg - 16], v32) != 0) { - if (mpcb->mpcb_wbcnt >= MAXWIN || copyin(fr, - &rwin[mpcb->mpcb_wbcnt], sizeof (*rwin)) != 0) - goto err; - - rwin[mpcb->mpcb_wbcnt].rw_local[reg - 16] = v32; - mpcb->mpcb_spbuf[mpcb->mpcb_wbcnt] = (caddr_t)rp->r_sp; - mpcb->mpcb_wbcnt++; - atomic_inc_64(&fasttrap_putreg_mpcb_cnt); - return; - } - } - - atomic_inc_64(&fasttrap_putreg_slow_cnt); - return; - -err: - /* - * If we couldn't record this register's value, the process is in an - * irrecoverable state and we have no choice but to euthanize it. - */ - kern_psignal(ttoproc(curthread), SIGILL); -} |
