1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
|
/*
* CDDL HEADER START
*
* The contents of this file are subject to the terms of the
* Common Development and Distribution License (the "License").
* You may not use this file except in compliance with the License.
*
* You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
* or http://www.opensolaris.org/os/licensing.
* See the License for the specific language governing permissions
* and limitations under the License.
*
* When distributing Covered Code, include this CDDL HEADER in each
* file and include the License file at usr/src/OPENSOLARIS.LICENSE.
* If applicable, add the following below this CDDL HEADER, with the
* fields enclosed by brackets "[]" replaced with your own identifying
* information: Portions Copyright [yyyy] [name of copyright owner]
*
* CDDL HEADER END
*
* Portions Copyright 2008 John Birrell <jb@freebsd.org>
*
* $FreeBSD$
*
*/
/*
* Copyright 2007 Sun Microsystems, Inc. All rights reserved.
* Use is subject to license terms.
*/
#define _ASM
#include <machine/asmacros.h>
#include <sys/cpuvar_defs.h>
#include <sys/dtrace.h>
#include "assym.s"
#define INTR_POP \
MEXITCOUNT; \
movq TF_RDI(%rsp),%rdi; \
movq TF_RSI(%rsp),%rsi; \
movq TF_RDX(%rsp),%rdx; \
movq TF_RCX(%rsp),%rcx; \
movq TF_R8(%rsp),%r8; \
movq TF_R9(%rsp),%r9; \
movq TF_RAX(%rsp),%rax; \
movq TF_RBX(%rsp),%rbx; \
movq TF_RBP(%rsp),%rbp; \
movq TF_R10(%rsp),%r10; \
movq TF_R11(%rsp),%r11; \
movq TF_R12(%rsp),%r12; \
movq TF_R13(%rsp),%r13; \
movq TF_R14(%rsp),%r14; \
movq TF_R15(%rsp),%r15; \
testb $SEL_RPL_MASK,TF_CS(%rsp); \
jz 1f; \
cli; \
swapgs; \
1: addq $TF_RIP,%rsp;
ENTRY(dtrace_invop_start)
/*
* #BP traps with %rip set to the next address. We need to decrement
* the value to indicate the address of the int3 (0xcc) instruction
* that we substituted.
*/
movq TF_RIP(%rsp), %rdi
decq %rdi
movq TF_RSP(%rsp), %rsi
movq TF_RAX(%rsp), %rdx
pushq (%rsi)
movq %rsp, %rsi
call dtrace_invop
ALTENTRY(dtrace_invop_callsite)
addq $8, %rsp
cmpl $DTRACE_INVOP_PUSHL_EBP, %eax
je bp_push
cmpl $DTRACE_INVOP_LEAVE, %eax
je bp_leave
cmpl $DTRACE_INVOP_NOP, %eax
je bp_nop
cmpl $DTRACE_INVOP_RET, %eax
je bp_ret
/* When all else fails handle the trap in the usual way. */
jmpq *dtrace_invop_calltrap_addr
bp_push:
/*
* We must emulate a "pushq %rbp". To do this, we pull the stack
* down 8 bytes, and then store the base pointer.
*/
INTR_POP
subq $16, %rsp /* make room for %rbp */
pushq %rax /* push temp */
movq 24(%rsp), %rax /* load calling RIP */
movq %rax, 8(%rsp) /* store calling RIP */
movq 32(%rsp), %rax /* load calling CS */
movq %rax, 16(%rsp) /* store calling CS */
movq 40(%rsp), %rax /* load calling RFLAGS */
movq %rax, 24(%rsp) /* store calling RFLAGS */
movq 48(%rsp), %rax /* load calling RSP */
subq $8, %rax /* make room for %rbp */
movq %rax, 32(%rsp) /* store calling RSP */
movq 56(%rsp), %rax /* load calling SS */
movq %rax, 40(%rsp) /* store calling SS */
movq 32(%rsp), %rax /* reload calling RSP */
movq %rbp, (%rax) /* store %rbp there */
popq %rax /* pop off temp */
iretq /* return from interrupt */
/*NOTREACHED*/
bp_leave:
/*
* We must emulate a "leave", which is the same as a "movq %rbp, %rsp"
* followed by a "popq %rbp". This is quite a bit simpler on amd64
* than it is on i386 -- we can exploit the fact that the %rsp is
* explicitly saved to effect the pop without having to reshuffle
* the other data pushed for the trap.
*/
INTR_POP
pushq %rax /* push temp */
movq 8(%rsp), %rax /* load calling RIP */
movq %rax, 8(%rsp) /* store calling RIP */
movq (%rbp), %rax /* get new %rbp */
addq $8, %rbp /* adjust new %rsp */
movq %rbp, 32(%rsp) /* store new %rsp */
movq %rax, %rbp /* set new %rbp */
popq %rax /* pop off temp */
iretq /* return from interrupt */
/*NOTREACHED*/
bp_nop:
/* We must emulate a "nop". */
INTR_POP
iretq
/*NOTREACHED*/
bp_ret:
INTR_POP
pushq %rax /* push temp */
movq 32(%rsp), %rax /* load %rsp */
movq (%rax), %rax /* load calling RIP */
movq %rax, 8(%rsp) /* store calling RIP */
addq $8, 32(%rsp) /* adjust new %rsp */
popq %rax /* pop off temp */
iretq /* return from interrupt */
/*NOTREACHED*/
END(dtrace_invop_start)
/*
void dtrace_invop_init(void)
*/
ENTRY(dtrace_invop_init)
movq $dtrace_invop_start, dtrace_invop_jump_addr(%rip)
ret
END(dtrace_invop_init)
/*
void dtrace_invop_uninit(void)
*/
ENTRY(dtrace_invop_uninit)
movq $0, dtrace_invop_jump_addr(%rip)
ret
END(dtrace_invop_uninit)
/*
greg_t dtrace_getfp(void)
*/
ENTRY(dtrace_getfp)
movq %rbp, %rax
ret
END(dtrace_getfp)
/*
uint32_t
dtrace_cas32(uint32_t *target, uint32_t cmp, uint32_t new)
*/
ENTRY(dtrace_cas32)
movl %esi, %eax
lock
cmpxchgl %edx, (%rdi)
ret
END(dtrace_cas32)
/*
void *
dtrace_casptr(void *target, void *cmp, void *new)
*/
ENTRY(dtrace_casptr)
movq %rsi, %rax
lock
cmpxchgq %rdx, (%rdi)
ret
END(dtrace_casptr)
/*
uintptr_t
dtrace_caller(int aframes)
*/
ENTRY(dtrace_caller)
movq $-1, %rax
ret
END(dtrace_caller)
/*
void
dtrace_copy(uintptr_t src, uintptr_t dest, size_t size)
*/
ENTRY(dtrace_copy)
pushq %rbp
movq %rsp, %rbp
xchgq %rdi, %rsi /* make %rsi source, %rdi dest */
movq %rdx, %rcx /* load count */
repz /* repeat for count ... */
smovb /* move from %ds:rsi to %ed:rdi */
leave
ret
END(dtrace_copy)
/*
void
dtrace_copystr(uintptr_t uaddr, uintptr_t kaddr, size_t size,
volatile uint16_t *flags)
*/
ENTRY(dtrace_copystr)
pushq %rbp
movq %rsp, %rbp
0:
movb (%rdi), %al /* load from source */
movb %al, (%rsi) /* store to destination */
addq $1, %rdi /* increment source pointer */
addq $1, %rsi /* increment destination pointer */
subq $1, %rdx /* decrement remaining count */
cmpb $0, %al
je 2f
testq $0xfff, %rdx /* test if count is 4k-aligned */
jnz 1f /* if not, continue with copying */
testq $CPU_DTRACE_BADADDR, (%rcx) /* load and test dtrace flags */
jnz 2f
1:
cmpq $0, %rdx
jne 0b
2:
leave
ret
END(dtrace_copystr)
/*
uintptr_t
dtrace_fulword(void *addr)
*/
ENTRY(dtrace_fulword)
movq (%rdi), %rax
ret
END(dtrace_fulword)
/*
uint8_t
dtrace_fuword8_nocheck(void *addr)
*/
ENTRY(dtrace_fuword8_nocheck)
xorq %rax, %rax
movb (%rdi), %al
ret
END(dtrace_fuword8_nocheck)
/*
uint16_t
dtrace_fuword16_nocheck(void *addr)
*/
ENTRY(dtrace_fuword16_nocheck)
xorq %rax, %rax
movw (%rdi), %ax
ret
END(dtrace_fuword16_nocheck)
/*
uint32_t
dtrace_fuword32_nocheck(void *addr)
*/
ENTRY(dtrace_fuword32_nocheck)
xorq %rax, %rax
movl (%rdi), %eax
ret
END(dtrace_fuword32_nocheck)
/*
uint64_t
dtrace_fuword64_nocheck(void *addr)
*/
ENTRY(dtrace_fuword64_nocheck)
movq (%rdi), %rax
ret
END(dtrace_fuword64_nocheck)
/*
void
dtrace_probe_error(dtrace_state_t *state, dtrace_epid_t epid, int which,
int fault, int fltoffs, uintptr_t illval)
*/
ENTRY(dtrace_probe_error)
pushq %rbp
movq %rsp, %rbp
subq $0x8, %rsp
movq %r9, (%rsp)
movq %r8, %r9
movq %rcx, %r8
movq %rdx, %rcx
movq %rsi, %rdx
movq %rdi, %rsi
movl dtrace_probeid_error(%rip), %edi
call dtrace_probe
addq $0x8, %rsp
leave
ret
END(dtrace_probe_error)
/*
void
dtrace_membar_producer(void)
*/
ENTRY(dtrace_membar_producer)
rep; ret /* use 2 byte return instruction when branch target */
/* AMD Software Optimization Guide - Section 6.2 */
END(dtrace_membar_producer)
/*
void
dtrace_membar_consumer(void)
*/
ENTRY(dtrace_membar_consumer)
rep; ret /* use 2 byte return instruction when branch target */
/* AMD Software Optimization Guide - Section 6.2 */
END(dtrace_membar_consumer)
/*
dtrace_icookie_t
dtrace_interrupt_disable(void)
*/
ENTRY(dtrace_interrupt_disable)
pushfq
popq %rax
cli
ret
END(dtrace_interrupt_disable)
/*
void
dtrace_interrupt_enable(dtrace_icookie_t cookie)
*/
ENTRY(dtrace_interrupt_enable)
pushq %rdi
popfq
ret
END(dtrace_interrupt_enable)
|