1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
|
/*-
* Copyright (c) 2003 Peter Wemm.
* Copyright (c) 1990 The Regents of the University of California.
* All rights reserved.
*
* This code is derived from software contributed to Berkeley by
* William Jolitz.
*
* Redistribution and use in source and binary forms, with or without
* modification, are permitted provided that the following conditions
* are met:
* 1. Redistributions of source code must retain the above copyright
* notice, this list of conditions and the following disclaimer.
* 2. Redistributions in binary form must reproduce the above copyright
* notice, this list of conditions and the following disclaimer in the
* documentation and/or other materials provided with the distribution.
* 3. All advertising materials mentioning features or use of this software
* must display the following acknowledgement:
* This product includes software developed by the University of
* California, Berkeley and its contributors.
* 4. Neither the name of the University nor the names of its contributors
* may be used to endorse or promote products derived from this software
* without specific prior written permission.
*
* THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
* ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
* IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
* ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
* FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
* DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
* OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
* HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
* LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
* OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
* SUCH DAMAGE.
*
* $FreeBSD$
*/
#include <machine/asmacros.h>
#include "assym.s"
/*****************************************************************************/
/* Scheduling */
/*****************************************************************************/
.text
/*
* cpu_throw()
*
* This is the second half of cpu_swtch(). It is used when the current
* thread is either a dummy or slated to die, and we no longer care
* about its state. This is only a slight optimization and is probably
* not worth it anymore. Note that we need to clear the pm_active bits so
* we do need the old proc if it still exists.
* %rdi = oldtd
* %rsi = newtd
*/
ENTRY(cpu_throw)
movl PCPU(CPUID), %eax
testq %rdi,%rdi /* no thread? */
jz 1f
/* release bit from old pm_active */
movq TD_PROC(%rdi), %rdx /* oldtd->td_proc */
movq P_VMSPACE(%rdx), %rdx /* proc->p_vmspace */
#ifdef SMP
lock
#endif
btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */
1:
movq TD_PCB(%rsi),%rdx /* newtd->td_proc */
movq PCB_CR3(%rdx),%rdx
movq %rdx,%cr3 /* new address space */
/* set bit in new pm_active */
movq TD_PROC(%rsi),%rdx
movq P_VMSPACE(%rdx), %rdx
#ifdef SMP
lock
#endif
btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */
jmp sw1
/*
* cpu_switch(old, new)
*
* Save the current thread state, then select the next thread to run
* and load its state.
* %rdi = oldtd
* %rsi = newtd
*/
ENTRY(cpu_switch)
/* Switch to new thread. First, save context. */
#ifdef INVARIANTS
testq %rdi,%rdi /* no thread? */
jz badsw2 /* no, panic */
#endif
movq TD_PCB(%rdi),%r8
movq (%rsp),%rax /* Hardware registers */
movq %rax,PCB_RIP(%r8)
movq %rbx,PCB_RBX(%r8)
movq %rsp,PCB_RSP(%r8)
movq %rbp,PCB_RBP(%r8)
movq %r12,PCB_R12(%r8)
movq %r13,PCB_R13(%r8)
movq %r14,PCB_R14(%r8)
movq %r15,PCB_R15(%r8)
pushfq /* PSL */
popq PCB_RFLAGS(%r8)
/* Save userland %fs */
movl $MSR_FSBASE,%ecx
rdmsr
movl %eax,PCB_FSBASE(%r8)
movl %edx,PCB_FSBASE+4(%r8)
/* Save userland %gs */
movl $MSR_KGSBASE,%ecx
rdmsr
movl %eax,PCB_GSBASE(%r8)
movl %edx,PCB_GSBASE+4(%r8)
/* Save segment selector numbers */
movl %ds,PCB_DS(%r8)
movl %es,PCB_ES(%r8)
movl %fs,PCB_FS(%r8)
movl %gs,PCB_GS(%r8)
/* have we used fp, and need a save? */
cmpq %rdi,PCPU(FPCURTHREAD)
jne 1f
pushq %rdi
pushq %rsi
addq $PCB_SAVEFPU,%r8 /* h/w bugs make saving complicated */
movq %r8, %rdi
call fpusave /* do it in a big C function */
popq %rsi
popq %rdi
1:
/* Save is done. Now fire up new thread. Leave old vmspace. */
#ifdef INVARIANTS
testq %rsi,%rsi /* no thread? */
jz badsw3 /* no, panic */
#endif
movq TD_PCB(%rsi),%r8
movl PCPU(CPUID), %eax
/* switch address space */
movq PCB_CR3(%r8),%rdx
movq %rdx,%cr3 /* new address space */
/* Release bit from old pmap->pm_active */
movq TD_PROC(%rdi), %rdx /* oldproc */
movq P_VMSPACE(%rdx), %rdx
#ifdef SMP
lock
#endif
btrl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* clear old */
/* Set bit in new pmap->pm_active */
movq TD_PROC(%rsi),%rdx /* newproc */
movq P_VMSPACE(%rdx), %rdx
#ifdef SMP
lock
#endif
btsl %eax, VM_PMAP+PM_ACTIVE(%rdx) /* set new */
sw1:
/*
* At this point, we've switched address spaces and are ready
* to load up the rest of the next context.
*/
movq TD_PCB(%rsi),%r8
/* Restore segment selector numbers */
movl PCB_DS(%r8),%ds
movl PCB_ES(%r8),%es
movl PCB_FS(%r8),%fs
/* Restore userland %gs while preserving kernel gsbase */
movl $MSR_GSBASE,%ecx
rdmsr
movl PCB_GS(%r8),%gs
wrmsr
/* Restore userland %fs */
movl $MSR_FSBASE,%ecx
movl PCB_FSBASE(%r8),%eax
movl PCB_FSBASE+4(%r8),%edx
wrmsr
/* Restore userland %gs */
movl $MSR_KGSBASE,%ecx
movl PCB_GSBASE(%r8),%eax
movl PCB_GSBASE+4(%r8),%edx
wrmsr
/* Update the TSS_RSP0 pointer for the next interrupt */
movq PCPU(TSSP), %rax
addq $COMMON_TSS_RSP0, %rax
leaq -16(%r8), %rbx
movq %rbx, (%rax)
movq %rbx, PCPU(RSP0)
/* Restore context. */
movq PCB_RBX(%r8),%rbx
movq PCB_RSP(%r8),%rsp
movq PCB_RBP(%r8),%rbp
movq PCB_R12(%r8),%r12
movq PCB_R13(%r8),%r13
movq PCB_R14(%r8),%r14
movq PCB_R15(%r8),%r15
movq PCB_RIP(%r8),%rax
movq %rax,(%rsp)
pushq PCB_RFLAGS(%r8)
popfq
movq %r8, PCPU(CURPCB)
movq %rsi, PCPU(CURTHREAD) /* into next thread */
ret
#ifdef INVARIANTS
badsw1:
pushq %rax
pushq %rcx
pushq %rdx
pushq %rbx
pushq %rbp
pushq %rsi
pushq %rdi
pushq %r8
pushq %r9
pushq %r10
pushq %r11
pushq %r12
pushq %r13
pushq %r14
pushq %r15
pushq $sw0_1
call __panic
sw0_1: .asciz "cpu_throw: no newthread supplied"
badsw2:
pushq %rax
pushq %rcx
pushq %rdx
pushq %rbx
pushq %rbp
pushq %rsi
pushq %rdi
pushq %r8
pushq %r9
pushq %r10
pushq %r11
pushq %r12
pushq %r13
pushq %r14
pushq %r15
pushq $sw0_2
call __panic
sw0_2: .asciz "cpu_switch: no curthread supplied"
badsw3:
pushq %rax
pushq %rcx
pushq %rdx
pushq %rbx
pushq %rbp
pushq %rsi
pushq %rdi
pushq %r8
pushq %r9
pushq %r10
pushq %r11
pushq %r12
pushq %r13
pushq %r14
pushq %r15
pushq $sw0_3
call __panic
sw0_3: .asciz "cpu_switch: no newthread supplied"
#endif
noswitch: .asciz "cpu_switch: called!"
nothrow: .asciz "cpu_throw: called!"
/*
* savectx(pcb)
* Update pcb, saving current processor state.
*/
ENTRY(savectx)
/* Fetch PCB. */
movq %rdi,%rcx
/* Save caller's return address. */
movq (%rsp),%rax
movq %rax,PCB_RIP(%rcx)
movq %cr3,%rax
movq %rax,PCB_CR3(%rcx)
movq %rbx,PCB_RBX(%rcx)
movq %rsp,PCB_RSP(%rcx)
movq %rbp,PCB_RBP(%rcx)
movq %r12,PCB_R12(%rcx)
movq %r13,PCB_R13(%rcx)
movq %r14,PCB_R14(%rcx)
movq %r15,PCB_R15(%rcx)
pushfq
popq PCB_RFLAGS(%rcx)
/*
* If fpcurthread == NULL, then the fpu h/w state is irrelevant and the
* state had better already be in the pcb. This is true for forks
* but not for dumps (the old book-keeping with FP flags in the pcb
* always lost for dumps because the dump pcb has 0 flags).
*
* If fpcurthread != NULL, then we have to save the fpu h/w state to
* fpcurthread's pcb and copy it to the requested pcb, or save to the
* requested pcb and reload. Copying is easier because we would
* have to handle h/w bugs for reloading. We used to lose the
* parent's fpu state for forks by forgetting to reload.
*/
pushfq
cli
movq PCPU(FPCURTHREAD),%rax
testq %rax,%rax
je 1f
pushq %rcx
pushq %rax
movq TD_PCB(%rax),%rdi
leaq PCB_SAVEFPU(%rdi),%rdi
call fpusave
popq %rax
popq %rcx
movq $PCB_SAVEFPU_SIZE,%rdx /* arg 3 */
leaq PCB_SAVEFPU(%rcx),%rsi /* arg 2 */
movq %rax,%rdi /* arg 1 */
call bcopy
1:
popfq
ret
|