diff options
Diffstat (limited to 'sys/amd64')
33 files changed, 825 insertions, 527 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index 5c68f81..8096d3c 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -48,10 +48,10 @@ IDTVEC(vec_name) ; \ movl $KPSEL,%eax ; \ mov %ax,%fs ; \ FAKE_MCOUNT(13*4(%esp)) ; \ - movl PCPU(CURPROC),%ebx ; \ - incl P_INTR_NESTING_LEVEL(%ebx) ; \ + movl PCPU(CURTHREAD),%ebx ; \ + incl TD_INTR_NESTING_LEVEL(%ebx) ; \ pushl intr_unit + (irq_num) * 4 ; \ - call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ + call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ addl $4, %esp ; \ movl $0, lapic+LA_EOI ; \ lock ; \ @@ -59,7 +59,7 @@ IDTVEC(vec_name) ; \ movl intr_countp + (irq_num) * 4, %eax ; \ lock ; \ incl (%eax) ; \ - decl P_INTR_NESTING_LEVEL(%ebx) ; \ + decl TD_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp doreti @@ -152,8 +152,8 @@ IDTVEC(vec_name) ; \ MASK_LEVEL_IRQ(irq_num) ; \ EOI_IRQ(irq_num) ; \ 0: ; \ - movl PCPU(CURPROC),%ebx ; \ - incl P_INTR_NESTING_LEVEL(%ebx) ; \ + movl PCPU(CURTHREAD),%ebx ; \ + incl TD_INTR_NESTING_LEVEL(%ebx) ; \ ; \ /* entry point used by doreti_unpend for HWIs. */ \ __CONCAT(Xresume,irq_num): ; \ @@ -162,7 +162,7 @@ __CONCAT(Xresume,irq_num): ; \ call sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ ; \ - decl P_INTR_NESTING_LEVEL(%ebx) ; \ + decl TD_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp doreti @@ -227,10 +227,10 @@ Xhardclock: movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ - movl PCPU(CURPROC),%ebx - incl P_INTR_NESTING_LEVEL(%ebx) + movl PCPU(CURTHREAD),%ebx + incl TD_INTR_NESTING_LEVEL(%ebx) call forwarded_hardclock - decl P_INTR_NESTING_LEVEL(%ebx) + decl TD_INTR_NESTING_LEVEL(%ebx) MEXITCOUNT jmp doreti @@ -252,10 +252,10 @@ Xstatclock: movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ FAKE_MCOUNT(13*4(%esp)) - movl PCPU(CURPROC),%ebx - incl P_INTR_NESTING_LEVEL(%ebx) + movl PCPU(CURTHREAD),%ebx + incl TD_INTR_NESTING_LEVEL(%ebx) call forwarded_statclock - decl P_INTR_NESTING_LEVEL(%ebx) + decl TD_INTR_NESTING_LEVEL(%ebx) MEXITCOUNT jmp doreti diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S index 0f2f7a8..075aa36 100644 --- a/sys/amd64/amd64/cpu_switch.S +++ b/sys/amd64/amd64/cpu_switch.S @@ -77,17 +77,17 @@ ENTRY(cpu_throw) ENTRY(cpu_switch) /* switch to new process. first, save context as needed */ - movl PCPU(CURPROC),%ecx + movl PCPU(CURTHREAD),%ecx /* if no process to save, don't bother */ testl %ecx,%ecx jz sw1 - - movl P_VMSPACE(%ecx), %edx + movl TD_PROC(%ecx), %eax + movl P_VMSPACE(%eax), %edx movl PCPU(CPUID), %eax btrl %eax, VM_PMAP+PM_ACTIVE(%edx) - movl P_ADDR(%ecx),%edx + movl TD_PCB(%ecx),%edx movl (%esp),%eax /* Hardware registers */ movl %eax,PCB_EIP(%edx) @@ -124,7 +124,7 @@ ENTRY(cpu_switch) #ifdef DEV_NPX /* have we used fp, and need a save? */ - cmpl %ecx,PCPU(NPXPROC) + cmpl %ecx,PCPU(NPXTHREAD) jne 1f addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */ pushl %edx @@ -133,7 +133,11 @@ ENTRY(cpu_switch) 1: #endif /* DEV_NPX */ +/*##########################################################################*/ +/*##########################################################################*/ +/*##########################################################################*/ /* save is done, now choose a new process */ + /* But still trashing space above the old "Top Of Stack".. */ sw1: #ifdef SMP @@ -143,17 +147,17 @@ sw1: cmpl $0,PCPU(CPUID) je 1f - movl PCPU(IDLEPROC), %eax - jmp sw1b + movl PCPU(IDLETHREAD), %eax + jmp sw1b /* Idle thread can run on any kernel context */ 1: #endif /* - * Choose a new process to schedule. chooseproc() returns idleproc + * Choose a new process to schedule. choosethread() returns idleproc * if it cannot find another process to run. */ sw1a: - call chooseproc /* trash ecx, edx, ret eax*/ + call choosethread /* trash ecx, edx, ret eax*/ #ifdef INVARIANTS testl %eax,%eax /* no process? */ @@ -163,15 +167,20 @@ sw1b: movl %eax,%ecx #ifdef INVARIANTS - cmpb $SRUN,P_STAT(%ecx) + movl TD_PROC(%ecx), %eax /* XXXKSE */ + cmpb $SRUN,P_STAT(%eax) jne badsw2 #endif - movl P_ADDR(%ecx),%edx + movl TD_PCB(%ecx),%edx #if defined(SWTCH_OPTIM_STATS) incl swtch_optim_stats #endif + +/*##########################################################################*/ +/*##########################################################################*/ +/*##########################################################################*/ /* switch address space */ movl %cr3,%ebx cmpl PCB_CR3(%edx),%ebx @@ -181,9 +190,8 @@ sw1b: incl tlb_flush_count #endif movl PCB_CR3(%edx),%ebx - movl %ebx,%cr3 + movl %ebx,%cr3 /* LOAD NEW PAGE TABLES */ 4: - movl PCPU(CPUID), %esi cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f @@ -191,12 +199,9 @@ sw1b: movl PCB_EXT(%edx), %edi /* new tss descriptor */ jmp 2f 1: - /* update common_tss.tss_esp0 pointer */ - movl %edx, %ebx /* pcb */ - addl $(UPAGES * PAGE_SIZE - 16), %ebx - movl %ebx, PCPU(COMMON_TSS) + TSS_ESP0 - + /* esp points to base of usable stack */ + movl %edx, PCPU(COMMON_TSS) + TSS_ESP0 /* stack is below pcb */ btrl %esi, private_tss jae 3f PCPU_ADDR(COMMON_TSSD, %edi) @@ -210,7 +215,9 @@ sw1b: movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ ltr %si 3: - movl P_VMSPACE(%ecx), %ebx + /* note in a vmspace that this cpu is using it */ + movl TD_PROC(%ecx),%eax /* get proc from thread XXXKSE */ + movl P_VMSPACE(%eax), %ebx /* get vmspace of proc */ movl PCPU(CPUID), %eax btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) @@ -233,22 +240,23 @@ sw1b: #endif /** GRAB_LOPRIO */ #endif /* SMP */ movl %edx, PCPU(CURPCB) - movl %ecx, PCPU(CURPROC) /* into next process */ + movl %ecx, PCPU(CURTHREAD) /* into next process */ #ifdef SMP /* XXX FIXME: we should be restoring the local APIC TPR */ #endif /* SMP */ - cmpl $0, PCB_USERLDT(%edx) - jnz 1f - movl _default_ldt,%eax - cmpl PCPU(CURRENTLDT),%eax - je 2f - lldt _default_ldt - movl %eax,PCPU(CURRENTLDT) + cmpl $0, PCB_USERLDT(%edx) /* if there is one */ + jnz 1f /* then use it */ + movl _default_ldt,%eax /* We will use the default */ + cmpl PCPU(CURRENTLDT),%eax /* check to see if already loaded */ + je 2f /* if so skip reload */ + lldt _default_ldt /* load the default... we trust it. */ + movl %eax,PCPU(CURRENTLDT) /* store what we have */ jmp 2f -1: pushl %edx - call set_user_ldt + +1: pushl %edx /* call a non-trusting routine */ + call set_user_ldt /* to check and load the ldt */ popl %edx 2: @@ -275,7 +283,7 @@ cpu_switch_load_gs: andl $0x0000fc00,%eax /* reserved bits */ pushl %ebx movl PCB_DR7(%edx),%ebx - andl $~0x0000fc00,%ebx + andl $~0x0000fc00,%ebx /* re-enable the restored watchpoints */ orl %ebx,%eax popl %ebx movl %eax,%dr7 @@ -322,25 +330,25 @@ ENTRY(savectx) #ifdef DEV_NPX /* - * If npxproc == NULL, then the npx h/w state is irrelevant and the + * If npxthread == NULL, then the npx h/w state is irrelevant and the * state had better already be in the pcb. This is true for forks * but not for dumps (the old book-keeping with FP flags in the pcb * always lost for dumps because the dump pcb has 0 flags). * - * If npxproc != NULL, then we have to save the npx h/w state to - * npxproc's pcb and copy it to the requested pcb, or save to the + * If npxthread != NULL, then we have to save the npx h/w state to + * npxthread's pcb and copy it to the requested pcb, or save to the * requested pcb and reload. Copying is easier because we would * have to handle h/w bugs for reloading. We used to lose the * parent's npx state for forks by forgetting to reload. */ pushfl cli - movl PCPU(NPXPROC),%eax + movl PCPU(NPXTHREAD),%eax testl %eax,%eax je 1f pushl %ecx - movl P_ADDR(%eax),%eax + movl TD_PCB(%eax),%eax leal PCB_SAVEFPU(%eax),%eax pushl %eax pushl %eax diff --git a/sys/amd64/amd64/db_interface.c b/sys/amd64/amd64/db_interface.c index a2e5018..7e78088 100644 --- a/sys/amd64/amd64/db_interface.c +++ b/sys/amd64/amd64/db_interface.c @@ -351,23 +351,24 @@ DB_SHOW_COMMAND(pcpu, db_show_pcpu) gd = GLOBALDATA; #endif db_printf("cpuid = %d\n", gd->gd_cpuid); - db_printf("curproc = "); - if (gd->gd_curproc != NULL) - db_printf("%p: pid %d \"%s\"\n", gd->gd_curproc, - gd->gd_curproc->p_pid, gd->gd_curproc->p_comm); + db_printf("curthread = "); + if (gd->gd_curthread != NULL) + db_printf("%p: pid %d \"%s\"\n", gd->gd_curthread, + gd->gd_curthread->td_proc->p_pid, gd->gd_curthread->td_proc->p_comm); else db_printf("none\n"); db_printf("curpcb = %p\n", gd->gd_curpcb); - db_printf("npxproc = "); - if (gd->gd_npxproc != NULL) - db_printf("%p: pid %d \"%s\"\n", gd->gd_npxproc, - gd->gd_npxproc->p_pid, gd->gd_npxproc->p_comm); + db_printf("npxthread = "); + if (gd->gd_npxthread != NULL) + db_printf("%p: pid %d \"%s\"\n", gd->gd_npxthread, + gd->gd_npxthread->td_proc->p_pid, gd->gd_npxthread->td_proc->p_comm); else db_printf("none\n"); - db_printf("idleproc = "); - if (gd->gd_idleproc != NULL) - db_printf("%p: pid %d \"%s\"\n", gd->gd_idleproc, - gd->gd_idleproc->p_pid, gd->gd_idleproc->p_comm); + db_printf("idlethread = "); + if (gd->gd_idlethread != NULL) + db_printf("%p: pid %d \"%s\"\n", gd->gd_idlethread, + gd->gd_idlethread->td_proc->p_pid, + gd->gd_idlethread->td_proc->p_comm); else db_printf("none\n"); diff --git a/sys/amd64/amd64/db_trace.c b/sys/amd64/amd64/db_trace.c index dc1e2f0..3a07050 100644 --- a/sys/amd64/amd64/db_trace.c +++ b/sys/amd64/amd64/db_trace.c @@ -291,13 +291,15 @@ db_stack_trace_cmd(addr, have_addr, count, modif) boolean_t first; struct pcb *pcb; struct proc *p; + struct thread *td; pid_t pid; if (count == -1) count = 1024; if (!have_addr) { - p = curproc; + td = curthread; + p = td->td_proc; frame = (struct i386_frame *)ddb_regs.tf_ebp; if (frame == NULL) frame = (struct i386_frame *)(ddb_regs.tf_esp - 4); @@ -310,8 +312,9 @@ db_stack_trace_cmd(addr, have_addr, count, modif) * The pcb for curproc is not valid at this point, * so fall back to the default case. */ - if (pid == curproc->p_pid) { - p = curproc; + if (pid == curthread->td_proc->p_pid) { + td = curthread; + p = td->td_proc; frame = (struct i386_frame *)ddb_regs.tf_ebp; if (frame == NULL) frame = (struct i386_frame *) @@ -333,7 +336,7 @@ db_stack_trace_cmd(addr, have_addr, count, modif) db_printf("pid %d swapped out\n", pid); return; } - pcb = &p->p_addr->u_pcb; + pcb = p->p_thread.td_pcb; /* XXXKSE */ frame = (struct i386_frame *)pcb->pcb_ebp; if (frame == NULL) frame = (struct i386_frame *) diff --git a/sys/amd64/amd64/fpu.c b/sys/amd64/amd64/fpu.c index 4fe3d04..84534ea 100644 --- a/sys/amd64/amd64/fpu.c +++ b/sys/amd64/amd64/fpu.c @@ -128,23 +128,23 @@ void stop_emulating __P((void)); #endif /* __GNUC__ */ #ifdef CPU_ENABLE_SSE -#define GET_FPU_CW(proc) \ +#define GET_FPU_CW(thread) \ (cpu_fxsr ? \ - (proc)->p_addr->u_pcb.pcb_save.sv_xmm.sv_env.en_cw : \ - (proc)->p_addr->u_pcb.pcb_save.sv_87.sv_env.en_cw) -#define GET_FPU_SW(proc) \ + (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_cw : \ + (thread)->td_pcb->pcb_save.sv_87.sv_env.en_cw) +#define GET_FPU_SW(thread) \ (cpu_fxsr ? \ - (proc)->p_addr->u_pcb.pcb_save.sv_xmm.sv_env.en_sw : \ - (proc)->p_addr->u_pcb.pcb_save.sv_87.sv_env.en_sw) + (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_sw : \ + (thread)->td_pcb->pcb_save.sv_87.sv_env.en_sw) #define GET_FPU_EXSW_PTR(pcb) \ (cpu_fxsr ? \ &(pcb)->pcb_save.sv_xmm.sv_ex_sw : \ &(pcb)->pcb_save.sv_87.sv_ex_sw) #else /* CPU_ENABLE_SSE */ -#define GET_FPU_CW(proc) \ - (proc->p_addr->u_pcb.pcb_save.sv_87.sv_env.en_cw) -#define GET_FPU_SW(proc) \ - (proc->p_addr->u_pcb.pcb_save.sv_87.sv_env.en_sw) +#define GET_FPU_CW(thread) \ + (thread->td_pcb->pcb_save.sv_87.sv_env.en_cw) +#define GET_FPU_SW(thread) \ + (thread->td_pcb->pcb_save.sv_87.sv_env.en_sw) #define GET_FPU_EXSW_PTR(pcb) \ (&(pcb)->pcb_save.sv_87.sv_ex_sw) #endif /* CPU_ENABLE_SSE */ @@ -241,7 +241,7 @@ static void npx_intr(dummy) void *dummy; { - struct proc *p; + struct thread *td; /* * The BUSY# latch must be cleared in all cases so that the next @@ -250,22 +250,22 @@ npx_intr(dummy) outb(0xf0, 0); /* - * npxproc is normally non-null here. In that case, schedule an + * npxthread is normally non-null here. In that case, schedule an * AST to finish the exception handling in the correct context - * (this interrupt may occur after the process has entered the + * (this interrupt may occur after the thread has entered the * kernel via a syscall or an interrupt). Otherwise, the npx - * state of the process that caused this interrupt must have been - * pushed to the process' pcb, and clearing of the busy latch + * state of the thread that caused this interrupt must have been + * pushed to the thread' pcb, and clearing of the busy latch * above has finished the (essentially null) handling of this * interrupt. Control will eventually return to the instruction * that caused it and it will repeat. We will eventually (usually * soon) win the race to handle the interrupt properly. */ - p = PCPU_GET(npxproc); - if (p != NULL) { - p->p_addr->u_pcb.pcb_flags |= PCB_NPXTRAP; + td = PCPU_GET(npxthread); + if (td != NULL) { + td->td_pcb->pcb_flags |= PCB_NPXTRAP; mtx_lock_spin(&sched_lock); - p->p_sflag |= PS_ASTPENDING; + td->td_kse->ke_flags |= KEF_ASTPENDING; mtx_unlock_spin(&sched_lock); } } @@ -570,7 +570,7 @@ npxinit(control) /* * fninit has the same h/w bugs as fnsave. Use the detoxified * fnsave to throw away any junk in the fpu. npxsave() initializes - * the fpu and sets npxproc = NULL as important side effects. + * the fpu and sets npxthread = NULL as important side effects. */ savecrit = critical_enter(); npxsave(&dummy); @@ -586,13 +586,13 @@ npxinit(control) * Free coprocessor (if we have it). */ void -npxexit(p) - struct proc *p; +npxexit(td) + struct thread *td; { critical_t savecrit; savecrit = critical_enter(); - if (p == PCPU_GET(npxproc)) + if (td == PCPU_GET(npxthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); critical_exit(savecrit); #ifdef NPX_DEBUG @@ -607,8 +607,9 @@ npxexit(p) */ if (masked_exceptions & 0x0d) log(LOG_ERR, - "pid %d (%s) exited with masked floating point exceptions 0x%02x\n", - p->p_pid, p->p_comm, masked_exceptions); + "pid %d (%s) exited with masked floating" + " point exceptions 0x%02x\n", + td->td_proc->p_pid, td->td_proc->p_comm, masked_exceptions); } #endif } @@ -809,8 +810,8 @@ npxtrap() u_long *exstat; if (!npx_exists) { - printf("npxtrap: npxproc = %p, curproc = %p, npx_exists = %d\n", - PCPU_GET(npxproc), curproc, npx_exists); + printf("npxtrap: npxthread = %p, curthread = %p, npx_exists = %d\n", + PCPU_GET(npxthread), curthread, npx_exists); panic("npxtrap from nowhere"); } savecrit = critical_enter(); @@ -820,18 +821,18 @@ npxtrap() * state to memory. Fetch the relevant parts of the state from * wherever they are. */ - if (PCPU_GET(npxproc) != curproc) { - control = GET_FPU_CW(curproc); - status = GET_FPU_SW(curproc); + if (PCPU_GET(npxthread) != curthread) { + control = GET_FPU_CW(curthread); + status = GET_FPU_SW(curthread); } else { fnstcw(&control); fnstsw(&status); } - exstat = GET_FPU_EXSW_PTR(&curproc->p_addr->u_pcb); + exstat = GET_FPU_EXSW_PTR(curthread->td_pcb); *exstat = status; - if (PCPU_GET(npxproc) != curproc) - GET_FPU_SW(curproc) &= ~0x80bf; + if (PCPU_GET(npxthread) != curthread) + GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); critical_exit(savecrit); @@ -841,7 +842,7 @@ npxtrap() /* * Implement device not available (DNA) exception * - * It would be better to switch FP context here (if curproc != npxproc) + * It would be better to switch FP context here (if curthread != npxthread) * and not necessarily for every context switch, but it is too hard to * access foreign pcb's. */ @@ -853,9 +854,9 @@ npxdna() if (!npx_exists) return (0); - if (PCPU_GET(npxproc) != NULL) { - printf("npxdna: npxproc = %p, curproc = %p\n", - PCPU_GET(npxproc), curproc); + if (PCPU_GET(npxthread) != NULL) { + printf("npxdna: npxthread = %p, curthread = %p\n", + PCPU_GET(npxthread), curthread); panic("npxdna"); } s = critical_enter(); @@ -863,7 +864,7 @@ npxdna() /* * Record new context early in case frstor causes an IRQ13. */ - PCPU_SET(npxproc, CURPROC); + PCPU_SET(npxthread, curthread); exstat = GET_FPU_EXSW_PTR(PCPU_GET(curpcb)); *exstat = 0; @@ -895,13 +896,13 @@ npxdna() * after the process has entered the kernel. It may even be delivered after * the fnsave here completes. A spurious IRQ13 for the fnsave is handled in * the same way as a very-late-arriving non-spurious IRQ13 from user mode: - * it is normally ignored at first because we set npxproc to NULL; it is + * it is normally ignored at first because we set npxthread to NULL; it is * normally retriggered in npxdna() after return to user mode. * * npxsave() must be called with interrupts disabled, so that it clears - * npxproc atomically with saving the state. We require callers to do the + * npxthread atomically with saving the state. We require callers to do the * disabling, since most callers need to disable interrupts anyway to call - * npxsave() atomically with checking npxproc. + * npxsave() atomically with checking npxthread. * * A previous version of npxsave() went to great lengths to excecute fnsave * with interrupts enabled in case executing it froze the CPU. This case @@ -917,7 +918,7 @@ npxsave(addr) fpusave(addr); start_emulating(); - PCPU_SET(npxproc, NULL); + PCPU_SET(npxthread, NULL); } static void diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index 0704db7..8f4be92 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -37,7 +37,7 @@ * $FreeBSD$ */ -#include "opt_upages.h" +#include "opt_kstack_pages.h" #include <sys/param.h> #include <sys/systm.h> @@ -50,6 +50,7 @@ #include <sys/mutex.h> #include <sys/socket.h> #include <sys/resourcevar.h> +#include <sys/user.h> /* XXX */ #ifdef KTR_PERCPU #include <sys/ktr.h> @@ -79,21 +80,31 @@ ASSYM(P_VMSPACE, offsetof(struct proc, p_vmspace)); ASSYM(VM_PMAP, offsetof(struct vmspace, vm_pmap)); ASSYM(PM_ACTIVE, offsetof(struct pmap, pm_active)); -ASSYM(P_ADDR, offsetof(struct proc, p_addr)); -ASSYM(P_INTR_NESTING_LEVEL, offsetof(struct proc, p_intr_nesting_level)); ASSYM(P_SFLAG, offsetof(struct proc, p_sflag)); ASSYM(P_STAT, offsetof(struct proc, p_stat)); -ASSYM(P_WCHAN, offsetof(struct proc, p_wchan)); +ASSYM(P_UAREA, offsetof(struct proc, p_uarea)); + +/*ASSYM(TD_STAT, offsetof(struct thread, td__stat));*/ +ASSYM(TD_FLAGS, offsetof(struct thread, td_flags)); +ASSYM(TD_WCHAN, offsetof(struct thread, td_wchan)); +ASSYM(TD_PCB, offsetof(struct thread, td_pcb)); +ASSYM(TD_KSE, offsetof(struct thread, td_kse)); +ASSYM(TD_PROC, offsetof(struct thread, td_proc)); +ASSYM(TD_INTR_NESTING_LEVEL, offsetof(struct thread, td_intr_nesting_level)); + +ASSYM(KE_FLAGS, offsetof(struct kse, ke_flags)); -ASSYM(PS_ASTPENDING, PS_ASTPENDING); -ASSYM(PS_NEEDRESCHED, PS_NEEDRESCHED); +ASSYM(KEF_ASTPENDING, KEF_ASTPENDING); +ASSYM(KEF_NEEDRESCHED, KEF_NEEDRESCHED); ASSYM(SSLEEP, SSLEEP); ASSYM(SRUN, SRUN); ASSYM(V_TRAP, offsetof(struct vmmeter, v_trap)); ASSYM(V_SYSCALL, offsetof(struct vmmeter, v_syscall)); ASSYM(V_INTR, offsetof(struct vmmeter, v_intr)); -ASSYM(UPAGES, UPAGES); +/* ASSYM(UPAGES, UPAGES);*/ +ASSYM(UAREA_PAGES, UAREA_PAGES); +ASSYM(KSTACK_PAGES, KSTACK_PAGES); ASSYM(PAGE_SIZE, PAGE_SIZE); ASSYM(NPTEPG, NPTEPG); ASSYM(NPDEPG, NPDEPG); @@ -133,9 +144,7 @@ ASSYM(PCB_SAVEFPU_SIZE, sizeof(union savefpu)); ASSYM(PCB_SAVE87_SIZE, sizeof(struct save87)); ASSYM(PCB_ONFAULT, offsetof(struct pcb, pcb_onfault)); -#ifdef SMP ASSYM(PCB_SIZE, sizeof(struct pcb)); -#endif ASSYM(TF_TRAPNO, offsetof(struct trapframe, tf_trapno)); ASSYM(TF_ERR, offsetof(struct trapframe, tf_err)); @@ -166,9 +175,9 @@ ASSYM(BI_ESYMTAB, offsetof(struct bootinfo, bi_esymtab)); ASSYM(BI_KERNEND, offsetof(struct bootinfo, bi_kernend)); ASSYM(GD_SIZEOF, sizeof(struct globaldata)); ASSYM(GD_PRVSPACE, offsetof(struct globaldata, gd_prvspace)); -ASSYM(GD_CURPROC, offsetof(struct globaldata, gd_curproc)); -ASSYM(GD_NPXPROC, offsetof(struct globaldata, gd_npxproc)); -ASSYM(GD_IDLEPROC, offsetof(struct globaldata, gd_idleproc)); +ASSYM(GD_CURTHREAD, offsetof(struct globaldata, gd_curthread)); +ASSYM(GD_NPXTHREAD, offsetof(struct globaldata, gd_npxthread)); +ASSYM(GD_IDLETHREAD, offsetof(struct globaldata, gd_idlethread)); ASSYM(GD_CURPCB, offsetof(struct globaldata, gd_curpcb)); ASSYM(GD_COMMON_TSS, offsetof(struct globaldata, gd_common_tss)); ASSYM(GD_SWITCHTIME, offsetof(struct globaldata, gd_switchtime)); diff --git a/sys/amd64/amd64/locore.S b/sys/amd64/amd64/locore.S index 97a1e16..ab3dad5 100644 --- a/sys/amd64/amd64/locore.S +++ b/sys/amd64/amd64/locore.S @@ -147,9 +147,11 @@ IdlePTD: .long 0 /* phys addr of kernel PTD */ #endif KPTphys: .long 0 /* phys addr of kernel page tables */ - .globl proc0paddr -proc0paddr: .long 0 /* address of proc 0 address space */ -p0upa: .long 0 /* phys addr of proc0's UPAGES */ + .globl proc0uarea, proc0kstack +proc0uarea: .long 0 /* address of proc 0 uarea space */ +proc0kstack: .long 0 /* address of proc 0 kstack space */ +p0upa: .long 0 /* phys addr of proc0's UAREA */ +p0kpa: .long 0 /* phys addr of proc0's STACK */ vm86phystk: .long 0 /* PA of vm86/bios stack */ @@ -369,13 +371,14 @@ NON_GPROF_ENTRY(btext) /* now running relocated at KERNBASE where the system is linked to run */ begin: /* set up bootstrap stack */ - movl proc0paddr,%eax /* location of in-kernel pages */ - leal UPAGES*PAGE_SIZE(%eax),%esp /* bootstrap stack end location */ + movl proc0kstack,%eax /* location of in-kernel stack */ + /* bootstrap stack end location */ + leal (KSTACK_PAGES*PAGE_SIZE-PCB_SIZE)(%eax),%esp xorl %ebp,%ebp /* mark end of frames */ movl IdlePTD,%esi - movl %esi,PCB_CR3(%eax) + movl %esi,(KSTACK_PAGES*PAGE_SIZE-PCB_SIZE+PCB_CR3)(%eax) testl $CPUID_PGE, R(cpu_feature) jz 1f @@ -762,10 +765,15 @@ no_kernend: movl %esi,R(IdlePTD) /* Allocate UPAGES */ - ALLOCPAGES(UPAGES) + ALLOCPAGES(UAREA_PAGES) movl %esi,R(p0upa) addl $KERNBASE, %esi - movl %esi, R(proc0paddr) + movl %esi, R(proc0uarea) + + ALLOCPAGES(KSTACK_PAGES) + movl %esi,R(p0kpa) + addl $KERNBASE, %esi + movl %esi, R(proc0kstack) ALLOCPAGES(1) /* vm86/bios stack */ movl %esi,R(vm86phystk) @@ -833,7 +841,12 @@ map_read_write: /* Map proc0's UPAGES in the physical way ... */ movl R(p0upa), %eax - movl $UPAGES, %ecx + movl $(UAREA_PAGES), %ecx + fillkptphys($PG_RW) + +/* Map proc0's KSTACK in the physical way ... */ + movl R(p0kpa), %eax + movl $(KSTACK_PAGES), %ecx fillkptphys($PG_RW) /* Map ISA hole */ diff --git a/sys/amd64/amd64/locore.s b/sys/amd64/amd64/locore.s index 97a1e16..ab3dad5 100644 --- a/sys/amd64/amd64/locore.s +++ b/sys/amd64/amd64/locore.s @@ -147,9 +147,11 @@ IdlePTD: .long 0 /* phys addr of kernel PTD */ #endif KPTphys: .long 0 /* phys addr of kernel page tables */ - .globl proc0paddr -proc0paddr: .long 0 /* address of proc 0 address space */ -p0upa: .long 0 /* phys addr of proc0's UPAGES */ + .globl proc0uarea, proc0kstack +proc0uarea: .long 0 /* address of proc 0 uarea space */ +proc0kstack: .long 0 /* address of proc 0 kstack space */ +p0upa: .long 0 /* phys addr of proc0's UAREA */ +p0kpa: .long 0 /* phys addr of proc0's STACK */ vm86phystk: .long 0 /* PA of vm86/bios stack */ @@ -369,13 +371,14 @@ NON_GPROF_ENTRY(btext) /* now running relocated at KERNBASE where the system is linked to run */ begin: /* set up bootstrap stack */ - movl proc0paddr,%eax /* location of in-kernel pages */ - leal UPAGES*PAGE_SIZE(%eax),%esp /* bootstrap stack end location */ + movl proc0kstack,%eax /* location of in-kernel stack */ + /* bootstrap stack end location */ + leal (KSTACK_PAGES*PAGE_SIZE-PCB_SIZE)(%eax),%esp xorl %ebp,%ebp /* mark end of frames */ movl IdlePTD,%esi - movl %esi,PCB_CR3(%eax) + movl %esi,(KSTACK_PAGES*PAGE_SIZE-PCB_SIZE+PCB_CR3)(%eax) testl $CPUID_PGE, R(cpu_feature) jz 1f @@ -762,10 +765,15 @@ no_kernend: movl %esi,R(IdlePTD) /* Allocate UPAGES */ - ALLOCPAGES(UPAGES) + ALLOCPAGES(UAREA_PAGES) movl %esi,R(p0upa) addl $KERNBASE, %esi - movl %esi, R(proc0paddr) + movl %esi, R(proc0uarea) + + ALLOCPAGES(KSTACK_PAGES) + movl %esi,R(p0kpa) + addl $KERNBASE, %esi + movl %esi, R(proc0kstack) ALLOCPAGES(1) /* vm86/bios stack */ movl %esi,R(vm86phystk) @@ -833,7 +841,12 @@ map_read_write: /* Map proc0's UPAGES in the physical way ... */ movl R(p0upa), %eax - movl $UPAGES, %ecx + movl $(UAREA_PAGES), %ecx + fillkptphys($PG_RW) + +/* Map proc0's KSTACK in the physical way ... */ + movl R(p0kpa), %eax + movl $(KSTACK_PAGES), %ecx fillkptphys($PG_RW) /* Map ISA hole */ diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index e0dade2..6711515 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -49,7 +49,7 @@ #include "opt_msgbuf.h" #include "opt_npx.h" #include "opt_perfmon.h" -#include "opt_upages.h" +#include "opt_kstack_pages.h" /* #include "opt_userconfig.h" */ #include <sys/param.h> @@ -289,14 +289,16 @@ osendsig(catcher, sig, mask, code) struct osigframe sf; struct osigframe *fp; struct proc *p; + struct thread *td; struct sigacts *psp; struct trapframe *regs; int oonstack; - p = curproc; + td = curthread; + p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); psp = p->p_sigacts; - regs = p->p_frame; + regs = td->td_frame; oonstack = sigonstack(regs->tf_esp); /* Allocate and validate space for the signal handler context. */ @@ -386,7 +388,7 @@ osendsig(catcher, sig, mask, code) if (regs->tf_eflags & PSL_VM) { /* XXX confusing names: `tf' isn't a trapframe; `regs' is. */ struct trapframe_vm86 *tf = (struct trapframe_vm86 *)regs; - struct vm86_kernel *vm86 = &p->p_addr->u_pcb.pcb_ext->ext_vm86; + struct vm86_kernel *vm86 = &td->td_pcb->pcb_ext->ext_vm86; sf.sf_siginfo.si_sc.sc_gs = tf->tf_vm86_gs; sf.sf_siginfo.si_sc.sc_fs = tf->tf_vm86_fs; @@ -409,7 +411,7 @@ osendsig(catcher, sig, mask, code) * ...Kill the process. */ PROC_LOCK(p); - sigexit(p, SIGILL); + sigexit(td, SIGILL); /* NOTREACHED */ } @@ -434,12 +436,14 @@ sendsig(catcher, sig, mask, code) { struct sigframe sf; struct proc *p; + struct thread *td; struct sigacts *psp; struct trapframe *regs; struct sigframe *sfp; int oonstack; - p = curproc; + td = curthread; + p = td->td_proc; PROC_LOCK_ASSERT(p, MA_OWNED); psp = p->p_sigacts; #ifdef COMPAT_43 @@ -448,7 +452,7 @@ sendsig(catcher, sig, mask, code) return; } #endif - regs = p->p_frame; + regs = td->td_frame; oonstack = sigonstack(regs->tf_esp); /* Save user context. */ @@ -528,7 +532,7 @@ sendsig(catcher, sig, mask, code) */ if (regs->tf_eflags & PSL_VM) { struct trapframe_vm86 *tf = (struct trapframe_vm86 *)regs; - struct vm86_kernel *vm86 = &p->p_addr->u_pcb.pcb_ext->ext_vm86; + struct vm86_kernel *vm86 = &td->td_pcb->pcb_ext->ext_vm86; sf.sf_uc.uc_mcontext.mc_gs = tf->tf_vm86_gs; sf.sf_uc.uc_mcontext.mc_fs = tf->tf_vm86_fs; @@ -561,7 +565,7 @@ sendsig(catcher, sig, mask, code) * ...Kill the process. */ PROC_LOCK(p); - sigexit(p, SIGILL); + sigexit(td, SIGILL); /* NOTREACHED */ } @@ -586,17 +590,18 @@ sendsig(catcher, sig, mask, code) */ #ifdef COMPAT_43 int -osigreturn(p, uap) - struct proc *p; +osigreturn(td, uap) + struct thread *td; struct osigreturn_args /* { struct osigcontext *sigcntxp; } */ *uap; { struct trapframe *regs; struct osigcontext *scp; + struct proc *p = td->td_proc; int eflags; - regs = p->p_frame; + regs = td->td_frame; scp = uap->sigcntxp; if (!useracc((caddr_t)scp, sizeof(*scp), VM_PROT_READ)) return (EFAULT); @@ -609,9 +614,9 @@ osigreturn(p, uap) * if pcb_ext == 0 or vm86_inited == 0, the user hasn't * set up the vm86 area, and we can't enter vm86 mode. */ - if (p->p_addr->u_pcb.pcb_ext == 0) + if (td->td_pcb->pcb_ext == 0) return (EINVAL); - vm86 = &p->p_addr->u_pcb.pcb_ext->ext_vm86; + vm86 = &td->td_pcb->pcb_ext->ext_vm86; if (vm86->vm86_inited == 0) return (EINVAL); @@ -697,12 +702,13 @@ osigreturn(p, uap) #endif int -sigreturn(p, uap) - struct proc *p; +sigreturn(td, uap) + struct thread *td; struct sigreturn_args /* { ucontext_t *sigcntxp; } */ *uap; { + struct proc *p = td->td_proc; struct trapframe *regs; ucontext_t *ucp; int cs, eflags; @@ -712,7 +718,7 @@ sigreturn(p, uap) if (!useracc((caddr_t)ucp, sizeof(struct osigcontext), VM_PROT_READ)) return (EFAULT); if (((struct osigcontext *)ucp)->sc_trapno == 0x01d516) - return (osigreturn(p, (struct osigreturn_args *)uap)); + return (osigreturn(td, (struct osigreturn_args *)uap)); /* * Since ucp is not an osigcontext but a ucontext_t, we have to * check again if all of it is accessible. A ucontext_t is @@ -724,7 +730,7 @@ sigreturn(p, uap) if (!useracc((caddr_t)ucp, sizeof(*ucp), VM_PROT_READ)) return (EFAULT); - regs = p->p_frame; + regs = td->td_frame; eflags = ucp->uc_mcontext.mc_eflags; if (eflags & PSL_VM) { struct trapframe_vm86 *tf = (struct trapframe_vm86 *)regs; @@ -734,9 +740,9 @@ sigreturn(p, uap) * if pcb_ext == 0 or vm86_inited == 0, the user hasn't * set up the vm86 area, and we can't enter vm86 mode. */ - if (p->p_addr->u_pcb.pcb_ext == 0) + if (td->td_pcb->pcb_ext == 0) return (EINVAL); - vm86 = &p->p_addr->u_pcb.pcb_ext->ext_vm86; + vm86 = &td->td_pcb->pcb_ext->ext_vm86; if (vm86->vm86_inited == 0) return (EINVAL); @@ -865,14 +871,14 @@ cpu_idle(void) * Clear registers on exec */ void -setregs(p, entry, stack, ps_strings) - struct proc *p; +setregs(td, entry, stack, ps_strings) + struct thread *td; u_long entry; u_long stack; u_long ps_strings; { - struct trapframe *regs = p->p_frame; - struct pcb *pcb = &p->p_addr->u_pcb; + struct trapframe *regs = td->td_frame; + struct pcb *pcb = td->td_pcb; if (pcb->pcb_ldt) user_ldt_free(pcb); @@ -925,7 +931,7 @@ setregs(p, entry, stack, ps_strings) * traps to the emulator (if it is done at all) mainly because * emulators don't provide an entry point for initialization. */ - p->p_addr->u_pcb.pcb_flags &= ~FP_SOFTFP; + td->td_pcb->pcb_flags &= ~FP_SOFTFP; /* * Arrange to trap the next npx or `fwait' instruction (see npx.c @@ -948,7 +954,7 @@ setregs(p, entry, stack, ps_strings) * Make sure sure edx is 0x0 on entry. Linux binaries depend * on it. */ - p->p_retval[1] = 0; + td->td_retval[1] = 0; } void @@ -1016,7 +1022,8 @@ extern int has_f00f_bug; static struct i386tss dblfault_tss; static char dblfault_stack[PAGE_SIZE]; -extern struct user *proc0paddr; +extern struct user *proc0uarea; +extern vm_offset_t proc0kstack; /* software prototypes -- in more palatable form */ @@ -1661,8 +1668,12 @@ init386(first) struct region_descriptor r_gdt, r_idt; #endif - proc0.p_addr = proc0paddr; - + proc_linkup(&proc0); + proc0.p_uarea = proc0uarea; + thread0 = &proc0.p_thread; + thread0->td_kstack = proc0kstack; + thread0->td_pcb = (struct pcb *) + (thread0->td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1; atdevbase = ISA_HOLE_START + KERNBASE; metadata_missing = 0; @@ -1721,10 +1732,11 @@ init386(first) lgdt(&r_gdt); /* setup curproc so that mutexes work */ - PCPU_SET(curproc, &proc0); + + PCPU_SET(curthread, thread0); PCPU_SET(spinlocks, NULL); - LIST_INIT(&proc0.p_contested); + LIST_INIT(&thread0->td_contested); /* * Initialize mutexes. @@ -1828,8 +1840,9 @@ init386(first) initializecpu(); /* Initialize CPU registers */ /* make an initial tss so cpu can get interrupt stack on syscall! */ - PCPU_SET(common_tss.tss_esp0, - (int) proc0.p_addr + UPAGES*PAGE_SIZE - 16); + /* Note: -16 is so we can grow the trapframe if we came from vm86 */ + PCPU_SET(common_tss.tss_esp0, thread0->td_kstack + + KSTACK_PAGES * PAGE_SIZE - sizeof(struct pcb) - 16); PCPU_SET(common_tss.tss_ss0, GSEL(GDATA_SEL, SEL_KPL)); gsel_tss = GSEL(GPROC0_SEL, SEL_KPL); private_tss = 0; @@ -1884,10 +1897,10 @@ init386(first) _udatasel = LSEL(LUDATA_SEL, SEL_UPL); /* setup proc 0's pcb */ - proc0.p_addr->u_pcb.pcb_flags = 0; - proc0.p_addr->u_pcb.pcb_cr3 = (int)IdlePTD; - proc0.p_addr->u_pcb.pcb_ext = 0; - proc0.p_frame = &proc0_tf; + thread0->td_pcb->pcb_flags = 0; /* XXXKSE */ + thread0->td_pcb->pcb_cr3 = (int)IdlePTD; + thread0->td_pcb->pcb_ext = 0; + thread0->td_frame = &proc0_tf; } #if defined(I586_CPU) && !defined(NO_F00F_HACK) @@ -1930,31 +1943,26 @@ f00f_hack(void *unused) { #endif /* defined(I586_CPU) && !NO_F00F_HACK */ int -ptrace_set_pc(p, addr) - struct proc *p; - unsigned long addr; +ptrace_set_pc(struct thread *td, unsigned long addr) { - p->p_frame->tf_eip = addr; + td->td_frame->tf_eip = addr; return (0); } int -ptrace_single_step(p) - struct proc *p; +ptrace_single_step(struct thread *td) { - p->p_frame->tf_eflags |= PSL_T; + td->td_frame->tf_eflags |= PSL_T; return (0); } int -fill_regs(p, regs) - struct proc *p; - struct reg *regs; +fill_regs(struct thread *td, struct reg *regs) { struct pcb *pcb; struct trapframe *tp; - tp = p->p_frame; + tp = td->td_frame; regs->r_fs = tp->tf_fs; regs->r_es = tp->tf_es; regs->r_ds = tp->tf_ds; @@ -1970,20 +1978,18 @@ fill_regs(p, regs) regs->r_eflags = tp->tf_eflags; regs->r_esp = tp->tf_esp; regs->r_ss = tp->tf_ss; - pcb = &p->p_addr->u_pcb; + pcb = td->td_pcb; regs->r_gs = pcb->pcb_gs; return (0); } int -set_regs(p, regs) - struct proc *p; - struct reg *regs; +set_regs(struct thread *td, struct reg *regs) { struct pcb *pcb; struct trapframe *tp; - tp = p->p_frame; + tp = td->td_frame; if (!EFL_SECURE(regs->r_eflags, tp->tf_eflags) || !CS_SECURE(regs->r_cs)) return (EINVAL); @@ -2002,7 +2008,7 @@ set_regs(p, regs) tp->tf_eflags = regs->r_eflags; tp->tf_esp = regs->r_esp; tp->tf_ss = regs->r_ss; - pcb = &p->p_addr->u_pcb; + pcb = td->td_pcb; pcb->pcb_gs = regs->r_gs; return (0); } @@ -2062,45 +2068,39 @@ set_fpregs_xmm(sv_87, sv_xmm) #endif /* CPU_ENABLE_SSE */ int -fill_fpregs(p, fpregs) - struct proc *p; - struct fpreg *fpregs; +fill_fpregs(struct thread *td, struct fpreg *fpregs) { #ifdef CPU_ENABLE_SSE if (cpu_fxsr) { - fill_fpregs_xmm(&p->p_addr->u_pcb.pcb_save.sv_xmm, + fill_fpregs_xmm(&td->td_pcb->pcb_save.sv_xmm, (struct save87 *)fpregs); return (0); } #endif /* CPU_ENABLE_SSE */ - bcopy(&p->p_addr->u_pcb.pcb_save.sv_87, fpregs, sizeof *fpregs); + bcopy(&td->td_pcb->pcb_save.sv_87, fpregs, sizeof *fpregs); return (0); } int -set_fpregs(p, fpregs) - struct proc *p; - struct fpreg *fpregs; +set_fpregs(struct thread *td, struct fpreg *fpregs) { #ifdef CPU_ENABLE_SSE if (cpu_fxsr) { set_fpregs_xmm((struct save87 *)fpregs, - &p->p_addr->u_pcb.pcb_save.sv_xmm); + &td->td_pcb->pcb_save.sv_xmm); return (0); } #endif /* CPU_ENABLE_SSE */ - bcopy(fpregs, &p->p_addr->u_pcb.pcb_save.sv_87, sizeof *fpregs); + bcopy(fpregs, &td->td_pcb->pcb_save.sv_87, sizeof *fpregs); return (0); } int -fill_dbregs(p, dbregs) - struct proc *p; - struct dbreg *dbregs; +fill_dbregs(struct thread *td, struct dbreg *dbregs) { struct pcb *pcb; - if (p == NULL) { + if (td == NULL) { dbregs->dr0 = rdr0(); dbregs->dr1 = rdr1(); dbregs->dr2 = rdr2(); @@ -2109,9 +2109,8 @@ fill_dbregs(p, dbregs) dbregs->dr5 = rdr5(); dbregs->dr6 = rdr6(); dbregs->dr7 = rdr7(); - } - else { - pcb = &p->p_addr->u_pcb; + } else { + pcb = td->td_pcb; dbregs->dr0 = pcb->pcb_dr0; dbregs->dr1 = pcb->pcb_dr1; dbregs->dr2 = pcb->pcb_dr2; @@ -2125,15 +2124,13 @@ fill_dbregs(p, dbregs) } int -set_dbregs(p, dbregs) - struct proc *p; - struct dbreg *dbregs; +set_dbregs(struct thread *td, struct dbreg *dbregs) { struct pcb *pcb; int i; u_int32_t mask1, mask2; - if (p == NULL) { + if (td == NULL) { load_dr0(dbregs->dr0); load_dr1(dbregs->dr1); load_dr2(dbregs->dr2); @@ -2142,8 +2139,7 @@ set_dbregs(p, dbregs) load_dr5(dbregs->dr5); load_dr6(dbregs->dr6); load_dr7(dbregs->dr7); - } - else { + } else { /* * Don't let an illegal value for dr7 get set. Specifically, * check for undefined settings. Setting these bit patterns @@ -2155,7 +2151,7 @@ set_dbregs(p, dbregs) if ((dbregs->dr7 & mask1) == mask2) return (EINVAL); - pcb = &p->p_addr->u_pcb; + pcb = td->td_pcb; /* * Don't let a process set a breakpoint that is not within the @@ -2172,7 +2168,7 @@ set_dbregs(p, dbregs) * from within kernel mode? */ - if (suser(p) != 0) { + if (suser_td(td) != 0) { if (dbregs->dr7 & 0x3) { /* dr0 is enabled */ if (dbregs->dr0 >= VM_MAXUSER_ADDRESS) diff --git a/sys/amd64/amd64/mem.c b/sys/amd64/amd64/mem.c index 5eded32..48ecff8 100644 --- a/sys/amd64/amd64/mem.c +++ b/sys/amd64/amd64/mem.c @@ -98,17 +98,17 @@ MALLOC_DEFINE(M_MEMDESC, "memdesc", "memory range descriptors"); struct mem_range_softc mem_range_softc; static int -mmclose(dev_t dev, int flags, int fmt, struct proc *p) +mmclose(dev_t dev, int flags, int fmt, struct thread *td) { switch (minor(dev)) { case 14: - p->p_frame->tf_eflags &= ~PSL_IOPL; + td->td_frame->tf_eflags &= ~PSL_IOPL; } return (0); } static int -mmopen(dev_t dev, int flags, int fmt, struct proc *p) +mmopen(dev_t dev, int flags, int fmt, struct thread *td) { int error; @@ -119,12 +119,12 @@ mmopen(dev_t dev, int flags, int fmt, struct proc *p) return (EPERM); break; case 14: - error = suser(p); + error = suser_td(td); if (error != 0) return (error); if (securelevel > 0) return (EPERM); - p->p_frame->tf_eflags |= PSL_IOPL; + td->td_frame->tf_eflags |= PSL_IOPL; break; } return (0); @@ -235,7 +235,7 @@ memmmap(dev_t dev, vm_offset_t offset, int prot) * and mem_range_attr_set. */ static int -mmioctl(dev_t dev, u_long cmd, caddr_t data, int flags, struct proc *p) +mmioctl(dev_t dev, u_long cmd, caddr_t data, int flags, struct thread *td) { int nd, error = 0; struct mem_range_op *mo = (struct mem_range_op *)data; diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index d912e0c..f558524 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -26,7 +26,7 @@ */ #include "opt_cpu.h" -#include "opt_upages.h" +#include "opt_kstack_pages.h" #ifdef SMP #include <machine/smptests.h> @@ -1960,8 +1960,8 @@ start_all_aps(u_int boot_addr) SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); /* allocate and set up an idle stack data page */ - stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); - for (i = 0; i < UPAGES; i++) + stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); /* XXXKSE */ + for (i = 0; i < KSTACK_PAGES; i++) SMPpt[pg + 1 + i] = (pt_entry_t) (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); @@ -1977,7 +1977,7 @@ start_all_aps(u_int boot_addr) outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif - bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE]; + bootSTK = &SMP_prvspace[x].idlekstack[KSTACK_PAGES * PAGE_SIZE]; bootAP = x; /* attempt to start the Application Processor */ @@ -2019,8 +2019,8 @@ start_all_aps(u_int boot_addr) */ /* Allocate and setup BSP idle stack */ - stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); - for (i = 0; i < UPAGES; i++) + stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); + for (i = 0; i < KSTACK_PAGES; i++) SMPpt[1 + i] = (pt_entry_t) (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); @@ -2241,7 +2241,7 @@ ap_init(void) * Set curproc to our per-cpu idleproc so that mutexes have * something unique to lock with. */ - PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(curthread, PCPU_GET(idlethread)); PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ @@ -2323,7 +2323,7 @@ forwarded_statclock(struct trapframe frame) { mtx_lock_spin(&sched_lock); - statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame)); + statclock_process(curthread->td_kse, TRAPF_PC(&frame), TRAPF_USERMODE(&frame)); mtx_unlock_spin(&sched_lock); } @@ -2354,7 +2354,7 @@ forwarded_hardclock(struct trapframe frame) { mtx_lock_spin(&sched_lock); - hardclock_process(curproc, TRAPF_USERMODE(&frame)); + hardclock_process(curthread, TRAPF_USERMODE(&frame)); mtx_unlock_spin(&sched_lock); } diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c index d912e0c..f558524 100644 --- a/sys/amd64/amd64/mptable.c +++ b/sys/amd64/amd64/mptable.c @@ -26,7 +26,7 @@ */ #include "opt_cpu.h" -#include "opt_upages.h" +#include "opt_kstack_pages.h" #ifdef SMP #include <machine/smptests.h> @@ -1960,8 +1960,8 @@ start_all_aps(u_int boot_addr) SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); /* allocate and set up an idle stack data page */ - stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); - for (i = 0; i < UPAGES; i++) + stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); /* XXXKSE */ + for (i = 0; i < KSTACK_PAGES; i++) SMPpt[pg + 1 + i] = (pt_entry_t) (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); @@ -1977,7 +1977,7 @@ start_all_aps(u_int boot_addr) outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif - bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE]; + bootSTK = &SMP_prvspace[x].idlekstack[KSTACK_PAGES * PAGE_SIZE]; bootAP = x; /* attempt to start the Application Processor */ @@ -2019,8 +2019,8 @@ start_all_aps(u_int boot_addr) */ /* Allocate and setup BSP idle stack */ - stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); - for (i = 0; i < UPAGES; i++) + stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); + for (i = 0; i < KSTACK_PAGES; i++) SMPpt[1 + i] = (pt_entry_t) (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); @@ -2241,7 +2241,7 @@ ap_init(void) * Set curproc to our per-cpu idleproc so that mutexes have * something unique to lock with. */ - PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(curthread, PCPU_GET(idlethread)); PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ @@ -2323,7 +2323,7 @@ forwarded_statclock(struct trapframe frame) { mtx_lock_spin(&sched_lock); - statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame)); + statclock_process(curthread->td_kse, TRAPF_PC(&frame), TRAPF_USERMODE(&frame)); mtx_unlock_spin(&sched_lock); } @@ -2354,7 +2354,7 @@ forwarded_hardclock(struct trapframe frame) { mtx_lock_spin(&sched_lock); - hardclock_process(curproc, TRAPF_USERMODE(&frame)); + hardclock_process(curthread, TRAPF_USERMODE(&frame)); mtx_unlock_spin(&sched_lock); } diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 3ce91ab..b41b11f 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -71,7 +71,7 @@ #include "opt_disable_pse.h" #include "opt_pmap.h" #include "opt_msgbuf.h" -#include "opt_upages.h" +#include "opt_kstack_pages.h" #include <sys/param.h> #include <sys/systm.h> @@ -171,7 +171,7 @@ vm_offset_t kernel_vm_end; static vm_zone_t pvzone; static struct vm_zone pvzone_store; static struct vm_object pvzone_obj; -static int pv_entry_count=0, pv_entry_max=0, pv_entry_high_water=0; +static int pv_entry_count = 0, pv_entry_max = 0, pv_entry_high_water = 0; static int pmap_pagedaemon_waken = 0; static struct pv_entry *pvinit; @@ -183,7 +183,7 @@ static pt_entry_t *CMAP2, *ptmmap; caddr_t CADDR1 = 0, ptvmmap = 0; static caddr_t CADDR2; static pt_entry_t *msgbufmap; -struct msgbuf *msgbufp=0; +struct msgbuf *msgbufp = 0; /* * Crashdump maps. @@ -819,7 +819,7 @@ retry: } /* - * Create the UPAGES for a new process. + * Create the Uarea stack for a new process. * This routine directly affects the fork perf for a process. */ void @@ -840,22 +840,22 @@ pmap_new_proc(p) */ upobj = p->p_upages_obj; if (upobj == NULL) { - upobj = vm_object_allocate(OBJT_DEFAULT, UPAGES); + upobj = vm_object_allocate(OBJT_DEFAULT, UAREA_PAGES); p->p_upages_obj = upobj; } - /* get a kernel virtual address for the UPAGES for this proc */ - up = (vm_offset_t)p->p_addr; + /* get a kernel virtual address for the U area for this proc */ + up = (vm_offset_t)p->p_uarea; if (up == 0) { - up = kmem_alloc_nofault(kernel_map, UPAGES * PAGE_SIZE); + up = kmem_alloc_nofault(kernel_map, UAREA_PAGES * PAGE_SIZE); if (up == 0) panic("pmap_new_proc: upage allocation failed"); - p->p_addr = (struct user *)up; + p->p_uarea = (struct user *)up; } ptek = (unsigned *)vtopte(up); - for (i = 0; i < UPAGES; i++) { + for (i = 0; i < UAREA_PAGES; i++) { /* * Get a kernel stack page */ @@ -892,7 +892,7 @@ pmap_new_proc(p) } /* - * Dispose the UPAGES for a process that has exited. + * Dispose the U-Area for a process that has exited. * This routine directly impacts the exit perf of a process. */ void @@ -906,9 +906,9 @@ pmap_dispose_proc(p) unsigned *ptek, oldpte; upobj = p->p_upages_obj; - up = (vm_offset_t)p->p_addr; + up = (vm_offset_t)p->p_uarea; ptek = (unsigned *)vtopte(up); - for (i = 0; i < UPAGES; i++) { + for (i = 0; i < UAREA_PAGES; i++) { m = vm_page_lookup(upobj, i); if (m == NULL) panic("pmap_dispose_proc: upage already missing?"); @@ -927,7 +927,7 @@ pmap_dispose_proc(p) } /* - * Allow the UPAGES for a process to be prejudicially paged out. + * Allow the U_AREA for a process to be prejudicially paged out. */ void pmap_swapout_proc(p) @@ -939,8 +939,8 @@ pmap_swapout_proc(p) vm_page_t m; upobj = p->p_upages_obj; - up = (vm_offset_t)p->p_addr; - for (i = 0; i < UPAGES; i++) { + up = (vm_offset_t)p->p_uarea; + for (i = 0; i < UAREA_PAGES; i++) { m = vm_page_lookup(upobj, i); if (m == NULL) panic("pmap_swapout_proc: upage already missing?"); @@ -951,7 +951,7 @@ pmap_swapout_proc(p) } /* - * Bring the UPAGES for a specified process back in. + * Bring the U-Area for a specified process back in. */ void pmap_swapin_proc(p) @@ -963,8 +963,8 @@ pmap_swapin_proc(p) vm_page_t m; upobj = p->p_upages_obj; - up = (vm_offset_t)p->p_addr; - for (i = 0; i < UPAGES; i++) { + up = (vm_offset_t)p->p_uarea; + for (i = 0; i < UAREA_PAGES; i++) { m = vm_page_grab(upobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); pmap_kenter(up + i * PAGE_SIZE, VM_PAGE_TO_PHYS(m)); if (m->valid != VM_PAGE_BITS_ALL) { @@ -980,6 +980,191 @@ pmap_swapin_proc(p) } } +/* + * Create the kernel stack (including pcb for i386) for a new thread. + * This routine directly affects the fork perf for a process and + * create performance for a thread. + */ +void +pmap_new_thread(td) + struct thread *td; +{ +#ifdef I386_CPU + int updateneeded = 0; +#endif + int i; + vm_object_t ksobj; + vm_page_t m; + vm_offset_t ks; + unsigned *ptek, oldpte; + + /* + * allocate object for the kstack + */ + ksobj = td->td_kstack_obj; + if (ksobj == NULL) { + ksobj = vm_object_allocate(OBJT_DEFAULT, KSTACK_PAGES); + td->td_kstack_obj = ksobj; + } + +#ifdef KSTACK_GUARD + /* get a kernel virtual address for the kstack for this proc */ + ks = (vm_offset_t)td->td_kstack; + if (ks == 0) { + ks = kmem_alloc_nofault(kernel_map, + (KSTACK_PAGES + 1) * PAGE_SIZE); + if (ks == 0) + panic("pmap_new_thread: kstack allocation failed"); + ks += PAGE_SIZE; + td->td_kstack = ks; + } + + ptek = (unsigned *)vtopte(ks - PAGE_SIZE); + oldpte = *ptek; + *ptek = 0; + if (oldpte) { +#ifdef I386_CPU + updateneeded = 1; +#else + invlpg(ks - PAGE_SIZE); +#endif + } + ptek++; +#else + /* get a kernel virtual address for the kstack for this proc */ + ks = (vm_offset_t)td->td_kstack; + if (ks == 0) { + ks = kmem_alloc_nofault(kernel_map, KSTACK_PAGES * PAGE_SIZE); + if (ks == 0) + panic("pmap_new_thread: kstack allocation failed"); + td->td_kstack = ks; + } +#endif + for (i = 0; i < KSTACK_PAGES; i++) { + /* + * Get a kernel stack page + */ + m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); + + /* + * Wire the page + */ + m->wire_count++; + cnt.v_wire_count++; + + oldpte = *(ptek + i); + /* + * Enter the page into the kernel address space. + */ + *(ptek + i) = VM_PAGE_TO_PHYS(m) | PG_RW | PG_V | pgeflag; + if (oldpte) { +#ifdef I386_CPU + updateneeded = 1; +#else + invlpg(ks + i * PAGE_SIZE); +#endif + } + + vm_page_wakeup(m); + vm_page_flag_clear(m, PG_ZERO); + vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); + m->valid = VM_PAGE_BITS_ALL; + } +#ifdef I386_CPU + if (updateneeded) + invltlb(); +#endif +} + +/* + * Dispose the kernel stack for a thread that has exited. + * This routine directly impacts the exit perf of a process and thread. + */ +void +pmap_dispose_thread(td) + struct thread *td; +{ + int i; + vm_object_t ksobj; + vm_offset_t ks; + vm_page_t m; + unsigned *ptek, oldpte; + + ksobj = td->td_kstack_obj; + ks = td->td_kstack; + ptek = (unsigned *)vtopte(ks); + for (i = 0; i < KSTACK_PAGES; i++) { + m = vm_page_lookup(ksobj, i); + if (m == NULL) + panic("pmap_dispose_thread: kstack already missing?"); + vm_page_busy(m); + oldpte = *(ptek + i); + *(ptek + i) = 0; +#ifndef I386_CPU + invlpg(ks + i * PAGE_SIZE); +#endif + vm_page_unwire(m, 0); + vm_page_free(m); + } +#ifdef I386_CPU + invltlb(); +#endif +} + +/* + * Allow the Kernel stack for a thread to be prejudicially paged out. + */ +void +pmap_swapout_thread(td) + struct thread *td; +{ + int i; + vm_object_t ksobj; + vm_offset_t ks; + vm_page_t m; + + ksobj = td->td_kstack_obj; + ks = td->td_kstack; + for (i = 0; i < KSTACK_PAGES; i++) { + m = vm_page_lookup(ksobj, i); + if (m == NULL) + panic("pmap_swapout_thread: kstack already missing?"); + vm_page_dirty(m); + vm_page_unwire(m, 0); + pmap_kremove(ks + i * PAGE_SIZE); + } +} + +/* + * Bring the kernel stack for a specified thread back in. + */ +void +pmap_swapin_thread(td) + struct thread *td; +{ + int i, rv; + vm_object_t ksobj; + vm_offset_t ks; + vm_page_t m; + + ksobj = td->td_kstack_obj; + ks = td->td_kstack; + for (i = 0; i < KSTACK_PAGES; i++) { + m = vm_page_grab(ksobj, i, VM_ALLOC_NORMAL | VM_ALLOC_RETRY); + pmap_kenter(ks + i * PAGE_SIZE, VM_PAGE_TO_PHYS(m)); + if (m->valid != VM_PAGE_BITS_ALL) { + rv = vm_pager_get_pages(ksobj, &m, 1, 0); + if (rv != VM_PAGER_OK) + panic("pmap_swapin_thread: cannot get kstack for proc: %d\n", td->td_proc->p_pid); + m = vm_page_lookup(ksobj, i); + m->valid = VM_PAGE_BITS_ALL; + } + vm_page_wire(m); + vm_page_wakeup(m); + vm_page_flag_set(m, PG_MAPPED | PG_WRITEABLE); + } +} + /*************************************************** * Page table page management routines..... ***************************************************/ @@ -1517,7 +1702,7 @@ pmap_collect() { int i; vm_page_t m; - static int warningdone=0; + static int warningdone = 0; if (pmap_pagedaemon_waken == 0) return; @@ -2333,7 +2518,7 @@ retry: pmap->pm_stats.resident_count += size >> PAGE_SHIFT; npdes = size >> PDRSHIFT; - for(i=0;i<npdes;i++) { + for(i = 0; i < npdes; i++) { pmap->pm_pdir[ptepindex] = (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V | PG_PS); ptepa += NBPDR; @@ -2444,7 +2629,7 @@ pmap_prefault(pmap, addra, entry) vm_page_t m, mpte; vm_object_t object; - if (!curproc || (pmap != vmspace_pmap(curproc->p_vmspace))) + if (!curthread || (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))) return; object = entry->object.vm_object; @@ -2561,6 +2746,7 @@ pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) vm_offset_t pdnxt; unsigned src_frame, dst_frame; vm_page_t m; + pd_entry_t saved_pde; if (dst_addr != src_addr) return; @@ -2580,7 +2766,7 @@ pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) invltlb(); #endif } - + saved_pde = (pd_entry_t)((u_int32_t)APTDpde & (PG_FRAME|PG_RW | PG_V)); for(addr = src_addr; addr < end_addr; addr = pdnxt) { unsigned *src_pte, *dst_pte; vm_page_t dstmpte, srcmpte; @@ -2637,6 +2823,16 @@ pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) * block. */ dstmpte = pmap_allocpte(dst_pmap, addr); + if (((u_int32_t)APTDpde & PG_FRAME) != + ((u_int32_t)saved_pde & PG_FRAME)) { + APTDpde = saved_pde; +printf ("IT HAPPENNED!"); +#if defined(SMP) + cpu_invltlb(); +#else + invltlb(); +#endif + } if ((*dst_pte == 0) && (ptetemp = *src_pte)) { /* * Clear the modified and @@ -2831,7 +3027,7 @@ pmap_remove_pages(pmap, sva, eva) vm_page_t m; #ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY - if (!curproc || (pmap != vmspace_pmap(curproc->p_vmspace))) { + if (!curthread || (pmap != vmspace_pmap(curthread->td_proc->p_vmspace))) { printf("warning: pmap_remove_pages called with non-current pmap\n"); return; } @@ -2839,8 +3035,8 @@ pmap_remove_pages(pmap, sva, eva) s = splvm(); for(pv = TAILQ_FIRST(&pmap->pm_pvlist); - pv; - pv = npv) { + pv; + pv = npv) { if (pv->pv_va >= eva || pv->pv_va < sva) { npv = TAILQ_NEXT(pv, pv_plist); @@ -2854,6 +3050,12 @@ pmap_remove_pages(pmap, sva, eva) #endif tpte = *pte; + if (tpte == 0) { + printf("TPTE at %p IS ZERO @ VA %08x\n", + pte, pv->pv_va); + panic("bad pte"); + } + /* * We cannot remove wired pages from a process' mapping at this time */ @@ -2861,15 +3063,19 @@ pmap_remove_pages(pmap, sva, eva) npv = TAILQ_NEXT(pv, pv_plist); continue; } - *pte = 0; m = PHYS_TO_VM_PAGE(tpte); + KASSERT(m->phys_addr == (tpte & PG_FRAME), + ("vm_page_t %p phys_addr mismatch %08x %08x", + m, m->phys_addr, tpte)); KASSERT(m < &vm_page_array[vm_page_array_size], ("pmap_remove_pages: bad tpte %x", tpte)); pv->pv_pmap->pm_stats.resident_count--; + *pte = 0; + /* * Update the vm_page_t clean and reference bits. */ @@ -2877,7 +3083,6 @@ pmap_remove_pages(pmap, sva, eva) vm_page_dirty(m); } - npv = TAILQ_NEXT(pv, pv_plist); TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist, pv, pv_plist); @@ -3255,11 +3460,13 @@ pmap_mincore(pmap, addr) } void -pmap_activate(struct proc *p) +pmap_activate(struct thread *td) { + struct proc *p = td->td_proc; pmap_t pmap; + u_int32_t cr3; - pmap = vmspace_pmap(p->p_vmspace); + pmap = vmspace_pmap(td->td_proc->p_vmspace); #if defined(SMP) pmap->pm_active |= 1 << PCPU_GET(cpuid); #else @@ -3268,7 +3475,20 @@ pmap_activate(struct proc *p) #if defined(SWTCH_OPTIM_STATS) tlb_flush_count++; #endif - load_cr3(p->p_addr->u_pcb.pcb_cr3 = vtophys(pmap->pm_pdir)); + cr3 = vtophys(pmap->pm_pdir); + /* XXXKSE this is wrong. + * pmap_activate is for the current thread on the current cpu + */ + if (p->p_flag & P_KSES) { + /* Make sure all other cr3 entries are updated. */ + /* what if they are running? XXXKSE (maybe abort them) */ + FOREACH_THREAD_IN_PROC(p, td) { + td->td_pcb->pcb_cr3 = cr3; + } + } else { + td->td_pcb->pcb_cr3 = cr3; + } + load_cr3(cr3); } vm_offset_t @@ -3301,14 +3521,14 @@ pmap_pid_dump(int pid) int i,j; index = 0; pmap = vmspace_pmap(p->p_vmspace); - for(i=0;i<1024;i++) { + for(i = 0; i < 1024; i++) { pd_entry_t *pde; unsigned *pte; unsigned base = i << PDRSHIFT; pde = &pmap->pm_pdir[i]; if (pde && pmap_pde_v(pde)) { - for(j=0;j<1024;j++) { + for(j = 0; j < 1024; j++) { unsigned va = base + (j << PAGE_SHIFT); if (va >= (vm_offset_t) VM_MIN_KERNEL_ADDRESS) { if (index) { diff --git a/sys/amd64/amd64/support.S b/sys/amd64/amd64/support.S index 55bc29c..81181cc 100644 --- a/sys/amd64/amd64/support.S +++ b/sys/amd64/amd64/support.S @@ -216,8 +216,8 @@ ENTRY(i586_bzero) * complicated since we avoid it if possible at all levels. We * want to localize the complications even when that increases them. * Here the extra work involves preserving CR0_TS in TS. - * `npxproc != NULL' is supposed to be the condition that all the - * FPU resources belong to an application, but npxproc and CR0_TS + * `npxthread != NULL' is supposed to be the condition that all the + * FPU resources belong to an application, but npxthread and CR0_TS * aren't set atomically enough for this condition to work in * interrupt handlers. * @@ -241,7 +241,7 @@ ENTRY(i586_bzero) * method. CR0_TS must be preserved although it is very likely to * always end up as clear. */ - cmpl $0,PCPU(NPXPROC) + cmpl $0,PCPU(NPXTHREAD) je i586_bz1 /* @@ -303,7 +303,7 @@ fpureg_i586_bzero_loop: cmpl $8,%ecx jae fpureg_i586_bzero_loop - cmpl $0,PCPU(NPXPROC) + cmpl $0,PCPU(NPXTHREAD) je i586_bz3 /* XXX check that the condition for cases 1-2 stayed false. */ @@ -517,7 +517,7 @@ ENTRY(i586_bcopy) sarb $1,kernel_fpu_lock jc small_i586_bcopy - cmpl $0,PCPU(NPXPROC) + cmpl $0,PCPU(NPXTHREAD) je i586_bc1 /* XXX turn off handling of cases 1-2, as above. */ @@ -593,7 +593,7 @@ large_i586_bcopy_loop: cmpl $64,%ecx jae 4b - cmpl $0,PCPU(NPXPROC) + cmpl $0,PCPU(NPXTHREAD) je i586_bc2 /* XXX check that the condition for cases 1-2 stayed false. */ @@ -991,14 +991,14 @@ ENTRY(fastmove) /* XXX grab FPU context atomically. */ cli -/* if (npxproc != NULL) { */ - cmpl $0,PCPU(NPXPROC) +/* if (npxthread != NULL) { */ + cmpl $0,PCPU(NPXTHREAD) je 6f /* fnsave(&curpcb->pcb_savefpu); */ movl PCPU(CURPCB),%eax fnsave PCB_SAVEFPU(%eax) -/* npxproc = NULL; */ - movl $0,PCPU(NPXPROC) +/* NPXTHREAD = NULL; */ + movl $0,PCPU(NPXTHREAD) /* } */ 6: /* now we own the FPU. */ @@ -1026,9 +1026,9 @@ ENTRY(fastmove) movl -4(%ebp),%edi /* stop_emulating(); */ clts -/* npxproc = curproc; */ - movl PCPU(CURPROC),%eax - movl %eax,PCPU(NPXPROC) +/* npxthread = curthread; */ + movl PCPU(CURTHREAD),%eax + movl %eax,PCPU(NPXTHREAD) movl PCPU(CURPCB),%eax /* XXX end of atomic FPU context grab. */ @@ -1113,8 +1113,8 @@ fastmove_loop: smsw %ax orb $CR0_TS,%al lmsw %ax -/* npxproc = NULL; */ - movl $0,PCPU(NPXPROC) +/* npxthread = NULL; */ + movl $0,PCPU(NPXTHREAD) /* XXX end of atomic FPU context ungrab. */ sti @@ -1154,7 +1154,7 @@ fastmove_fault: smsw %ax orb $CR0_TS,%al lmsw %ax - movl $0,PCPU(NPXPROC) + movl $0,PCPU(NPXTHREAD) /* XXX end of atomic FPU context ungrab. */ sti diff --git a/sys/amd64/amd64/support.s b/sys/amd64/amd64/support.s index 55bc29c..81181cc 100644 --- a/sys/amd64/amd64/support.s +++ b/sys/amd64/amd64/support.s @@ -216,8 +216,8 @@ ENTRY(i586_bzero) * complicated since we avoid it if possible at all levels. We * want to localize the complications even when that increases them. * Here the extra work involves preserving CR0_TS in TS. - * `npxproc != NULL' is supposed to be the condition that all the - * FPU resources belong to an application, but npxproc and CR0_TS + * `npxthread != NULL' is supposed to be the condition that all the + * FPU resources belong to an application, but npxthread and CR0_TS * aren't set atomically enough for this condition to work in * interrupt handlers. * @@ -241,7 +241,7 @@ ENTRY(i586_bzero) * method. CR0_TS must be preserved although it is very likely to * always end up as clear. */ - cmpl $0,PCPU(NPXPROC) + cmpl $0,PCPU(NPXTHREAD) je i586_bz1 /* @@ -303,7 +303,7 @@ fpureg_i586_bzero_loop: cmpl $8,%ecx jae fpureg_i586_bzero_loop - cmpl $0,PCPU(NPXPROC) + cmpl $0,PCPU(NPXTHREAD) je i586_bz3 /* XXX check that the condition for cases 1-2 stayed false. */ @@ -517,7 +517,7 @@ ENTRY(i586_bcopy) sarb $1,kernel_fpu_lock jc small_i586_bcopy - cmpl $0,PCPU(NPXPROC) + cmpl $0,PCPU(NPXTHREAD) je i586_bc1 /* XXX turn off handling of cases 1-2, as above. */ @@ -593,7 +593,7 @@ large_i586_bcopy_loop: cmpl $64,%ecx jae 4b - cmpl $0,PCPU(NPXPROC) + cmpl $0,PCPU(NPXTHREAD) je i586_bc2 /* XXX check that the condition for cases 1-2 stayed false. */ @@ -991,14 +991,14 @@ ENTRY(fastmove) /* XXX grab FPU context atomically. */ cli -/* if (npxproc != NULL) { */ - cmpl $0,PCPU(NPXPROC) +/* if (npxthread != NULL) { */ + cmpl $0,PCPU(NPXTHREAD) je 6f /* fnsave(&curpcb->pcb_savefpu); */ movl PCPU(CURPCB),%eax fnsave PCB_SAVEFPU(%eax) -/* npxproc = NULL; */ - movl $0,PCPU(NPXPROC) +/* NPXTHREAD = NULL; */ + movl $0,PCPU(NPXTHREAD) /* } */ 6: /* now we own the FPU. */ @@ -1026,9 +1026,9 @@ ENTRY(fastmove) movl -4(%ebp),%edi /* stop_emulating(); */ clts -/* npxproc = curproc; */ - movl PCPU(CURPROC),%eax - movl %eax,PCPU(NPXPROC) +/* npxthread = curthread; */ + movl PCPU(CURTHREAD),%eax + movl %eax,PCPU(NPXTHREAD) movl PCPU(CURPCB),%eax /* XXX end of atomic FPU context grab. */ @@ -1113,8 +1113,8 @@ fastmove_loop: smsw %ax orb $CR0_TS,%al lmsw %ax -/* npxproc = NULL; */ - movl $0,PCPU(NPXPROC) +/* npxthread = NULL; */ + movl $0,PCPU(NPXTHREAD) /* XXX end of atomic FPU context ungrab. */ sti @@ -1154,7 +1154,7 @@ fastmove_fault: smsw %ax orb $CR0_TS,%al lmsw %ax - movl $0,PCPU(NPXPROC) + movl $0,PCPU(NPXTHREAD) /* XXX end of atomic FPU context ungrab. */ sti diff --git a/sys/amd64/amd64/swtch.s b/sys/amd64/amd64/swtch.s index 0f2f7a8..075aa36 100644 --- a/sys/amd64/amd64/swtch.s +++ b/sys/amd64/amd64/swtch.s @@ -77,17 +77,17 @@ ENTRY(cpu_throw) ENTRY(cpu_switch) /* switch to new process. first, save context as needed */ - movl PCPU(CURPROC),%ecx + movl PCPU(CURTHREAD),%ecx /* if no process to save, don't bother */ testl %ecx,%ecx jz sw1 - - movl P_VMSPACE(%ecx), %edx + movl TD_PROC(%ecx), %eax + movl P_VMSPACE(%eax), %edx movl PCPU(CPUID), %eax btrl %eax, VM_PMAP+PM_ACTIVE(%edx) - movl P_ADDR(%ecx),%edx + movl TD_PCB(%ecx),%edx movl (%esp),%eax /* Hardware registers */ movl %eax,PCB_EIP(%edx) @@ -124,7 +124,7 @@ ENTRY(cpu_switch) #ifdef DEV_NPX /* have we used fp, and need a save? */ - cmpl %ecx,PCPU(NPXPROC) + cmpl %ecx,PCPU(NPXTHREAD) jne 1f addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */ pushl %edx @@ -133,7 +133,11 @@ ENTRY(cpu_switch) 1: #endif /* DEV_NPX */ +/*##########################################################################*/ +/*##########################################################################*/ +/*##########################################################################*/ /* save is done, now choose a new process */ + /* But still trashing space above the old "Top Of Stack".. */ sw1: #ifdef SMP @@ -143,17 +147,17 @@ sw1: cmpl $0,PCPU(CPUID) je 1f - movl PCPU(IDLEPROC), %eax - jmp sw1b + movl PCPU(IDLETHREAD), %eax + jmp sw1b /* Idle thread can run on any kernel context */ 1: #endif /* - * Choose a new process to schedule. chooseproc() returns idleproc + * Choose a new process to schedule. choosethread() returns idleproc * if it cannot find another process to run. */ sw1a: - call chooseproc /* trash ecx, edx, ret eax*/ + call choosethread /* trash ecx, edx, ret eax*/ #ifdef INVARIANTS testl %eax,%eax /* no process? */ @@ -163,15 +167,20 @@ sw1b: movl %eax,%ecx #ifdef INVARIANTS - cmpb $SRUN,P_STAT(%ecx) + movl TD_PROC(%ecx), %eax /* XXXKSE */ + cmpb $SRUN,P_STAT(%eax) jne badsw2 #endif - movl P_ADDR(%ecx),%edx + movl TD_PCB(%ecx),%edx #if defined(SWTCH_OPTIM_STATS) incl swtch_optim_stats #endif + +/*##########################################################################*/ +/*##########################################################################*/ +/*##########################################################################*/ /* switch address space */ movl %cr3,%ebx cmpl PCB_CR3(%edx),%ebx @@ -181,9 +190,8 @@ sw1b: incl tlb_flush_count #endif movl PCB_CR3(%edx),%ebx - movl %ebx,%cr3 + movl %ebx,%cr3 /* LOAD NEW PAGE TABLES */ 4: - movl PCPU(CPUID), %esi cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f @@ -191,12 +199,9 @@ sw1b: movl PCB_EXT(%edx), %edi /* new tss descriptor */ jmp 2f 1: - /* update common_tss.tss_esp0 pointer */ - movl %edx, %ebx /* pcb */ - addl $(UPAGES * PAGE_SIZE - 16), %ebx - movl %ebx, PCPU(COMMON_TSS) + TSS_ESP0 - + /* esp points to base of usable stack */ + movl %edx, PCPU(COMMON_TSS) + TSS_ESP0 /* stack is below pcb */ btrl %esi, private_tss jae 3f PCPU_ADDR(COMMON_TSSD, %edi) @@ -210,7 +215,9 @@ sw1b: movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ ltr %si 3: - movl P_VMSPACE(%ecx), %ebx + /* note in a vmspace that this cpu is using it */ + movl TD_PROC(%ecx),%eax /* get proc from thread XXXKSE */ + movl P_VMSPACE(%eax), %ebx /* get vmspace of proc */ movl PCPU(CPUID), %eax btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) @@ -233,22 +240,23 @@ sw1b: #endif /** GRAB_LOPRIO */ #endif /* SMP */ movl %edx, PCPU(CURPCB) - movl %ecx, PCPU(CURPROC) /* into next process */ + movl %ecx, PCPU(CURTHREAD) /* into next process */ #ifdef SMP /* XXX FIXME: we should be restoring the local APIC TPR */ #endif /* SMP */ - cmpl $0, PCB_USERLDT(%edx) - jnz 1f - movl _default_ldt,%eax - cmpl PCPU(CURRENTLDT),%eax - je 2f - lldt _default_ldt - movl %eax,PCPU(CURRENTLDT) + cmpl $0, PCB_USERLDT(%edx) /* if there is one */ + jnz 1f /* then use it */ + movl _default_ldt,%eax /* We will use the default */ + cmpl PCPU(CURRENTLDT),%eax /* check to see if already loaded */ + je 2f /* if so skip reload */ + lldt _default_ldt /* load the default... we trust it. */ + movl %eax,PCPU(CURRENTLDT) /* store what we have */ jmp 2f -1: pushl %edx - call set_user_ldt + +1: pushl %edx /* call a non-trusting routine */ + call set_user_ldt /* to check and load the ldt */ popl %edx 2: @@ -275,7 +283,7 @@ cpu_switch_load_gs: andl $0x0000fc00,%eax /* reserved bits */ pushl %ebx movl PCB_DR7(%edx),%ebx - andl $~0x0000fc00,%ebx + andl $~0x0000fc00,%ebx /* re-enable the restored watchpoints */ orl %ebx,%eax popl %ebx movl %eax,%dr7 @@ -322,25 +330,25 @@ ENTRY(savectx) #ifdef DEV_NPX /* - * If npxproc == NULL, then the npx h/w state is irrelevant and the + * If npxthread == NULL, then the npx h/w state is irrelevant and the * state had better already be in the pcb. This is true for forks * but not for dumps (the old book-keeping with FP flags in the pcb * always lost for dumps because the dump pcb has 0 flags). * - * If npxproc != NULL, then we have to save the npx h/w state to - * npxproc's pcb and copy it to the requested pcb, or save to the + * If npxthread != NULL, then we have to save the npx h/w state to + * npxthread's pcb and copy it to the requested pcb, or save to the * requested pcb and reload. Copying is easier because we would * have to handle h/w bugs for reloading. We used to lose the * parent's npx state for forks by forgetting to reload. */ pushfl cli - movl PCPU(NPXPROC),%eax + movl PCPU(NPXTHREAD),%eax testl %eax,%eax je 1f pushl %ecx - movl P_ADDR(%eax),%eax + movl TD_PCB(%eax),%eax leal PCB_SAVEFPU(%eax),%eax pushl %eax pushl %eax diff --git a/sys/amd64/amd64/sys_machdep.c b/sys/amd64/amd64/sys_machdep.c index 23c1cdd..acca4e7 100644 --- a/sys/amd64/amd64/sys_machdep.c +++ b/sys/amd64/amd64/sys_machdep.c @@ -35,7 +35,7 @@ * */ -#include "opt_upages.h" +#include "opt_kstack_pages.h" #include <sys/param.h> #include <sys/systm.h> @@ -65,10 +65,10 @@ -static int i386_get_ldt __P((struct proc *, char *)); -static int i386_set_ldt __P((struct proc *, char *)); -static int i386_get_ioperm __P((struct proc *, char *)); -static int i386_set_ioperm __P((struct proc *, char *)); +static int i386_get_ldt __P((struct thread *, char *)); +static int i386_set_ldt __P((struct thread *, char *)); +static int i386_get_ioperm __P((struct thread *, char *)); +static int i386_set_ioperm __P((struct thread *, char *)); #ifdef SMP static void set_user_ldt_rv __P((struct pcb *)); #endif @@ -81,28 +81,28 @@ struct sysarch_args { #endif int -sysarch(p, uap) - struct proc *p; +sysarch(td, uap) + struct thread *td; register struct sysarch_args *uap; { int error = 0; switch(uap->op) { case I386_GET_LDT: - error = i386_get_ldt(p, uap->parms); + error = i386_get_ldt(td, uap->parms); break; case I386_SET_LDT: - error = i386_set_ldt(p, uap->parms); + error = i386_set_ldt(td, uap->parms); break; case I386_GET_IOPERM: - error = i386_get_ioperm(p, uap->parms); + error = i386_get_ioperm(td, uap->parms); break; case I386_SET_IOPERM: - error = i386_set_ioperm(p, uap->parms); + error = i386_set_ioperm(td, uap->parms); break; case I386_VM86: - error = vm86_sysarch(p, uap->parms); + error = vm86_sysarch(td, uap->parms); break; default: error = EOPNOTSUPP; @@ -112,7 +112,7 @@ sysarch(p, uap) } int -i386_extend_pcb(struct proc *p) +i386_extend_pcb(struct thread *td) { int i, offset; u_long *addr; @@ -127,12 +127,18 @@ i386_extend_pcb(struct proc *p) 0, /* default 32 size */ 0 /* granularity */ }; + struct proc *p = td->td_proc; + if (td->td_proc->p_flag & P_KSES) + return (EINVAL); /* XXXKSE */ +/* XXXKSE All the code below only works in 1:1 needs changing */ ext = (struct pcb_ext *)kmem_alloc(kernel_map, ctob(IOPAGES+1)); if (ext == 0) return (ENOMEM); bzero(ext, sizeof(struct pcb_ext)); - ext->ext_tss.tss_esp0 = (unsigned)p->p_addr + ctob(UPAGES) - 16; + /* -16 is so we can convert a trapframe into vm86trapframe inplace */ + ext->ext_tss.tss_esp0 = td->td_kstack + ctob(KSTACK_PAGES) - + sizeof(struct pcb) - 16; ext->ext_tss.tss_ss0 = GSEL(GDATA_SEL, SEL_KPL); /* * The last byte of the i/o map must be followed by an 0xff byte. @@ -153,21 +159,21 @@ i386_extend_pcb(struct proc *p) ssd.ssd_limit -= ((unsigned)&ext->ext_tss - (unsigned)ext); ssdtosd(&ssd, &ext->ext_tssd); - KASSERT(p == curproc, ("giving a TSS to non-curproc")); - KASSERT(p->p_addr->u_pcb.pcb_ext == 0, ("already have a TSS!")); + KASSERT(p == curthread->td_proc, ("giving a TSS to non-curproc")); + KASSERT(td->td_pcb->pcb_ext == 0, ("already have a TSS!")); mtx_lock_spin(&sched_lock); - p->p_addr->u_pcb.pcb_ext = ext; + td->td_pcb->pcb_ext = ext; /* switch to the new TSS after syscall completes */ - p->p_sflag |= PS_NEEDRESCHED; + td->td_kse->ke_flags |= KEF_NEEDRESCHED; mtx_unlock_spin(&sched_lock); return 0; } static int -i386_set_ioperm(p, args) - struct proc *p; +i386_set_ioperm(td, args) + struct thread *td; char *args; { int i, error; @@ -177,7 +183,7 @@ i386_set_ioperm(p, args) if ((error = copyin(args, &ua, sizeof(struct i386_ioperm_args))) != 0) return (error); - if ((error = suser(p)) != 0) + if ((error = suser_td(td)) != 0) return (error); if (securelevel > 0) return (EPERM); @@ -188,10 +194,10 @@ i386_set_ioperm(p, args) * cause confusion. This probably requires a global 'usage registry'. */ - if (p->p_addr->u_pcb.pcb_ext == 0) - if ((error = i386_extend_pcb(p)) != 0) + if (td->td_pcb->pcb_ext == 0) + if ((error = i386_extend_pcb(td)) != 0) return (error); - iomap = (char *)p->p_addr->u_pcb.pcb_ext->ext_iomap; + iomap = (char *)td->td_pcb->pcb_ext->ext_iomap; if (ua.start + ua.length > IOPAGES * PAGE_SIZE * NBBY) return (EINVAL); @@ -206,8 +212,8 @@ i386_set_ioperm(p, args) } static int -i386_get_ioperm(p, args) - struct proc *p; +i386_get_ioperm(td, args) + struct thread *td; char *args; { int i, state, error; @@ -219,12 +225,12 @@ i386_get_ioperm(p, args) if (ua.start >= IOPAGES * PAGE_SIZE * NBBY) return (EINVAL); - if (p->p_addr->u_pcb.pcb_ext == 0) { + if (td->td_pcb->pcb_ext == 0) { ua.length = 0; goto done; } - iomap = (char *)p->p_addr->u_pcb.pcb_ext->ext_iomap; + iomap = (char *)td->td_pcb->pcb_ext->ext_iomap; i = ua.start; state = (iomap[i >> 3] >> (i & 7)) & 1; @@ -351,12 +357,12 @@ user_ldt_free(struct pcb *pcb) } static int -i386_get_ldt(p, args) - struct proc *p; +i386_get_ldt(td, args) + struct thread *td; char *args; { int error = 0; - struct pcb *pcb = &p->p_addr->u_pcb; + struct pcb *pcb = td->td_pcb; struct pcb_ldt *pcb_ldt = pcb->pcb_ldt; int nldt, num; union descriptor *lp; @@ -388,19 +394,19 @@ i386_get_ldt(p, args) error = copyout(lp, uap->descs, num * sizeof(union descriptor)); if (!error) - p->p_retval[0] = num; + td->td_retval[0] = num; return(error); } static int -i386_set_ldt(p, args) - struct proc *p; +i386_set_ldt(td, args) + struct thread *td; char *args; { int error = 0, i, n; int largest_ld; - struct pcb *pcb = &p->p_addr->u_pcb; + struct pcb *pcb = td->td_pcb; struct pcb_ldt *pcb_ldt = pcb->pcb_ldt; struct i386_ldt_args ua, *uap = &ua; caddr_t old_ldt_base; @@ -530,7 +536,7 @@ i386_set_ldt(p, args) &((union descriptor *)(pcb_ldt->ldt_base))[uap->start], uap->num * sizeof(union descriptor)); if (!error) - p->p_retval[0] = uap->start; + td->td_retval[0] = uap->start; critical_exit(savecrit); return(error); diff --git a/sys/amd64/amd64/trap.c b/sys/amd64/amd64/trap.c index 4e72f9b..ee036c2 100644 --- a/sys/amd64/amd64/trap.c +++ b/sys/amd64/amd64/trap.c @@ -173,7 +173,8 @@ void trap(frame) struct trapframe frame; { - struct proc *p = curproc; + struct thread *td = curthread; + struct proc *p = td->td_proc; u_int sticks = 0; int i = 0, ucode = 0, type, code; vm_offset_t eva; @@ -225,8 +226,8 @@ restart: ((frame.tf_eflags & PSL_VM) && !in_vm86call)) { /* user trap */ - sticks = p->p_sticks; - p->p_frame = &frame; + sticks = td->td_kse->ke_sticks; + td->td_frame = &frame; switch (type) { case T_PRIVINFLT: /* privileged instruction fault */ @@ -444,7 +445,7 @@ restart: if (in_vm86call) break; - if (p->p_intr_nesting_level != 0) + if (td->td_intr_nesting_level != 0) break; /* @@ -620,7 +621,7 @@ restart: #endif user: - userret(p, &frame, sticks); + userret(td, &frame, sticks); if (mtx_owned(&Giant)) /* XXX why would Giant be owned here? */ mtx_unlock(&Giant); out: @@ -660,7 +661,7 @@ trap_pfault(frame, usermode, eva) if (p == NULL || (!usermode && va < VM_MAXUSER_ADDRESS && - (p->p_intr_nesting_level != 0 || + (td->td_intr_nesting_level != 0 || PCPU_GET(curpcb) == NULL || PCPU_GET(curpcb)->pcb_onfault == NULL))) { trap_fatal(frame, eva); @@ -696,7 +697,7 @@ trap_pfault(frame, usermode, eva) * a growable stack region, or if the stack * growth succeeded. */ - if (!grow_stack (p, va)) + if (!grow_stack (td, va)) rv = KERN_FAILURE; else /* Fault in the user page: */ @@ -728,7 +729,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (p->p_intr_nesting_level == 0 && + if (td->td_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; @@ -756,7 +757,8 @@ trap_pfault(frame, usermode, eva) vm_map_t map = 0; int rv = 0; vm_prot_t ftype; - struct proc *p = curproc; + struct thread *td = curthread; + struct proc *p = td->td_proc; va = trunc_page(eva); if (va >= KERNBASE) { @@ -839,7 +841,7 @@ trap_pfault(frame, usermode, eva) return (0); nogo: if (!usermode) { - if (p->p_intr_nesting_level == 0 && + if (td->td_intr_nesting_level == 0 && PCPU_GET(curpcb) != NULL && PCPU_GET(curpcb)->pcb_onfault != NULL) { frame->tf_eip = (int)PCPU_GET(curpcb)->pcb_onfault; @@ -972,6 +974,7 @@ dblfault_handler() int trapwrite(addr) unsigned addr; { + struct thread *td; struct proc *p; vm_offset_t va; struct vmspace *vm; @@ -984,7 +987,8 @@ int trapwrite(addr) if (va >= VM_MAXUSER_ADDRESS) return (1); - p = curproc; + td = curthread; + p = td->td_proc; vm = p->p_vmspace; PROC_LOCK(p); @@ -1021,7 +1025,8 @@ syscall(frame) caddr_t params; int i; struct sysent *callp; - struct proc *p = curproc; + struct thread *td = curthread; + struct proc *p = td->td_proc; u_int sticks; int error; int narg; @@ -1039,8 +1044,8 @@ syscall(frame) } #endif - sticks = p->p_sticks; - p->p_frame = &frame; + sticks = td->td_kse->ke_sticks; + td->td_frame = &frame; params = (caddr_t)frame.tf_esp + sizeof(int); code = frame.tf_eax; @@ -1109,17 +1114,17 @@ syscall(frame) ktrsyscall(p->p_tracep, code, narg, args); } #endif - p->p_retval[0] = 0; - p->p_retval[1] = frame.tf_edx; + td->td_retval[0] = 0; + td->td_retval[1] = frame.tf_edx; STOPEVENT(p, S_SCE, narg); - error = (*callp->sy_call)(p, args); + error = (*callp->sy_call)(td, args); switch (error) { case 0: - frame.tf_eax = p->p_retval[0]; - frame.tf_edx = p->p_retval[1]; + frame.tf_eax = td->td_retval[0]; + frame.tf_edx = td->td_retval[1]; frame.tf_eflags &= ~PSL_C; break; @@ -1158,11 +1163,11 @@ bad: /* * Handle reschedule and other end-of-syscall issues */ - userret(p, &frame, sticks); + userret(td, &frame, sticks); #ifdef KTRACE if (KTRPOINT(p, KTR_SYSRET)) { - ktrsysret(p->p_tracep, code, error, p->p_retval[0]); + ktrsysret(p->p_tracep, code, error, td->td_retval[0]); } #endif @@ -1183,7 +1188,7 @@ bad: STOPEVENT(p, S_SCX, code); #ifdef WITNESS - if (witness_list(p)) { + if (witness_list(td)) { panic("system call %s returning with mutex(s) held\n", syscallnames[code]); } diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c index 53cedff..83ddc21 100644 --- a/sys/amd64/amd64/vm_machdep.c +++ b/sys/amd64/amd64/vm_machdep.c @@ -47,7 +47,7 @@ #endif #include "opt_reset.h" #include "opt_isa.h" -#include "opt_upages.h" +#include "opt_kstack_pages.h" #include <sys/param.h> #include <sys/systm.h> @@ -117,19 +117,24 @@ vm_fault_quick(v, prot) * ready to run and return to user mode. */ void -cpu_fork(p1, p2, flags) - register struct proc *p1, *p2; +cpu_fork(td1, p2, flags) + register struct thread *td1; + register struct proc *p2; int flags; { + register struct proc *p1; + struct thread *td2; struct pcb *pcb2; #ifdef DEV_NPX int savecrit; #endif + p1 = td1->td_proc; + td2 = &p2->p_thread; if ((flags & RFPROC) == 0) { if ((flags & RFMEM) == 0) { /* unshare user LDT */ - struct pcb *pcb1 = &p1->p_addr->u_pcb; + struct pcb *pcb1 = td1->td_pcb; struct pcb_ldt *pcb_ldt = pcb1->pcb_ldt; if (pcb_ldt && pcb_ldt->ldt_refcnt > 1) { pcb_ldt = user_ldt_alloc(pcb1,pcb_ldt->ldt_len); @@ -145,30 +150,32 @@ cpu_fork(p1, p2, flags) /* Ensure that p1's pcb is up to date. */ #ifdef DEV_NPX - if (p1 == curproc) - p1->p_addr->u_pcb.pcb_gs = rgs(); + if (td1 == curthread) + td1->td_pcb->pcb_gs = rgs(); savecrit = critical_enter(); - if (PCPU_GET(npxproc) == p1) - npxsave(&p1->p_addr->u_pcb.pcb_save); + if (PCPU_GET(npxthread) == td1) + npxsave(&td1->td_pcb->pcb_save); critical_exit(savecrit); #endif + /* Point the pcb to the top of the stack */ + pcb2 = (struct pcb *)(td2->td_kstack + KSTACK_PAGES * PAGE_SIZE) - 1; + td2->td_pcb = pcb2; + /* Copy p1's pcb. */ - p2->p_addr->u_pcb = p1->p_addr->u_pcb; - pcb2 = &p2->p_addr->u_pcb; + bcopy(td1->td_pcb, pcb2, sizeof(*pcb2)); /* * Create a new fresh stack for the new process. * Copy the trap frame for the return to user mode as if from a * syscall. This copies most of the user mode register values. */ - p2->p_frame = (struct trapframe *) - ((int)p2->p_addr + UPAGES * PAGE_SIZE - 16) - 1; - bcopy(p1->p_frame, p2->p_frame, sizeof(struct trapframe)); + td2->td_frame = (struct trapframe *)td2->td_pcb - 1; + bcopy(td1->td_frame, td2->td_frame, sizeof(struct trapframe)); - p2->p_frame->tf_eax = 0; /* Child returns zero */ - p2->p_frame->tf_eflags &= ~PSL_C; /* success */ - p2->p_frame->tf_edx = 1; + td2->td_frame->tf_eax = 0; /* Child returns zero */ + td2->td_frame->tf_eflags &= ~PSL_C; /* success */ + td2->td_frame->tf_edx = 1; /* * Set registers for trampoline to user mode. Leave space for the @@ -178,8 +185,8 @@ cpu_fork(p1, p2, flags) pcb2->pcb_edi = 0; pcb2->pcb_esi = (int)fork_return; /* fork_trampoline argument */ pcb2->pcb_ebp = 0; - pcb2->pcb_esp = (int)p2->p_frame - sizeof(void *); - pcb2->pcb_ebx = (int)p2; /* fork_trampoline argument */ + pcb2->pcb_esp = (int)td2->td_frame - sizeof(void *); + pcb2->pcb_ebx = (int)td2; /* fork_trampoline argument */ pcb2->pcb_eip = (int)fork_trampoline; /*- * pcb2->pcb_dr*: cloned above. @@ -228,8 +235,8 @@ cpu_fork(p1, p2, flags) * This is needed to make kernel threads stay in kernel mode. */ void -cpu_set_fork_handler(p, func, arg) - struct proc *p; +cpu_set_fork_handler(td, func, arg) + struct thread *td; void (*func) __P((void *)); void *arg; { @@ -237,18 +244,18 @@ cpu_set_fork_handler(p, func, arg) * Note that the trap frame follows the args, so the function * is really called like this: func(arg, frame); */ - p->p_addr->u_pcb.pcb_esi = (int) func; /* function */ - p->p_addr->u_pcb.pcb_ebx = (int) arg; /* first arg */ + td->td_pcb->pcb_esi = (int) func; /* function */ + td->td_pcb->pcb_ebx = (int) arg; /* first arg */ } void -cpu_exit(p) - register struct proc *p; +cpu_exit(td) + register struct thread *td; { - struct pcb *pcb = &p->p_addr->u_pcb; + struct pcb *pcb = td->td_pcb; #ifdef DEV_NPX - npxexit(p); + npxexit(td); #endif if (pcb->pcb_ext != 0) { /* @@ -280,25 +287,29 @@ cpu_wait(p) * Dump the machine specific header information at the start of a core dump. */ int -cpu_coredump(p, vp, cred) - struct proc *p; +cpu_coredump(td, vp, cred) + struct thread *td; struct vnode *vp; struct ucred *cred; { + struct proc *p = td->td_proc; int error; caddr_t tempuser; - tempuser = malloc(ctob(UPAGES), M_TEMP, M_WAITOK | M_ZERO); + tempuser = malloc(ctob(UAREA_PAGES + KSTACK_PAGES), M_TEMP, M_WAITOK | M_ZERO); if (!tempuser) return EINVAL; - bcopy(p->p_addr, tempuser, sizeof(struct user)); - bcopy(p->p_frame, - tempuser + ((caddr_t) p->p_frame - (caddr_t) p->p_addr), + bcopy(p->p_uarea, tempuser, sizeof(struct user)); +#if 0 /* XXXKSE - broken, fixme!!!!! td_frame is in kstack! */ + bcopy(td->td_frame, + tempuser + ((caddr_t) td->td_frame - (caddr_t) p->p_uarea), sizeof(struct trapframe)); +#endif - error = vn_rdwr(UIO_WRITE, vp, (caddr_t) tempuser, ctob(UPAGES), - (off_t)0, UIO_SYSSPACE, IO_UNIT, cred, (int *)NULL, p); + error = vn_rdwr(UIO_WRITE, vp, (caddr_t) tempuser, + ctob(UAREA_PAGES + KSTACK_PAGES), + (off_t)0, UIO_SYSSPACE, IO_UNIT, cred, (int *)NULL, td); free(tempuser, M_TEMP); diff --git a/sys/amd64/include/cpu.h b/sys/amd64/include/cpu.h index 04ef7a0..a7783a0 100644 --- a/sys/amd64/include/cpu.h +++ b/sys/amd64/include/cpu.h @@ -56,8 +56,8 @@ #define cpu_exec(p) /* nothing */ #define cpu_swapin(p) /* nothing */ -#define cpu_getstack(p) ((p)->p_frame->tf_esp) -#define cpu_setstack(p, ap) ((p)->p_frame->tf_esp = (ap)) +#define cpu_getstack(td) ((td)->td_frame->tf_esp) +#define cpu_setstack(td, ap) ((td)->td_frame->tf_esp = (ap)) #define TRAPF_USERMODE(framep) \ ((ISPL((framep)->tf_cs) == SEL_UPL) || ((framep)->tf_eflags & PSL_VM)) diff --git a/sys/amd64/include/fpu.h b/sys/amd64/include/fpu.h index 67ce661..589a4be 100644 --- a/sys/amd64/include/fpu.h +++ b/sys/amd64/include/fpu.h @@ -142,7 +142,7 @@ union savefpu { #ifdef _KERNEL int npxdna __P((void)); -void npxexit __P((struct proc *p)); +void npxexit __P((struct thread *td)); void npxinit __P((int control)); void npxsave __P((union savefpu *addr)); int npxtrap __P((void)); diff --git a/sys/amd64/include/md_var.h b/sys/amd64/include/md_var.h index bc9cb75..c3564c2 100644 --- a/sys/amd64/include/md_var.h +++ b/sys/amd64/include/md_var.h @@ -61,7 +61,7 @@ extern char sigcode[]; extern int szsigcode, szosigcode; typedef void alias_for_inthand_t __P((u_int cs, u_int ef, u_int esp, u_int ss)); -struct proc; +struct thread; struct reg; struct fpreg; struct dbreg; @@ -80,9 +80,9 @@ void doreti_popl_es __P((void)) __asm(__STRING(doreti_popl_es)); void doreti_popl_es_fault __P((void)) __asm(__STRING(doreti_popl_es_fault)); void doreti_popl_fs __P((void)) __asm(__STRING(doreti_popl_fs)); void doreti_popl_fs_fault __P((void)) __asm(__STRING(doreti_popl_fs_fault)); -int fill_fpregs __P((struct proc *, struct fpreg *)); -int fill_regs __P((struct proc *p, struct reg *regs)); -int fill_dbregs __P((struct proc *p, struct dbreg *dbregs)); +int fill_fpregs __P((struct thread *, struct fpreg *)); +int fill_regs __P((struct thread *p, struct reg *regs)); +int fill_dbregs __P((struct thread *p, struct dbreg *dbregs)); void fillw __P((int /*u_short*/ pat, void *base, size_t cnt)); void i486_bzero __P((void *buf, size_t len)); void i586_bcopy __P((const void *from, void *to, size_t len)); diff --git a/sys/amd64/include/mptable.h b/sys/amd64/include/mptable.h index d912e0c..f558524 100644 --- a/sys/amd64/include/mptable.h +++ b/sys/amd64/include/mptable.h @@ -26,7 +26,7 @@ */ #include "opt_cpu.h" -#include "opt_upages.h" +#include "opt_kstack_pages.h" #ifdef SMP #include <machine/smptests.h> @@ -1960,8 +1960,8 @@ start_all_aps(u_int boot_addr) SMPpt[pg] = (pt_entry_t)(PG_V | PG_RW | vtophys(gd)); /* allocate and set up an idle stack data page */ - stack = (char *)kmem_alloc(kernel_map, UPAGES*PAGE_SIZE); - for (i = 0; i < UPAGES; i++) + stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); /* XXXKSE */ + for (i = 0; i < KSTACK_PAGES; i++) SMPpt[pg + 1 + i] = (pt_entry_t) (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); @@ -1977,7 +1977,7 @@ start_all_aps(u_int boot_addr) outb(CMOS_DATA, BIOS_WARM); /* 'warm-start' */ #endif - bootSTK = &SMP_prvspace[x].idlestack[UPAGES*PAGE_SIZE]; + bootSTK = &SMP_prvspace[x].idlekstack[KSTACK_PAGES * PAGE_SIZE]; bootAP = x; /* attempt to start the Application Processor */ @@ -2019,8 +2019,8 @@ start_all_aps(u_int boot_addr) */ /* Allocate and setup BSP idle stack */ - stack = (char *)kmem_alloc(kernel_map, UPAGES * PAGE_SIZE); - for (i = 0; i < UPAGES; i++) + stack = (char *)kmem_alloc(kernel_map, KSTACK_PAGES * PAGE_SIZE); + for (i = 0; i < KSTACK_PAGES; i++) SMPpt[1 + i] = (pt_entry_t) (PG_V | PG_RW | vtophys(PAGE_SIZE * i + stack)); @@ -2241,7 +2241,7 @@ ap_init(void) * Set curproc to our per-cpu idleproc so that mutexes have * something unique to lock with. */ - PCPU_SET(curproc, PCPU_GET(idleproc)); + PCPU_SET(curthread, PCPU_GET(idlethread)); PCPU_SET(spinlocks, NULL); /* lock against other AP's that are waking up */ @@ -2323,7 +2323,7 @@ forwarded_statclock(struct trapframe frame) { mtx_lock_spin(&sched_lock); - statclock_process(curproc, TRAPF_PC(&frame), TRAPF_USERMODE(&frame)); + statclock_process(curthread->td_kse, TRAPF_PC(&frame), TRAPF_USERMODE(&frame)); mtx_unlock_spin(&sched_lock); } @@ -2354,7 +2354,7 @@ forwarded_hardclock(struct trapframe frame) { mtx_lock_spin(&sched_lock); - hardclock_process(curproc, TRAPF_USERMODE(&frame)); + hardclock_process(curthread, TRAPF_USERMODE(&frame)); mtx_unlock_spin(&sched_lock); } diff --git a/sys/amd64/include/mutex.h b/sys/amd64/include/mutex.h index b0fe512..ae37b23 100644 --- a/sys/amd64/include/mutex.h +++ b/sys/amd64/include/mutex.h @@ -250,7 +250,7 @@ extern struct mtx clock_lock; pushl %ecx ; \ pushl %ebx ; \ movl $(MTX_UNOWNED) , %eax ; \ - movl PCPU(CURPROC), %ebx ; \ + movl PCPU(CURTHREAD), %ebx ; \ pushfl ; \ popl %ecx ; \ cli ; \ diff --git a/sys/amd64/include/npx.h b/sys/amd64/include/npx.h index 67ce661..589a4be 100644 --- a/sys/amd64/include/npx.h +++ b/sys/amd64/include/npx.h @@ -142,7 +142,7 @@ union savefpu { #ifdef _KERNEL int npxdna __P((void)); -void npxexit __P((struct proc *p)); +void npxexit __P((struct thread *td)); void npxinit __P((int control)); void npxsave __P((union savefpu *addr)); int npxtrap __P((void)); diff --git a/sys/amd64/include/pcb_ext.h b/sys/amd64/include/pcb_ext.h index 7d6fb08..c6f9fc0 100644 --- a/sys/amd64/include/pcb_ext.h +++ b/sys/amd64/include/pcb_ext.h @@ -53,7 +53,7 @@ struct pcb_ldt { #ifdef _KERNEL -int i386_extend_pcb __P((struct proc *)); +int i386_extend_pcb __P((struct thread *)); void set_user_ldt __P((struct pcb *)); struct pcb_ldt *user_ldt_alloc __P((struct pcb *, int)); void user_ldt_free __P((struct pcb *)); diff --git a/sys/amd64/include/pcpu.h b/sys/amd64/include/pcpu.h index 37388aa..f02354d 100644 --- a/sys/amd64/include/pcpu.h +++ b/sys/amd64/include/pcpu.h @@ -52,19 +52,19 @@ * other processors" */ struct globaldata { - struct globaldata *gd_prvspace; /* self-reference */ - struct proc *gd_curproc; /* current process */ - struct proc *gd_idleproc; /* idle process */ - struct proc *gd_npxproc; - struct pcb *gd_curpcb; /* current pcb */ - struct timeval gd_switchtime; - struct i386tss gd_common_tss; - int gd_switchticks; - struct segment_descriptor gd_common_tssd; - struct segment_descriptor *gd_tss_gdt; - int gd_currentldt; - u_int gd_cpuid; /* this cpu number */ - u_int gd_other_cpus; /* all other cpus */ + struct globaldata *gd_prvspace; /* self-reference */ + struct thread *gd_curthread; + struct thread *gd_npxthread; + struct pcb *gd_curpcb; + struct thread *gd_idlethread; + struct timeval gd_switchtime; + struct i386tss gd_common_tss; + int gd_switchticks; + struct segment_descriptor gd_common_tssd; + struct segment_descriptor *gd_tss_gdt; + int gd_currentldt; + u_int gd_cpuid; + u_int gd_other_cpus; SLIST_ENTRY(globaldata) gd_allcpu; struct lock_list_entry *gd_spinlocks; #ifdef KTR_PERCPU diff --git a/sys/amd64/include/proc.h b/sys/amd64/include/proc.h index 4217c4f..9e9e449 100644 --- a/sys/amd64/include/proc.h +++ b/sys/amd64/include/proc.h @@ -42,6 +42,9 @@ /* * Machine-dependent part of the proc structure for i386. */ +struct mdthread { +}; + struct mdproc { }; diff --git a/sys/amd64/include/reg.h b/sys/amd64/include/reg.h index 47856a3..f4c6ae5 100644 --- a/sys/amd64/include/reg.h +++ b/sys/amd64/include/reg.h @@ -143,10 +143,10 @@ struct dbreg { /* * XXX these interfaces are MI, so they should be declared in a MI place. */ -int set_fpregs __P((struct proc *, struct fpreg *)); -int set_regs __P((struct proc *p, struct reg *regs)); -void setregs __P((struct proc *, u_long, u_long, u_long)); -int set_dbregs __P((struct proc *p, struct dbreg *dbregs)); +int set_fpregs __P((struct thread *, struct fpreg *)); +int set_regs __P((struct thread *p, struct reg *regs)); +void setregs __P((struct thread *, u_long, u_long, u_long)); +int set_dbregs __P((struct thread *p, struct dbreg *dbregs)); #endif #endif /* !_MACHINE_REG_H_ */ diff --git a/sys/amd64/isa/atpic_vector.S b/sys/amd64/isa/atpic_vector.S index 6b48737..3a42af4 100644 --- a/sys/amd64/isa/atpic_vector.S +++ b/sys/amd64/isa/atpic_vector.S @@ -61,8 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - movl PCPU(CURPROC),%ebx ; \ - incl P_INTR_NESTING_LEVEL(%ebx) ; \ + movl PCPU(CURTHREAD),%ebx ; \ + incl TD_INTR_NESTING_LEVEL(%ebx) ; \ pushl intr_unit + (irq_num) * 4 ; \ call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -70,7 +70,7 @@ IDTVEC(vec_name) ; \ incl cnt+V_INTR ; /* book-keeping can wait */ \ movl intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ - decl P_INTR_NESTING_LEVEL(%ebx) ; \ + decl TD_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp doreti @@ -104,14 +104,14 @@ IDTVEC(vec_name) ; \ movb %al,imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - movl PCPU(CURPROC),%ebx ; \ - incl P_INTR_NESTING_LEVEL(%ebx) ; \ + movl PCPU(CURTHREAD),%ebx ; \ + incl TD_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ call sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ - decl P_INTR_NESTING_LEVEL(%ebx) ; \ + decl TD_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* doreti, but it's probably better to use less cache. */ \ diff --git a/sys/amd64/isa/icu_vector.S b/sys/amd64/isa/icu_vector.S index 6b48737..3a42af4 100644 --- a/sys/amd64/isa/icu_vector.S +++ b/sys/amd64/isa/icu_vector.S @@ -61,8 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - movl PCPU(CURPROC),%ebx ; \ - incl P_INTR_NESTING_LEVEL(%ebx) ; \ + movl PCPU(CURTHREAD),%ebx ; \ + incl TD_INTR_NESTING_LEVEL(%ebx) ; \ pushl intr_unit + (irq_num) * 4 ; \ call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -70,7 +70,7 @@ IDTVEC(vec_name) ; \ incl cnt+V_INTR ; /* book-keeping can wait */ \ movl intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ - decl P_INTR_NESTING_LEVEL(%ebx) ; \ + decl TD_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp doreti @@ -104,14 +104,14 @@ IDTVEC(vec_name) ; \ movb %al,imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - movl PCPU(CURPROC),%ebx ; \ - incl P_INTR_NESTING_LEVEL(%ebx) ; \ + movl PCPU(CURTHREAD),%ebx ; \ + incl TD_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ call sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ - decl P_INTR_NESTING_LEVEL(%ebx) ; \ + decl TD_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* doreti, but it's probably better to use less cache. */ \ diff --git a/sys/amd64/isa/icu_vector.s b/sys/amd64/isa/icu_vector.s index 6b48737..3a42af4 100644 --- a/sys/amd64/isa/icu_vector.s +++ b/sys/amd64/isa/icu_vector.s @@ -61,8 +61,8 @@ IDTVEC(vec_name) ; \ mov $KPSEL,%ax ; \ mov %ax,%fs ; \ FAKE_MCOUNT((12+ACTUALLY_PUSHED)*4(%esp)) ; \ - movl PCPU(CURPROC),%ebx ; \ - incl P_INTR_NESTING_LEVEL(%ebx) ; \ + movl PCPU(CURTHREAD),%ebx ; \ + incl TD_INTR_NESTING_LEVEL(%ebx) ; \ pushl intr_unit + (irq_num) * 4 ; \ call *intr_handler + (irq_num) * 4 ; /* do the work ASAP */ \ enable_icus ; /* (re)enable ASAP (helps edge trigger?) */ \ @@ -70,7 +70,7 @@ IDTVEC(vec_name) ; \ incl cnt+V_INTR ; /* book-keeping can wait */ \ movl intr_countp + (irq_num) * 4,%eax ; \ incl (%eax) ; \ - decl P_INTR_NESTING_LEVEL(%ebx) ; \ + decl TD_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ jmp doreti @@ -104,14 +104,14 @@ IDTVEC(vec_name) ; \ movb %al,imen + IRQ_BYTE(irq_num) ; \ outb %al,$icu+ICU_IMR_OFFSET ; \ enable_icus ; \ - movl PCPU(CURPROC),%ebx ; \ - incl P_INTR_NESTING_LEVEL(%ebx) ; \ + movl PCPU(CURTHREAD),%ebx ; \ + incl TD_INTR_NESTING_LEVEL(%ebx) ; \ __CONCAT(Xresume,irq_num): ; \ FAKE_MCOUNT(13*4(%esp)) ; /* XXX late to avoid double count */ \ pushl $irq_num; /* pass the IRQ */ \ call sched_ithd ; \ addl $4, %esp ; /* discard the parameter */ \ - decl P_INTR_NESTING_LEVEL(%ebx) ; \ + decl TD_INTR_NESTING_LEVEL(%ebx) ; \ MEXITCOUNT ; \ /* We could usually avoid the following jmp by inlining some of */ \ /* doreti, but it's probably better to use less cache. */ \ diff --git a/sys/amd64/isa/npx.c b/sys/amd64/isa/npx.c index 4fe3d04..84534ea 100644 --- a/sys/amd64/isa/npx.c +++ b/sys/amd64/isa/npx.c @@ -128,23 +128,23 @@ void stop_emulating __P((void)); #endif /* __GNUC__ */ #ifdef CPU_ENABLE_SSE -#define GET_FPU_CW(proc) \ +#define GET_FPU_CW(thread) \ (cpu_fxsr ? \ - (proc)->p_addr->u_pcb.pcb_save.sv_xmm.sv_env.en_cw : \ - (proc)->p_addr->u_pcb.pcb_save.sv_87.sv_env.en_cw) -#define GET_FPU_SW(proc) \ + (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_cw : \ + (thread)->td_pcb->pcb_save.sv_87.sv_env.en_cw) +#define GET_FPU_SW(thread) \ (cpu_fxsr ? \ - (proc)->p_addr->u_pcb.pcb_save.sv_xmm.sv_env.en_sw : \ - (proc)->p_addr->u_pcb.pcb_save.sv_87.sv_env.en_sw) + (thread)->td_pcb->pcb_save.sv_xmm.sv_env.en_sw : \ + (thread)->td_pcb->pcb_save.sv_87.sv_env.en_sw) #define GET_FPU_EXSW_PTR(pcb) \ (cpu_fxsr ? \ &(pcb)->pcb_save.sv_xmm.sv_ex_sw : \ &(pcb)->pcb_save.sv_87.sv_ex_sw) #else /* CPU_ENABLE_SSE */ -#define GET_FPU_CW(proc) \ - (proc->p_addr->u_pcb.pcb_save.sv_87.sv_env.en_cw) -#define GET_FPU_SW(proc) \ - (proc->p_addr->u_pcb.pcb_save.sv_87.sv_env.en_sw) +#define GET_FPU_CW(thread) \ + (thread->td_pcb->pcb_save.sv_87.sv_env.en_cw) +#define GET_FPU_SW(thread) \ + (thread->td_pcb->pcb_save.sv_87.sv_env.en_sw) #define GET_FPU_EXSW_PTR(pcb) \ (&(pcb)->pcb_save.sv_87.sv_ex_sw) #endif /* CPU_ENABLE_SSE */ @@ -241,7 +241,7 @@ static void npx_intr(dummy) void *dummy; { - struct proc *p; + struct thread *td; /* * The BUSY# latch must be cleared in all cases so that the next @@ -250,22 +250,22 @@ npx_intr(dummy) outb(0xf0, 0); /* - * npxproc is normally non-null here. In that case, schedule an + * npxthread is normally non-null here. In that case, schedule an * AST to finish the exception handling in the correct context - * (this interrupt may occur after the process has entered the + * (this interrupt may occur after the thread has entered the * kernel via a syscall or an interrupt). Otherwise, the npx - * state of the process that caused this interrupt must have been - * pushed to the process' pcb, and clearing of the busy latch + * state of the thread that caused this interrupt must have been + * pushed to the thread' pcb, and clearing of the busy latch * above has finished the (essentially null) handling of this * interrupt. Control will eventually return to the instruction * that caused it and it will repeat. We will eventually (usually * soon) win the race to handle the interrupt properly. */ - p = PCPU_GET(npxproc); - if (p != NULL) { - p->p_addr->u_pcb.pcb_flags |= PCB_NPXTRAP; + td = PCPU_GET(npxthread); + if (td != NULL) { + td->td_pcb->pcb_flags |= PCB_NPXTRAP; mtx_lock_spin(&sched_lock); - p->p_sflag |= PS_ASTPENDING; + td->td_kse->ke_flags |= KEF_ASTPENDING; mtx_unlock_spin(&sched_lock); } } @@ -570,7 +570,7 @@ npxinit(control) /* * fninit has the same h/w bugs as fnsave. Use the detoxified * fnsave to throw away any junk in the fpu. npxsave() initializes - * the fpu and sets npxproc = NULL as important side effects. + * the fpu and sets npxthread = NULL as important side effects. */ savecrit = critical_enter(); npxsave(&dummy); @@ -586,13 +586,13 @@ npxinit(control) * Free coprocessor (if we have it). */ void -npxexit(p) - struct proc *p; +npxexit(td) + struct thread *td; { critical_t savecrit; savecrit = critical_enter(); - if (p == PCPU_GET(npxproc)) + if (td == PCPU_GET(npxthread)) npxsave(&PCPU_GET(curpcb)->pcb_save); critical_exit(savecrit); #ifdef NPX_DEBUG @@ -607,8 +607,9 @@ npxexit(p) */ if (masked_exceptions & 0x0d) log(LOG_ERR, - "pid %d (%s) exited with masked floating point exceptions 0x%02x\n", - p->p_pid, p->p_comm, masked_exceptions); + "pid %d (%s) exited with masked floating" + " point exceptions 0x%02x\n", + td->td_proc->p_pid, td->td_proc->p_comm, masked_exceptions); } #endif } @@ -809,8 +810,8 @@ npxtrap() u_long *exstat; if (!npx_exists) { - printf("npxtrap: npxproc = %p, curproc = %p, npx_exists = %d\n", - PCPU_GET(npxproc), curproc, npx_exists); + printf("npxtrap: npxthread = %p, curthread = %p, npx_exists = %d\n", + PCPU_GET(npxthread), curthread, npx_exists); panic("npxtrap from nowhere"); } savecrit = critical_enter(); @@ -820,18 +821,18 @@ npxtrap() * state to memory. Fetch the relevant parts of the state from * wherever they are. */ - if (PCPU_GET(npxproc) != curproc) { - control = GET_FPU_CW(curproc); - status = GET_FPU_SW(curproc); + if (PCPU_GET(npxthread) != curthread) { + control = GET_FPU_CW(curthread); + status = GET_FPU_SW(curthread); } else { fnstcw(&control); fnstsw(&status); } - exstat = GET_FPU_EXSW_PTR(&curproc->p_addr->u_pcb); + exstat = GET_FPU_EXSW_PTR(curthread->td_pcb); *exstat = status; - if (PCPU_GET(npxproc) != curproc) - GET_FPU_SW(curproc) &= ~0x80bf; + if (PCPU_GET(npxthread) != curthread) + GET_FPU_SW(curthread) &= ~0x80bf; else fnclex(); critical_exit(savecrit); @@ -841,7 +842,7 @@ npxtrap() /* * Implement device not available (DNA) exception * - * It would be better to switch FP context here (if curproc != npxproc) + * It would be better to switch FP context here (if curthread != npxthread) * and not necessarily for every context switch, but it is too hard to * access foreign pcb's. */ @@ -853,9 +854,9 @@ npxdna() if (!npx_exists) return (0); - if (PCPU_GET(npxproc) != NULL) { - printf("npxdna: npxproc = %p, curproc = %p\n", - PCPU_GET(npxproc), curproc); + if (PCPU_GET(npxthread) != NULL) { + printf("npxdna: npxthread = %p, curthread = %p\n", + PCPU_GET(npxthread), curthread); panic("npxdna"); } s = critical_enter(); @@ -863,7 +864,7 @@ npxdna() /* * Record new context early in case frstor causes an IRQ13. */ - PCPU_SET(npxproc, CURPROC); + PCPU_SET(npxthread, curthread); exstat = GET_FPU_EXSW_PTR(PCPU_GET(curpcb)); *exstat = 0; @@ -895,13 +896,13 @@ npxdna() * after the process has entered the kernel. It may even be delivered after * the fnsave here completes. A spurious IRQ13 for the fnsave is handled in * the same way as a very-late-arriving non-spurious IRQ13 from user mode: - * it is normally ignored at first because we set npxproc to NULL; it is + * it is normally ignored at first because we set npxthread to NULL; it is * normally retriggered in npxdna() after return to user mode. * * npxsave() must be called with interrupts disabled, so that it clears - * npxproc atomically with saving the state. We require callers to do the + * npxthread atomically with saving the state. We require callers to do the * disabling, since most callers need to disable interrupts anyway to call - * npxsave() atomically with checking npxproc. + * npxsave() atomically with checking npxthread. * * A previous version of npxsave() went to great lengths to excecute fnsave * with interrupts enabled in case executing it froze the CPU. This case @@ -917,7 +918,7 @@ npxsave(addr) fpusave(addr); start_emulating(); - PCPU_SET(npxproc, NULL); + PCPU_SET(npxthread, NULL); } static void |