diff options
Diffstat (limited to 'sys/amd64/amd64')
| -rw-r--r-- | sys/amd64/amd64/apic_vector.S | 22 | ||||
| -rw-r--r-- | sys/amd64/amd64/cpu_switch.S | 190 | ||||
| -rw-r--r-- | sys/amd64/amd64/machdep.c | 36 | ||||
| -rw-r--r-- | sys/amd64/amd64/mp_machdep.c | 9 | ||||
| -rw-r--r-- | sys/amd64/amd64/mptable.c | 9 | ||||
| -rw-r--r-- | sys/amd64/amd64/pmap.c | 129 | ||||
| -rw-r--r-- | sys/amd64/amd64/swtch.s | 190 |
7 files changed, 423 insertions, 162 deletions
diff --git a/sys/amd64/amd64/apic_vector.S b/sys/amd64/amd64/apic_vector.S index e4b6ea5..d91ff1c 100644 --- a/sys/amd64/amd64/apic_vector.S +++ b/sys/amd64/amd64/apic_vector.S @@ -3,6 +3,7 @@ * $FreeBSD$ */ +#include "opt_swtch.h" #include <machine/apic.h> #include <machine/smp.h> @@ -648,7 +649,28 @@ Xrendezvous: POP_FRAME iret +#ifdef LAZY_SWITCH +/* + * Clean up when we lose out on the lazy context switch optimization. + * ie: when we are about to release a PTD but a cpu is still borrowing it. + */ + SUPERALIGN_TEXT + .globl Xlazypmap +Xlazypmap: + PUSH_FRAME + movl $KDSEL, %eax + mov %ax, %ds /* use KERNEL data segment */ + mov %ax, %es + movl $KPSEL, %eax + mov %ax, %fs + + call pmap_lazyfix_action + movl $0, lapic+LA_EOI /* End Of Interrupt to APIC */ + POP_FRAME + iret +#endif + .data .globl apic_pin_trigger diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S index aaf0ec2..793e634 100644 --- a/sys/amd64/amd64/cpu_switch.S +++ b/sys/amd64/amd64/cpu_switch.S @@ -37,30 +37,16 @@ */ #include "opt_npx.h" +#include "opt_swtch.h" #include <machine/asmacros.h> -#ifdef SMP -#include <machine/apic.h> -#include <machine/smptests.h> /* CHEAP_TPR, GRAB_LOPRIO */ -#endif - #include "assym.s" /*****************************************************************************/ /* Scheduling */ /*****************************************************************************/ - .data - - .globl panic - -#ifdef SWTCH_OPTIM_STATS - .globl swtch_optim_stats, tlb_flush_count -swtch_optim_stats: .long 0 /* number of _swtch_optims */ -tlb_flush_count: .long 0 -#endif - .text /* @@ -68,30 +54,60 @@ tlb_flush_count: .long 0 * * This is the second half of cpu_swtch(). It is used when the current * thread is either a dummy or slated to die, and we no longer care - * about its state. + * about its state. This is only a slight optimization and is probably + * not worth it anymore. Note that we need to clear the pm_active bits so + * we do need the old proc if it still exists. + * 0(%esp) = ret + * 4(%esp) = oldtd + * 8(%esp) = newtd */ ENTRY(cpu_throw) + movl PCPU(CPUID), %esi + movl 4(%esp),%ecx /* Old thread */ + testl %ecx,%ecx /* no thread? */ + jz 1f + /* release bit from old pm_active */ + movl TD_PROC(%ecx), %eax /* thread->td_proc */ + movl P_VMSPACE(%eax), %ebx /* proc->p_vmspace */ +#ifdef SMP + lock +#endif + btrl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* clear old */ +1: + movl 8(%esp),%ecx /* New thread */ + movl TD_PCB(%ecx),%edx +#ifdef SWTCH_OPTIM_STATS + incl tlb_flush_count +#endif + movl PCB_CR3(%edx),%eax + movl %eax,%cr3 /* new address space */ + /* set bit in new pm_active */ + movl TD_PROC(%ecx),%eax + movl P_VMSPACE(%eax), %ebx +#ifdef SMP + lock +#endif + btsl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* set new */ jmp sw1 /* - * cpu_switch() + * cpu_switch(old, new) * * Save the current thread state, then select the next thread to run * and load its state. + * 0(%esp) = ret + * 4(%esp) = oldtd + * 8(%esp) = newtd */ ENTRY(cpu_switch) - /* Switch to new thread. First, save context as needed. */ - movl PCPU(CURTHREAD),%ecx - - /* If no thread to save, don't save it (XXX shouldn't happen). */ - testl %ecx,%ecx - jz sw1 + /* Switch to new thread. First, save context. */ + movl 4(%esp),%ecx - movl TD_PROC(%ecx), %eax - movl P_VMSPACE(%eax), %edx - movl PCPU(CPUID), %eax - btrl %eax, VM_PMAP+PM_ACTIVE(%edx) +#ifdef INVARIANTS + testl %ecx,%ecx /* no thread? */ + jz badsw2 /* no, panic */ +#endif movl TD_PCB(%ecx),%edx @@ -125,10 +141,6 @@ ENTRY(cpu_switch) movl %eax,PCB_DR0(%edx) 1: -#ifdef SMP - /* XXX FIXME: we should be saving the local APIC TPR */ -#endif - #ifdef DEV_NPX /* have we used fp, and need a save? */ cmpl %ecx,PCPU(FPCURTHREAD) @@ -140,56 +152,76 @@ ENTRY(cpu_switch) 1: #endif - /* Save is done. Now choose a new thread. */ - /* XXX still trashing space above the old "Top Of Stack". */ -sw1: + /* Save is done. Now fire up new thread. Leave old vmspace. */ + movl %ecx,%edi + movl 8(%esp),%ecx /* New thread */ +#ifdef INVARIANTS + testl %ecx,%ecx /* no thread? */ + jz badsw3 /* no, panic */ +#endif + movl TD_PCB(%ecx),%edx + movl PCPU(CPUID), %esi -#ifdef SMP - /* - * Stop scheduling if smp_active has become zero (for rebooting) and - * we are not the BSP. - */ - cmpl $0,smp_active - jne 1f - cmpl $0,PCPU(CPUID) + /* switch address space */ + movl PCB_CR3(%edx),%eax +#ifdef LAZY_SWITCH + cmpl $0,lazy_flush_enable je 1f - movl PCPU(IDLETHREAD), %eax - jmp sw1b + cmpl %eax,IdlePTD /* Kernel address space? */ +#ifdef SWTCH_OPTIM_STATS + je 3f +#else + je sw1 +#endif 1: + movl %cr3,%ebx /* The same address space? */ + cmpl %ebx,%eax +#ifdef SWTCH_OPTIM_STATS + je 2f /* Yes, skip all that cruft */ +#else + je sw1 +#endif #endif - /* - * Choose a new thread to schedule. choosethread() returns idlethread - * if it cannot find another thread to run. - */ - call choosethread /* Trash ecx, edx; ret eax. */ +#ifdef SWTCH_OPTIM_STATS + incl tlb_flush_count +#endif + movl %eax,%cr3 /* new address space */ -#ifdef INVARIANTS - testl %eax,%eax /* no thread? */ - jz badsw3 /* no, panic */ + /* Release bit from old pmap->pm_active */ + movl TD_PROC(%edi), %eax /* oldproc */ + movl P_VMSPACE(%eax), %ebx +#ifdef SMP + lock #endif + btrl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* clear old */ -sw1b: - movl %eax,%ecx - movl TD_PCB(%ecx),%edx + /* Set bit in new pmap->pm_active */ + movl TD_PROC(%ecx),%eax /* newproc */ + movl P_VMSPACE(%eax), %ebx +#ifdef SMP + lock +#endif + btsl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* set new */ +#ifdef LAZY_SWITCH #ifdef SWTCH_OPTIM_STATS + jmp sw1 + +2: /* same address space */ incl swtch_optim_stats -#endif + jmp sw1 - /* switch address space */ - movl %cr3,%ebx /* The same address space? */ - cmpl PCB_CR3(%edx),%ebx - je 4f /* Yes, skip all that cruft */ -#ifdef SWTCH_OPTIM_STATS - decl swtch_optim_stats - incl tlb_flush_count +3: /* kernel address space */ + incl lazy_flush_count +#endif #endif - movl PCB_CR3(%edx),%ebx /* Tell the CPU about the */ - movl %ebx,%cr3 /* new address space */ -4: - movl PCPU(CPUID), %esi +sw1: + /* + * At this point, we've switched address spaces and are ready + * to load up the rest of the next context. + */ cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f /* If not, use the default */ btsl %esi, private_tss /* mark use of private tss */ @@ -221,11 +253,6 @@ sw1b: movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ ltr %si 3: - /* Note in vmspace that this cpu is using it. */ - movl TD_PROC(%ecx),%eax - movl P_VMSPACE(%eax), %ebx - movl PCPU(CPUID), %eax - btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) /* Restore context. */ movl PCB_EBX(%edx),%ebx @@ -241,10 +268,6 @@ sw1b: movl %edx, PCPU(CURPCB) movl %ecx, PCPU(CURTHREAD) /* into next thread */ -#ifdef SMP - /* XXX FIXME: we should be restoring the local APIC TPR */ -#endif - /* * Determine the LDT to use and load it if is the default one and * that is not the current one. @@ -301,12 +324,23 @@ cpu_switch_load_gs: ret #ifdef INVARIANTS +badsw1: + pushal + pushl $sw0_1 + call panic +sw0_1: .asciz "cpu_throw: no newthread supplied" + +badsw2: + pushal + pushl $sw0_2 + call panic +sw0_2: .asciz "cpu_switch: no curthread supplied" + badsw3: pushal pushl $sw0_3 call panic - -sw0_3: .asciz "cpu_switch: choosethread returned NULL" +sw0_3: .asciz "cpu_switch: no newthread supplied" #endif /* diff --git a/sys/amd64/amd64/machdep.c b/sys/amd64/amd64/machdep.c index b389828..baab5cb 100644 --- a/sys/amd64/amd64/machdep.c +++ b/sys/amd64/amd64/machdep.c @@ -49,6 +49,7 @@ #include "opt_msgbuf.h" #include "opt_npx.h" #include "opt_perfmon.h" +#include "opt_swtch.h" #include "opt_kstack_pages.h" #include <sys/param.h> @@ -151,11 +152,40 @@ int _udatasel, _ucodesel; u_int atdevbase; #if defined(SWTCH_OPTIM_STATS) -extern int swtch_optim_stats; +int stupid_switch; +SYSCTL_INT(_debug, OID_AUTO, stupid_switch, + CTLFLAG_RW, &stupid_switch, 0, ""); +int swtch_optim_stats; SYSCTL_INT(_debug, OID_AUTO, swtch_optim_stats, - CTLFLAG_RD, &swtch_optim_stats, 0, ""); + CTLFLAG_RW, &swtch_optim_stats, 0, ""); +int tlb_flush_count; SYSCTL_INT(_debug, OID_AUTO, tlb_flush_count, - CTLFLAG_RD, &tlb_flush_count, 0, ""); + CTLFLAG_RW, &tlb_flush_count, 0, ""); +int lazy_flush_count; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_count, + CTLFLAG_RW, &lazy_flush_count, 0, ""); +int lazy_flush_fixup; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_fixup, + CTLFLAG_RW, &lazy_flush_fixup, 0, ""); +#ifdef SMP +int lazy_flush_smpfixup; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpfixup, + CTLFLAG_RW, &lazy_flush_smpfixup, 0, ""); +int lazy_flush_smpipi; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpipi, + CTLFLAG_RW, &lazy_flush_smpipi, 0, ""); +int lazy_flush_smpbadcr3; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpbadcr3, + CTLFLAG_RW, &lazy_flush_smpbadcr3, 0, ""); +int lazy_flush_smpmiss; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_smpmiss, + CTLFLAG_RW, &lazy_flush_smpmiss, 0, ""); +#endif +#endif +#ifdef LAZY_SWITCH +int lazy_flush_enable = 1; +SYSCTL_INT(_debug, OID_AUTO, lazy_flush_enable, + CTLFLAG_RW, &lazy_flush_enable, 0, ""); #endif int cold = 1; diff --git a/sys/amd64/amd64/mp_machdep.c b/sys/amd64/amd64/mp_machdep.c index 954a7fd..4cc8bf5 100644 --- a/sys/amd64/amd64/mp_machdep.c +++ b/sys/amd64/amd64/mp_machdep.c @@ -27,6 +27,7 @@ #include "opt_cpu.h" #include "opt_kstack_pages.h" +#include "opt_swtch.h" #ifdef SMP #include <machine/smptests.h> @@ -634,6 +635,12 @@ mp_enable(u_int boot_addr) setidt(XSTATCLOCK_OFFSET, Xstatclock, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#ifdef LAZY_SWITCH + /* install an inter-CPU IPI for lazy pmap release */ + setidt(XLAZYPMAP_OFFSET, Xlazypmap, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif + /* install an inter-CPU IPI for all-CPU rendezvous */ setidt(XRENDEZVOUS_OFFSET, Xrendezvous, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); @@ -2598,7 +2605,7 @@ ap_init(void) binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); - cpu_throw(); /* doesn't return */ + cpu_throw(NULL, choosethread()); /* doesn't return */ panic("scheduler returned us to %s", __func__); } diff --git a/sys/amd64/amd64/mptable.c b/sys/amd64/amd64/mptable.c index 954a7fd..4cc8bf5 100644 --- a/sys/amd64/amd64/mptable.c +++ b/sys/amd64/amd64/mptable.c @@ -27,6 +27,7 @@ #include "opt_cpu.h" #include "opt_kstack_pages.h" +#include "opt_swtch.h" #ifdef SMP #include <machine/smptests.h> @@ -634,6 +635,12 @@ mp_enable(u_int boot_addr) setidt(XSTATCLOCK_OFFSET, Xstatclock, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#ifdef LAZY_SWITCH + /* install an inter-CPU IPI for lazy pmap release */ + setidt(XLAZYPMAP_OFFSET, Xlazypmap, + SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); +#endif + /* install an inter-CPU IPI for all-CPU rendezvous */ setidt(XRENDEZVOUS_OFFSET, Xrendezvous, SDT_SYS386IGT, SEL_KPL, GSEL(GCODE_SEL, SEL_KPL)); @@ -2598,7 +2605,7 @@ ap_init(void) binuptime(PCPU_PTR(switchtime)); PCPU_SET(switchticks, ticks); - cpu_throw(); /* doesn't return */ + cpu_throw(NULL, choosethread()); /* doesn't return */ panic("scheduler returned us to %s", __func__); } diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 76b45b4..77c53e0 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -102,6 +102,7 @@ #include "opt_pmap.h" #include "opt_msgbuf.h" #include "opt_kstack_pages.h" +#include "opt_swtch.h" #include <sys/param.h> #include <sys/systm.h> @@ -184,6 +185,9 @@ struct pmap kernel_pmap_store; LIST_HEAD(pmaplist, pmap); static struct pmaplist allpmaps; static struct mtx allpmaps_lock; +#if defined(SMP) && defined(LAZY_SWITCH) +static struct mtx lazypmap_lock; +#endif vm_paddr_t avail_start; /* PA of first available physical page */ vm_paddr_t avail_end; /* PA of last available physical page */ @@ -336,6 +340,9 @@ pmap_bootstrap(firstaddr, loadaddr) kernel_pmap->pm_active = -1; /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvlist); LIST_INIT(&allpmaps); +#if defined(SMP) && defined(LAZY_SWITCH) + mtx_init(&lazypmap_lock, "lazypmap", NULL, MTX_SPIN); +#endif mtx_init(&allpmaps_lock, "allpmaps", NULL, MTX_SPIN); mtx_lock_spin(&allpmaps_lock); LIST_INSERT_HEAD(&allpmaps, kernel_pmap, pm_list); @@ -1486,6 +1493,121 @@ pmap_allocpte(pmap_t pmap, vm_offset_t va) * Pmap allocation/deallocation routines. ***************************************************/ +#ifdef LAZY_SWITCH +#ifdef SMP +/* + * Deal with a SMP shootdown of other users of the pmap that we are + * trying to dispose of. This can be a bit hairy. + */ +static u_int *lazymask; +static u_int lazyptd; +static volatile u_int lazywait; + +void pmap_lazyfix_action(void); + +void +pmap_lazyfix_action(void) +{ + u_int mymask = PCPU_GET(cpumask); + + if (rcr3() == lazyptd) { + load_cr3(PCPU_GET(curpcb)->pcb_cr3); +#ifdef SWTCH_OPTIM_STATS + atomic_add_int(&lazy_flush_smpfixup, 1); + } else { + if (*lazymask & mymask) + lazy_flush_smpbadcr3++; + else + lazy_flush_smpmiss++; +#endif + } + atomic_clear_int(lazymask, mymask); + atomic_store_rel_int(&lazywait, 1); +} + +static void +pmap_lazyfix_self(u_int mymask) +{ + + if (rcr3() == lazyptd) { + load_cr3(PCPU_GET(curpcb)->pcb_cr3); +#ifdef SWTCH_OPTIM_STATS + lazy_flush_fixup++; + } else { + if (*lazymask & mymask) + lazy_flush_smpbadcr3++; + else + lazy_flush_smpmiss++; +#endif + } + atomic_clear_int(lazymask, mymask); +} + + +static void +pmap_lazyfix(pmap_t pmap) +{ + u_int mymask = PCPU_GET(cpumask); + u_int mask; + register u_int spins; + + while ((mask = pmap->pm_active) != 0) { + spins = 50000000; + mask = mask & -mask; /* Find least significant set bit */ + mtx_lock_spin(&lazypmap_lock); +#ifdef PAE + lazyptd = vtophys(pmap->pm_pdpt); +#else + lazyptd = vtophys(pmap->pm_pdir); +#endif + if (mask == mymask) { + lazymask = &pmap->pm_active; + pmap_lazyfix_self(mymask); + } else { + atomic_store_rel_int((u_int *)&lazymask, + (u_int)&pmap->pm_active); + atomic_store_rel_int(&lazywait, 0); + ipi_selected(mask, IPI_LAZYPMAP); + while (lazywait == 0) { + ia32_pause(); + if (--spins == 0) + break; + } +#ifdef SWTCH_OPTIM_STATS + lazy_flush_smpipi++; +#endif + } + mtx_unlock_spin(&lazypmap_lock); + if (spins == 0) + printf("pmap_lazyfix: spun for 50000000\n"); + } +} + +#else /* SMP */ + +/* + * Cleaning up on uniprocessor is easy. For various reasons, we're + * unlikely to have to even execute this code, including the fact + * that the cleanup is deferred until the parent does a wait(2), which + * means that another userland process has run. + */ +static void +pmap_lazyfix(pmap_t pmap) +{ + u_int cr3; + + cr3 = vtophys(pmap->pm_pdir); + if (cr3 == rcr3()) { + load_cr3(PCPU_GET(curpcb)->pcb_cr3); + pmap->pm_active &= ~(PCPU_GET(cpumask)); +#ifdef SWTCH_OPTIM_STATS + lazy_flush_fixup++; +#endif + } +} +#endif /* SMP */ +#endif /* LAZY_SWITCH */ + /* * Release any resources held by the given physical map. * Called when a pmap initialized by pmap_pinit is being released. @@ -1507,6 +1629,9 @@ pmap_release(pmap_t pmap) ("pmap_release: pmap resident count %ld != 0", pmap->pm_stats.resident_count)); +#ifdef LAZY_SWITCH + pmap_lazyfix(pmap); +#endif mtx_lock_spin(&allpmaps_lock); LIST_REMOVE(pmap, pm_list); mtx_unlock_spin(&allpmaps_lock); @@ -3321,9 +3446,10 @@ pmap_activate(struct thread *td) pmap_t pmap; u_int32_t cr3; + critical_enter(); pmap = vmspace_pmap(td->td_proc->p_vmspace); #if defined(SMP) - pmap->pm_active |= PCPU_GET(cpumask); + atomic_set_int(&pmap->pm_active, PCPU_GET(cpumask)); #else pmap->pm_active |= 1; #endif @@ -3348,6 +3474,7 @@ pmap_activate(struct thread *td) #ifdef SWTCH_OPTIM_STATS tlb_flush_count++; #endif + critical_exit(); } vm_offset_t diff --git a/sys/amd64/amd64/swtch.s b/sys/amd64/amd64/swtch.s index aaf0ec2..793e634 100644 --- a/sys/amd64/amd64/swtch.s +++ b/sys/amd64/amd64/swtch.s @@ -37,30 +37,16 @@ */ #include "opt_npx.h" +#include "opt_swtch.h" #include <machine/asmacros.h> -#ifdef SMP -#include <machine/apic.h> -#include <machine/smptests.h> /* CHEAP_TPR, GRAB_LOPRIO */ -#endif - #include "assym.s" /*****************************************************************************/ /* Scheduling */ /*****************************************************************************/ - .data - - .globl panic - -#ifdef SWTCH_OPTIM_STATS - .globl swtch_optim_stats, tlb_flush_count -swtch_optim_stats: .long 0 /* number of _swtch_optims */ -tlb_flush_count: .long 0 -#endif - .text /* @@ -68,30 +54,60 @@ tlb_flush_count: .long 0 * * This is the second half of cpu_swtch(). It is used when the current * thread is either a dummy or slated to die, and we no longer care - * about its state. + * about its state. This is only a slight optimization and is probably + * not worth it anymore. Note that we need to clear the pm_active bits so + * we do need the old proc if it still exists. + * 0(%esp) = ret + * 4(%esp) = oldtd + * 8(%esp) = newtd */ ENTRY(cpu_throw) + movl PCPU(CPUID), %esi + movl 4(%esp),%ecx /* Old thread */ + testl %ecx,%ecx /* no thread? */ + jz 1f + /* release bit from old pm_active */ + movl TD_PROC(%ecx), %eax /* thread->td_proc */ + movl P_VMSPACE(%eax), %ebx /* proc->p_vmspace */ +#ifdef SMP + lock +#endif + btrl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* clear old */ +1: + movl 8(%esp),%ecx /* New thread */ + movl TD_PCB(%ecx),%edx +#ifdef SWTCH_OPTIM_STATS + incl tlb_flush_count +#endif + movl PCB_CR3(%edx),%eax + movl %eax,%cr3 /* new address space */ + /* set bit in new pm_active */ + movl TD_PROC(%ecx),%eax + movl P_VMSPACE(%eax), %ebx +#ifdef SMP + lock +#endif + btsl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* set new */ jmp sw1 /* - * cpu_switch() + * cpu_switch(old, new) * * Save the current thread state, then select the next thread to run * and load its state. + * 0(%esp) = ret + * 4(%esp) = oldtd + * 8(%esp) = newtd */ ENTRY(cpu_switch) - /* Switch to new thread. First, save context as needed. */ - movl PCPU(CURTHREAD),%ecx - - /* If no thread to save, don't save it (XXX shouldn't happen). */ - testl %ecx,%ecx - jz sw1 + /* Switch to new thread. First, save context. */ + movl 4(%esp),%ecx - movl TD_PROC(%ecx), %eax - movl P_VMSPACE(%eax), %edx - movl PCPU(CPUID), %eax - btrl %eax, VM_PMAP+PM_ACTIVE(%edx) +#ifdef INVARIANTS + testl %ecx,%ecx /* no thread? */ + jz badsw2 /* no, panic */ +#endif movl TD_PCB(%ecx),%edx @@ -125,10 +141,6 @@ ENTRY(cpu_switch) movl %eax,PCB_DR0(%edx) 1: -#ifdef SMP - /* XXX FIXME: we should be saving the local APIC TPR */ -#endif - #ifdef DEV_NPX /* have we used fp, and need a save? */ cmpl %ecx,PCPU(FPCURTHREAD) @@ -140,56 +152,76 @@ ENTRY(cpu_switch) 1: #endif - /* Save is done. Now choose a new thread. */ - /* XXX still trashing space above the old "Top Of Stack". */ -sw1: + /* Save is done. Now fire up new thread. Leave old vmspace. */ + movl %ecx,%edi + movl 8(%esp),%ecx /* New thread */ +#ifdef INVARIANTS + testl %ecx,%ecx /* no thread? */ + jz badsw3 /* no, panic */ +#endif + movl TD_PCB(%ecx),%edx + movl PCPU(CPUID), %esi -#ifdef SMP - /* - * Stop scheduling if smp_active has become zero (for rebooting) and - * we are not the BSP. - */ - cmpl $0,smp_active - jne 1f - cmpl $0,PCPU(CPUID) + /* switch address space */ + movl PCB_CR3(%edx),%eax +#ifdef LAZY_SWITCH + cmpl $0,lazy_flush_enable je 1f - movl PCPU(IDLETHREAD), %eax - jmp sw1b + cmpl %eax,IdlePTD /* Kernel address space? */ +#ifdef SWTCH_OPTIM_STATS + je 3f +#else + je sw1 +#endif 1: + movl %cr3,%ebx /* The same address space? */ + cmpl %ebx,%eax +#ifdef SWTCH_OPTIM_STATS + je 2f /* Yes, skip all that cruft */ +#else + je sw1 +#endif #endif - /* - * Choose a new thread to schedule. choosethread() returns idlethread - * if it cannot find another thread to run. - */ - call choosethread /* Trash ecx, edx; ret eax. */ +#ifdef SWTCH_OPTIM_STATS + incl tlb_flush_count +#endif + movl %eax,%cr3 /* new address space */ -#ifdef INVARIANTS - testl %eax,%eax /* no thread? */ - jz badsw3 /* no, panic */ + /* Release bit from old pmap->pm_active */ + movl TD_PROC(%edi), %eax /* oldproc */ + movl P_VMSPACE(%eax), %ebx +#ifdef SMP + lock #endif + btrl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* clear old */ -sw1b: - movl %eax,%ecx - movl TD_PCB(%ecx),%edx + /* Set bit in new pmap->pm_active */ + movl TD_PROC(%ecx),%eax /* newproc */ + movl P_VMSPACE(%eax), %ebx +#ifdef SMP + lock +#endif + btsl %esi, VM_PMAP+PM_ACTIVE(%ebx) /* set new */ +#ifdef LAZY_SWITCH #ifdef SWTCH_OPTIM_STATS + jmp sw1 + +2: /* same address space */ incl swtch_optim_stats -#endif + jmp sw1 - /* switch address space */ - movl %cr3,%ebx /* The same address space? */ - cmpl PCB_CR3(%edx),%ebx - je 4f /* Yes, skip all that cruft */ -#ifdef SWTCH_OPTIM_STATS - decl swtch_optim_stats - incl tlb_flush_count +3: /* kernel address space */ + incl lazy_flush_count +#endif #endif - movl PCB_CR3(%edx),%ebx /* Tell the CPU about the */ - movl %ebx,%cr3 /* new address space */ -4: - movl PCPU(CPUID), %esi +sw1: + /* + * At this point, we've switched address spaces and are ready + * to load up the rest of the next context. + */ cmpl $0, PCB_EXT(%edx) /* has pcb extension? */ je 1f /* If not, use the default */ btsl %esi, private_tss /* mark use of private tss */ @@ -221,11 +253,6 @@ sw1b: movl $GPROC0_SEL*8, %esi /* GSEL(entry, SEL_KPL) */ ltr %si 3: - /* Note in vmspace that this cpu is using it. */ - movl TD_PROC(%ecx),%eax - movl P_VMSPACE(%eax), %ebx - movl PCPU(CPUID), %eax - btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) /* Restore context. */ movl PCB_EBX(%edx),%ebx @@ -241,10 +268,6 @@ sw1b: movl %edx, PCPU(CURPCB) movl %ecx, PCPU(CURTHREAD) /* into next thread */ -#ifdef SMP - /* XXX FIXME: we should be restoring the local APIC TPR */ -#endif - /* * Determine the LDT to use and load it if is the default one and * that is not the current one. @@ -301,12 +324,23 @@ cpu_switch_load_gs: ret #ifdef INVARIANTS +badsw1: + pushal + pushl $sw0_1 + call panic +sw0_1: .asciz "cpu_throw: no newthread supplied" + +badsw2: + pushal + pushl $sw0_2 + call panic +sw0_2: .asciz "cpu_switch: no curthread supplied" + badsw3: pushal pushl $sw0_3 call panic - -sw0_3: .asciz "cpu_switch: choosethread returned NULL" +sw0_3: .asciz "cpu_switch: no newthread supplied" #endif /* |
