diff options
author | alc <alc@FreeBSD.org> | 1999-04-02 17:59:49 +0000 |
---|---|---|
committer | alc <alc@FreeBSD.org> | 1999-04-02 17:59:49 +0000 |
commit | 11c705d43cba03107a07e70aec6cde667cdb6f64 (patch) | |
tree | 47ee9725506fe292c78ce9d1e5642f9f9e89555f /sys | |
parent | fb816fb02dea8e5cbc77c9ff73a6f9680164e25c (diff) | |
download | FreeBSD-src-11c705d43cba03107a07e70aec6cde667cdb6f64.zip FreeBSD-src-11c705d43cba03107a07e70aec6cde667cdb6f64.tar.gz |
Put in place the infrastructure for improved UP and SMP TLB management.
In particular, replace the unused field pmap::pm_flag by pmap::pm_active,
which is a bit mask representing which processors have the pmap activated.
(Thus, it is a simple Boolean on UPs.)
Also, eliminate an unnecessary memory reference from cpu_switch()
in swtch.s.
Assisted by: John S. Dyson <dyson@iquest.net>
Tested by: Luoqi Chen <luoqi@watermarkgroup.com>,
Poul-Henning Kamp <phk@critter.freebsd.dk>
Diffstat (limited to 'sys')
-rw-r--r-- | sys/amd64/amd64/cpu_switch.S | 43 | ||||
-rw-r--r-- | sys/amd64/amd64/genassym.c | 3 | ||||
-rw-r--r-- | sys/amd64/amd64/pmap.c | 47 | ||||
-rw-r--r-- | sys/amd64/amd64/swtch.s | 43 | ||||
-rw-r--r-- | sys/amd64/include/pmap.h | 7 | ||||
-rw-r--r-- | sys/i386/i386/genassym.c | 3 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 47 | ||||
-rw-r--r-- | sys/i386/i386/swtch.s | 43 | ||||
-rw-r--r-- | sys/i386/include/pmap.h | 7 |
9 files changed, 174 insertions, 69 deletions
diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S index f09c049..7e97a09 100644 --- a/sys/amd64/amd64/cpu_switch.S +++ b/sys/amd64/amd64/cpu_switch.S @@ -33,7 +33,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: swtch.s,v 1.76 1999/03/18 04:22:23 jlemon Exp $ + * $Id: swtch.s,v 1.77 1999/03/20 18:44:13 alc Exp $ */ #include "npx.h" @@ -460,18 +460,25 @@ ENTRY(cpu_switch) movb %al, P_LASTCPU(%ecx) movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */ #endif /* SMP */ + movl P_VMSPACE(%ecx), %edx +#ifdef SMP + movl _cpuid, %eax +#else + xorl %eax, %eax +#endif /* SMP */ + btrl %eax, VM_PMAP+PM_ACTIVE(%edx) - movl P_ADDR(%ecx),%ecx + movl P_ADDR(%ecx),%edx movl (%esp),%eax /* Hardware registers */ - movl %eax,PCB_EIP(%ecx) - movl %ebx,PCB_EBX(%ecx) - movl %esp,PCB_ESP(%ecx) - movl %ebp,PCB_EBP(%ecx) - movl %esi,PCB_ESI(%ecx) - movl %edi,PCB_EDI(%ecx) - movl %fs,PCB_FS(%ecx) - movl %gs,PCB_GS(%ecx) + movl %eax,PCB_EIP(%edx) + movl %ebx,PCB_EBX(%edx) + movl %esp,PCB_ESP(%edx) + movl %ebp,PCB_EBP(%edx) + movl %esi,PCB_ESI(%edx) + movl %edi,PCB_EDI(%edx) + movl %fs,PCB_FS(%edx) + movl %gs,PCB_GS(%edx) #ifdef SMP movl _mp_lock, %eax @@ -481,16 +488,15 @@ ENTRY(cpu_switch) je badsw4 /* yes, bad medicine! */ #endif /* DIAGNOSTIC */ andl $COUNT_FIELD, %eax /* clear CPU portion */ - movl %eax, PCB_MPNEST(%ecx) /* store it */ + movl %eax, PCB_MPNEST(%edx) /* store it */ #endif /* SMP */ #if NNPX > 0 /* have we used fp, and need a save? */ - movl _curproc,%eax - cmpl %eax,_npxproc + cmpl %ecx,_npxproc jne 1f - addl $PCB_SAVEFPU,%ecx /* h/w bugs make saving complicated */ - pushl %ecx + addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */ + pushl %edx call _npxsave /* do it in a big C function */ popl %eax 1: @@ -678,6 +684,13 @@ swtch_com: ltr %si 3: #endif /* VM86 */ + movl P_VMSPACE(%ecx), %ebx +#ifdef SMP + movl _cpuid, %eax +#else + xorl %eax, %eax +#endif + btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) /* restore context */ movl PCB_EBX(%edx),%ebx diff --git a/sys/amd64/amd64/genassym.c b/sys/amd64/amd64/genassym.c index 25ea72f..539f3e2 100644 --- a/sys/amd64/amd64/genassym.c +++ b/sys/amd64/amd64/genassym.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)genassym.c 5.11 (Berkeley) 5/10/91 - * $Id: genassym.c,v 1.63 1999/02/22 15:13:34 bde Exp $ + * $Id: genassym.c,v 1.64 1999/02/28 10:53:28 bde Exp $ */ #include "opt_vm86.h" @@ -84,6 +84,7 @@ main() printf("#define\tP_BACK %#x\n", OS(proc, p_procq.tqe_prev)); printf("#define\tP_VMSPACE %#x\n", OS(proc, p_vmspace)); printf("#define\tVM_PMAP %#x\n", OS(vmspace, vm_pmap)); + printf("#define\tPM_ACTIVE %#x\n", OS(pmap, pm_active)); printf("#define\tP_ADDR %#x\n", OS(proc, p_addr)); printf("#define\tP_PRI %#x\n", OS(proc, p_priority)); printf("#define\tP_RTPRIO_TYPE %#x\n", OS(proc, p_rtprio.type)); diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 92d56ef..ec6c00c 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.224 1999/03/05 08:05:44 alc Exp $ + * $Id: pmap.c,v 1.225 1999/03/13 07:31:29 alc Exp $ */ /* @@ -329,8 +329,8 @@ pmap_bootstrap(firstaddr, loadaddr) kernel_pmap = &kernel_pmap_store; kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD); - kernel_pmap->pm_count = 1; + kernel_pmap->pm_active = -1; /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvlist); nkpt = NKPT; @@ -734,6 +734,34 @@ invltlb_1pg( vm_offset_t va) { } } +static __inline void +pmap_TLB_invalidate(pmap_t pmap, vm_offset_t va) +{ +#if defined(SMP) + if (pmap->pm_active & (1 << cpuid)) + cpu_invlpg((void *)va); + if (pmap->pm_active & other_cpus) + smp_invltlb(); +#else + if (pmap->pm_active) + invltlb_1pg(va); +#endif +} + +static __inline void +pmap_TLB_invalidate_all(pmap_t pmap) +{ +#if defined(SMP) + if (pmap->pm_active & (1 << cpuid)) + cpu_invltlb(); + if (pmap->pm_active & other_cpus) + smp_invltlb(); +#else + if (pmap->pm_active) + invltlb(); +#endif +} + static unsigned * get_ptbase(pmap) pmap_t pmap; @@ -1196,8 +1224,8 @@ pmap_pinit0(pmap) pmap->pm_pdir = (pd_entry_t *)kmem_alloc_pageable(kernel_map, PAGE_SIZE); pmap_kenter((vm_offset_t) pmap->pm_pdir, (vm_offset_t) IdlePTD); - pmap->pm_flags = 0; pmap->pm_count = 1; + pmap->pm_active = 0; pmap->pm_ptphint = NULL; TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -1260,8 +1288,8 @@ pmap_pinit(pmap) *(unsigned *) (pmap->pm_pdir + PTDPTDI) = VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW | PG_A | PG_M; - pmap->pm_flags = 0; pmap->pm_count = 1; + pmap->pm_active = 0; pmap->pm_ptphint = NULL; TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -3391,11 +3419,18 @@ pmap_mincore(pmap, addr) void pmap_activate(struct proc *p) { + pmap_t pmap; + + pmap = vmspace_pmap(p->p_vmspace); +#if defined(SMP) + pmap->pm_active |= 1 << cpuid; +#else + pmap->pm_active |= 1; +#endif #if defined(SWTCH_OPTIM_STATS) tlb_flush_count++; #endif - load_cr3(p->p_addr->u_pcb.pcb_cr3 = - vtophys(vmspace_pmap(p->p_vmspace)->pm_pdir)); + load_cr3(p->p_addr->u_pcb.pcb_cr3 = vtophys(pmap->pm_pdir)); } vm_offset_t diff --git a/sys/amd64/amd64/swtch.s b/sys/amd64/amd64/swtch.s index f09c049..7e97a09 100644 --- a/sys/amd64/amd64/swtch.s +++ b/sys/amd64/amd64/swtch.s @@ -33,7 +33,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: swtch.s,v 1.76 1999/03/18 04:22:23 jlemon Exp $ + * $Id: swtch.s,v 1.77 1999/03/20 18:44:13 alc Exp $ */ #include "npx.h" @@ -460,18 +460,25 @@ ENTRY(cpu_switch) movb %al, P_LASTCPU(%ecx) movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */ #endif /* SMP */ + movl P_VMSPACE(%ecx), %edx +#ifdef SMP + movl _cpuid, %eax +#else + xorl %eax, %eax +#endif /* SMP */ + btrl %eax, VM_PMAP+PM_ACTIVE(%edx) - movl P_ADDR(%ecx),%ecx + movl P_ADDR(%ecx),%edx movl (%esp),%eax /* Hardware registers */ - movl %eax,PCB_EIP(%ecx) - movl %ebx,PCB_EBX(%ecx) - movl %esp,PCB_ESP(%ecx) - movl %ebp,PCB_EBP(%ecx) - movl %esi,PCB_ESI(%ecx) - movl %edi,PCB_EDI(%ecx) - movl %fs,PCB_FS(%ecx) - movl %gs,PCB_GS(%ecx) + movl %eax,PCB_EIP(%edx) + movl %ebx,PCB_EBX(%edx) + movl %esp,PCB_ESP(%edx) + movl %ebp,PCB_EBP(%edx) + movl %esi,PCB_ESI(%edx) + movl %edi,PCB_EDI(%edx) + movl %fs,PCB_FS(%edx) + movl %gs,PCB_GS(%edx) #ifdef SMP movl _mp_lock, %eax @@ -481,16 +488,15 @@ ENTRY(cpu_switch) je badsw4 /* yes, bad medicine! */ #endif /* DIAGNOSTIC */ andl $COUNT_FIELD, %eax /* clear CPU portion */ - movl %eax, PCB_MPNEST(%ecx) /* store it */ + movl %eax, PCB_MPNEST(%edx) /* store it */ #endif /* SMP */ #if NNPX > 0 /* have we used fp, and need a save? */ - movl _curproc,%eax - cmpl %eax,_npxproc + cmpl %ecx,_npxproc jne 1f - addl $PCB_SAVEFPU,%ecx /* h/w bugs make saving complicated */ - pushl %ecx + addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */ + pushl %edx call _npxsave /* do it in a big C function */ popl %eax 1: @@ -678,6 +684,13 @@ swtch_com: ltr %si 3: #endif /* VM86 */ + movl P_VMSPACE(%ecx), %ebx +#ifdef SMP + movl _cpuid, %eax +#else + xorl %eax, %eax +#endif + btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) /* restore context */ movl PCB_EBX(%edx),%ebx diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index 79350a5..f6cd46f 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -42,7 +42,7 @@ * * from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90 * from: @(#)pmap.h 7.4 (Berkeley) 5/12/91 - * $Id: pmap.h,v 1.58 1999/03/02 16:20:39 dg Exp $ + * $Id: pmap.h,v 1.59 1999/03/11 18:28:46 dg Exp $ */ #ifndef _MACHINE_PMAP_H_ @@ -199,16 +199,13 @@ struct pmap { vm_object_t pm_pteobj; /* Container for pte's */ TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ int pm_count; /* reference count */ - int pm_flags; /* pmap flags */ + int pm_active; /* active on cpus */ struct pmap_statistics pm_stats; /* pmap statistics */ struct vm_page *pm_ptphint; /* pmap ptp hint */ }; #define pmap_resident_count(pmap) (pmap)->pm_stats.resident_count -#define PM_FLAG_LOCKED 0x1 -#define PM_FLAG_WANTED 0x2 - typedef struct pmap *pmap_t; #ifdef KERNEL diff --git a/sys/i386/i386/genassym.c b/sys/i386/i386/genassym.c index 25ea72f..539f3e2 100644 --- a/sys/i386/i386/genassym.c +++ b/sys/i386/i386/genassym.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)genassym.c 5.11 (Berkeley) 5/10/91 - * $Id: genassym.c,v 1.63 1999/02/22 15:13:34 bde Exp $ + * $Id: genassym.c,v 1.64 1999/02/28 10:53:28 bde Exp $ */ #include "opt_vm86.h" @@ -84,6 +84,7 @@ main() printf("#define\tP_BACK %#x\n", OS(proc, p_procq.tqe_prev)); printf("#define\tP_VMSPACE %#x\n", OS(proc, p_vmspace)); printf("#define\tVM_PMAP %#x\n", OS(vmspace, vm_pmap)); + printf("#define\tPM_ACTIVE %#x\n", OS(pmap, pm_active)); printf("#define\tP_ADDR %#x\n", OS(proc, p_addr)); printf("#define\tP_PRI %#x\n", OS(proc, p_priority)); printf("#define\tP_RTPRIO_TYPE %#x\n", OS(proc, p_rtprio.type)); diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index 92d56ef..ec6c00c 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.224 1999/03/05 08:05:44 alc Exp $ + * $Id: pmap.c,v 1.225 1999/03/13 07:31:29 alc Exp $ */ /* @@ -329,8 +329,8 @@ pmap_bootstrap(firstaddr, loadaddr) kernel_pmap = &kernel_pmap_store; kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + (u_int)IdlePTD); - kernel_pmap->pm_count = 1; + kernel_pmap->pm_active = -1; /* don't allow deactivation */ TAILQ_INIT(&kernel_pmap->pm_pvlist); nkpt = NKPT; @@ -734,6 +734,34 @@ invltlb_1pg( vm_offset_t va) { } } +static __inline void +pmap_TLB_invalidate(pmap_t pmap, vm_offset_t va) +{ +#if defined(SMP) + if (pmap->pm_active & (1 << cpuid)) + cpu_invlpg((void *)va); + if (pmap->pm_active & other_cpus) + smp_invltlb(); +#else + if (pmap->pm_active) + invltlb_1pg(va); +#endif +} + +static __inline void +pmap_TLB_invalidate_all(pmap_t pmap) +{ +#if defined(SMP) + if (pmap->pm_active & (1 << cpuid)) + cpu_invltlb(); + if (pmap->pm_active & other_cpus) + smp_invltlb(); +#else + if (pmap->pm_active) + invltlb(); +#endif +} + static unsigned * get_ptbase(pmap) pmap_t pmap; @@ -1196,8 +1224,8 @@ pmap_pinit0(pmap) pmap->pm_pdir = (pd_entry_t *)kmem_alloc_pageable(kernel_map, PAGE_SIZE); pmap_kenter((vm_offset_t) pmap->pm_pdir, (vm_offset_t) IdlePTD); - pmap->pm_flags = 0; pmap->pm_count = 1; + pmap->pm_active = 0; pmap->pm_ptphint = NULL; TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -1260,8 +1288,8 @@ pmap_pinit(pmap) *(unsigned *) (pmap->pm_pdir + PTDPTDI) = VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW | PG_A | PG_M; - pmap->pm_flags = 0; pmap->pm_count = 1; + pmap->pm_active = 0; pmap->pm_ptphint = NULL; TAILQ_INIT(&pmap->pm_pvlist); bzero(&pmap->pm_stats, sizeof pmap->pm_stats); @@ -3391,11 +3419,18 @@ pmap_mincore(pmap, addr) void pmap_activate(struct proc *p) { + pmap_t pmap; + + pmap = vmspace_pmap(p->p_vmspace); +#if defined(SMP) + pmap->pm_active |= 1 << cpuid; +#else + pmap->pm_active |= 1; +#endif #if defined(SWTCH_OPTIM_STATS) tlb_flush_count++; #endif - load_cr3(p->p_addr->u_pcb.pcb_cr3 = - vtophys(vmspace_pmap(p->p_vmspace)->pm_pdir)); + load_cr3(p->p_addr->u_pcb.pcb_cr3 = vtophys(pmap->pm_pdir)); } vm_offset_t diff --git a/sys/i386/i386/swtch.s b/sys/i386/i386/swtch.s index f09c049..7e97a09 100644 --- a/sys/i386/i386/swtch.s +++ b/sys/i386/i386/swtch.s @@ -33,7 +33,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: swtch.s,v 1.76 1999/03/18 04:22:23 jlemon Exp $ + * $Id: swtch.s,v 1.77 1999/03/20 18:44:13 alc Exp $ */ #include "npx.h" @@ -460,18 +460,25 @@ ENTRY(cpu_switch) movb %al, P_LASTCPU(%ecx) movb $0xff, P_ONCPU(%ecx) /* "leave" the cpu */ #endif /* SMP */ + movl P_VMSPACE(%ecx), %edx +#ifdef SMP + movl _cpuid, %eax +#else + xorl %eax, %eax +#endif /* SMP */ + btrl %eax, VM_PMAP+PM_ACTIVE(%edx) - movl P_ADDR(%ecx),%ecx + movl P_ADDR(%ecx),%edx movl (%esp),%eax /* Hardware registers */ - movl %eax,PCB_EIP(%ecx) - movl %ebx,PCB_EBX(%ecx) - movl %esp,PCB_ESP(%ecx) - movl %ebp,PCB_EBP(%ecx) - movl %esi,PCB_ESI(%ecx) - movl %edi,PCB_EDI(%ecx) - movl %fs,PCB_FS(%ecx) - movl %gs,PCB_GS(%ecx) + movl %eax,PCB_EIP(%edx) + movl %ebx,PCB_EBX(%edx) + movl %esp,PCB_ESP(%edx) + movl %ebp,PCB_EBP(%edx) + movl %esi,PCB_ESI(%edx) + movl %edi,PCB_EDI(%edx) + movl %fs,PCB_FS(%edx) + movl %gs,PCB_GS(%edx) #ifdef SMP movl _mp_lock, %eax @@ -481,16 +488,15 @@ ENTRY(cpu_switch) je badsw4 /* yes, bad medicine! */ #endif /* DIAGNOSTIC */ andl $COUNT_FIELD, %eax /* clear CPU portion */ - movl %eax, PCB_MPNEST(%ecx) /* store it */ + movl %eax, PCB_MPNEST(%edx) /* store it */ #endif /* SMP */ #if NNPX > 0 /* have we used fp, and need a save? */ - movl _curproc,%eax - cmpl %eax,_npxproc + cmpl %ecx,_npxproc jne 1f - addl $PCB_SAVEFPU,%ecx /* h/w bugs make saving complicated */ - pushl %ecx + addl $PCB_SAVEFPU,%edx /* h/w bugs make saving complicated */ + pushl %edx call _npxsave /* do it in a big C function */ popl %eax 1: @@ -678,6 +684,13 @@ swtch_com: ltr %si 3: #endif /* VM86 */ + movl P_VMSPACE(%ecx), %ebx +#ifdef SMP + movl _cpuid, %eax +#else + xorl %eax, %eax +#endif + btsl %eax, VM_PMAP+PM_ACTIVE(%ebx) /* restore context */ movl PCB_EBX(%edx),%ebx diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h index 79350a5..f6cd46f 100644 --- a/sys/i386/include/pmap.h +++ b/sys/i386/include/pmap.h @@ -42,7 +42,7 @@ * * from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90 * from: @(#)pmap.h 7.4 (Berkeley) 5/12/91 - * $Id: pmap.h,v 1.58 1999/03/02 16:20:39 dg Exp $ + * $Id: pmap.h,v 1.59 1999/03/11 18:28:46 dg Exp $ */ #ifndef _MACHINE_PMAP_H_ @@ -199,16 +199,13 @@ struct pmap { vm_object_t pm_pteobj; /* Container for pte's */ TAILQ_HEAD(,pv_entry) pm_pvlist; /* list of mappings in pmap */ int pm_count; /* reference count */ - int pm_flags; /* pmap flags */ + int pm_active; /* active on cpus */ struct pmap_statistics pm_stats; /* pmap statistics */ struct vm_page *pm_ptphint; /* pmap ptp hint */ }; #define pmap_resident_count(pmap) (pmap)->pm_stats.resident_count -#define PM_FLAG_LOCKED 0x1 -#define PM_FLAG_WANTED 0x2 - typedef struct pmap *pmap_t; #ifdef KERNEL |