diff options
Diffstat (limited to 'sys/i386')
-rw-r--r-- | sys/i386/i386/initcpu.c | 12 | ||||
-rw-r--r-- | sys/i386/i386/machdep.c | 3 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 101 | ||||
-rw-r--r-- | sys/i386/i386/vm_machdep.c | 6 | ||||
-rw-r--r-- | sys/i386/include/cpufunc.h | 7 | ||||
-rw-r--r-- | sys/i386/isa/npx.c | 10 |
6 files changed, 100 insertions, 39 deletions
diff --git a/sys/i386/i386/initcpu.c b/sys/i386/i386/initcpu.c index 7025021..bc6a55b 100644 --- a/sys/i386/i386/initcpu.c +++ b/sys/i386/i386/initcpu.c @@ -826,14 +826,18 @@ initializecpucache(void) * CPUID_SS feature even though the native CPU supports it. */ TUNABLE_INT_FETCH("hw.clflush_disable", &hw_clflush_disable); - if (vm_guest != VM_GUEST_NO && hw_clflush_disable == -1) + if (vm_guest != VM_GUEST_NO && hw_clflush_disable == -1) { cpu_feature &= ~CPUID_CLFSH; + cpu_stdext_feature &= ~CPUID_STDEXT_CLFLUSHOPT; + } /* - * Allow to disable CLFLUSH feature manually by - * hw.clflush_disable tunable. + * The kernel's use of CLFLUSH{,OPT} can be disabled manually + * by setting the hw.clflush_disable tunable. */ - if (hw_clflush_disable == 1) + if (hw_clflush_disable == 1) { cpu_feature &= ~CPUID_CLFSH; + cpu_stdext_feature &= ~CPUID_STDEXT_CLFLUSHOPT; + } #if defined(PC98) && !defined(CPU_UPGRADE_HW_CACHE) /* diff --git a/sys/i386/i386/machdep.c b/sys/i386/i386/machdep.c index f55900c..607b08b 100644 --- a/sys/i386/i386/machdep.c +++ b/sys/i386/i386/machdep.c @@ -3157,8 +3157,7 @@ init386(first) #else register_t -init386(first) - int first; +init386(int first) { struct gate_descriptor *gdp; int gsel_tss, metadata_missing, x, pa; diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index 299ee77..c383f34 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -517,7 +517,14 @@ pmap_bootstrap(vm_paddr_t firstaddr) for (i = 1; i < NKPT; i++) PTD[i] = 0; - /* Initialize the PAT MSR if present. */ + /* + * Initialize the PAT MSR if present. + * pmap_init_pat() clears and sets CR4_PGE, which, as a + * side-effect, invalidates stale PG_G TLB entries that might + * have been created in our pre-boot environment. We assume + * that PAT support implies PGE and in reverse, PGE presence + * comes with PAT. Both features were added for Pentium Pro. + */ pmap_init_pat(); /* Turn on PG_G on kernel page(s) */ @@ -545,7 +552,10 @@ pmap_init_pat(void) pat_table[PAT_WRITE_PROTECTED] = 3; pat_table[PAT_UNCACHED] = 3; - /* Bail if this CPU doesn't implement PAT. */ + /* + * Bail if this CPU doesn't implement PAT. + * We assume that PAT support implies PGE. + */ if ((cpu_feature & CPUID_PAT) == 0) { for (i = 0; i < PAT_INDEX_SIZE; i++) pat_index[i] = pat_table[i]; @@ -1222,9 +1232,8 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force) if ((cpu_feature & CPUID_SS) != 0 && !force) ; /* If "Self Snoop" is supported and allowed, do nothing. */ - else if ((cpu_feature & CPUID_CLFSH) != 0 && + else if ((cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0 && eva - sva < PMAP_CLFLUSH_THRESHOLD) { - #ifdef DEV_APIC /* * XXX: Some CPUs fault, hang, or trash the local APIC @@ -1236,16 +1245,29 @@ pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva, boolean_t force) return; #endif /* - * Otherwise, do per-cache line flush. Use the mfence + * Otherwise, do per-cache line flush. Use the sfence * instruction to insure that previous stores are * included in the write-back. The processor * propagates flush to other processors in the cache * coherence domain. */ - mfence(); + sfence(); + for (; sva < eva; sva += cpu_clflush_line_size) + clflushopt(sva); + sfence(); + } else if ((cpu_feature & CPUID_CLFSH) != 0 && + eva - sva < PMAP_CLFLUSH_THRESHOLD) { + if (pmap_kextract(sva) == lapic_paddr) + return; + /* + * Writes are ordered by CLFLUSH on Intel CPUs. + */ + if (cpu_vendor_id != CPU_VENDOR_INTEL) + mfence(); for (; sva < eva; sva += cpu_clflush_line_size) clflush(sva); - mfence(); + if (cpu_vendor_id != CPU_VENDOR_INTEL) + mfence(); } else { /* @@ -2675,6 +2697,7 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) vm_paddr_t mptepa; vm_page_t mpte; struct spglist free; + vm_offset_t sva; PMAP_LOCK_ASSERT(pmap, MA_OWNED); oldpde = *pde; @@ -2697,8 +2720,9 @@ pmap_demote_pde(pmap_t pmap, pd_entry_t *pde, vm_offset_t va) va >> PDRSHIFT, VM_ALLOC_NOOBJ | VM_ALLOC_NORMAL | VM_ALLOC_WIRED)) == NULL) { SLIST_INIT(&free); - pmap_remove_pde(pmap, pde, trunc_4mpage(va), &free); - pmap_invalidate_page(pmap, trunc_4mpage(va)); + sva = trunc_4mpage(va); + pmap_remove_pde(pmap, pde, sva, &free); + pmap_invalidate_range(pmap, sva, sva + NBPDR - 1); pmap_free_zero_pages(&free); CTR2(KTR_PMAP, "pmap_demote_pde: failure for va %#x" " in pmap %p", va, pmap); @@ -2869,9 +2893,24 @@ pmap_remove_pde(pmap_t pmap, pd_entry_t *pdq, vm_offset_t sva, /* * Machines that don't support invlpg, also don't support * PG_G. + * + * When workaround_erratum383 is false, a promotion to a 2M/4M + * page mapping does not invalidate the 512/1024 4K page mappings + * from the TLB. Consequently, at this point, the TLB may + * hold both 4K and 2M/4M page mappings. Therefore, the entire + * range of addresses must be invalidated here. In contrast, + * when workaround_erratum383 is true, a promotion does + * invalidate the 512/1024 4K page mappings, and so a single INVLPG + * suffices to invalidate the 2M/4M page mapping. */ - if (oldpde & PG_G) - pmap_invalidate_page(kernel_pmap, sva); + if ((oldpde & PG_G) != 0) { + if (workaround_erratum383) + pmap_invalidate_page(kernel_pmap, sva); + else + pmap_invalidate_range(kernel_pmap, sva, + sva + NBPDR - 1); + } + pmap->pm_stats.resident_count -= NBPDR / PAGE_SIZE; if (oldpde & PG_MANAGED) { pvh = pa_to_pvh(oldpde & PG_PS_FRAME); @@ -3181,9 +3220,14 @@ retry: if (newpde != oldpde) { if (!pde_cmpset(pde, oldpde, newpde)) goto retry; - if (oldpde & PG_G) - pmap_invalidate_page(pmap, sva); - else + if (oldpde & PG_G) { + /* See pmap_remove_pde() for explanation. */ + if (workaround_erratum383) + pmap_invalidate_page(kernel_pmap, sva); + else + pmap_invalidate_range(kernel_pmap, sva, + sva + NBPDR - 1); + } else anychanged = TRUE; } return (anychanged); @@ -5316,8 +5360,10 @@ pmap_flush_page(vm_page_t m) { struct sysmaps *sysmaps; vm_offset_t sva, eva; + bool useclflushopt; - if ((cpu_feature & CPUID_CLFSH) != 0) { + useclflushopt = (cpu_stdext_feature & CPUID_STDEXT_CLFLUSHOPT) != 0; + if (useclflushopt || (cpu_feature & CPUID_CLFSH) != 0) { sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; mtx_lock(&sysmaps->lock); if (*sysmaps->CMAP2) @@ -5330,14 +5376,25 @@ pmap_flush_page(vm_page_t m) eva = sva + PAGE_SIZE; /* - * Use mfence despite the ordering implied by - * mtx_{un,}lock() because clflush is not guaranteed - * to be ordered by any other instruction. + * Use mfence or sfence despite the ordering implied by + * mtx_{un,}lock() because clflush on non-Intel CPUs + * and clflushopt are not guaranteed to be ordered by + * any other instruction. */ - mfence(); - for (; sva < eva; sva += cpu_clflush_line_size) - clflush(sva); - mfence(); + if (useclflushopt) + sfence(); + else if (cpu_vendor_id != CPU_VENDOR_INTEL) + mfence(); + for (; sva < eva; sva += cpu_clflush_line_size) { + if (useclflushopt) + clflushopt(sva); + else + clflush(sva); + } + if (useclflushopt) + sfence(); + else if (cpu_vendor_id != CPU_VENDOR_INTEL) + mfence(); *sysmaps->CMAP2 = 0; sched_unpin(); mtx_unlock(&sysmaps->lock); diff --git a/sys/i386/i386/vm_machdep.c b/sys/i386/i386/vm_machdep.c index c0e65d5..a76a5ec 100644 --- a/sys/i386/i386/vm_machdep.c +++ b/sys/i386/i386/vm_machdep.c @@ -209,11 +209,7 @@ alloc_fpusave(int flags) * ready to run and return to user mode. */ void -cpu_fork(td1, p2, td2, flags) - register struct thread *td1; - register struct proc *p2; - struct thread *td2; - int flags; +cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags) { register struct proc *p1; struct pcb *pcb2; diff --git a/sys/i386/include/cpufunc.h b/sys/i386/include/cpufunc.h index d3a7e08..02ed27a 100644 --- a/sys/i386/include/cpufunc.h +++ b/sys/i386/include/cpufunc.h @@ -175,6 +175,13 @@ mfence(void) __asm __volatile("mfence" : : : "memory"); } +static __inline void +sfence(void) +{ + + __asm __volatile("sfence" : : : "memory"); +} + #ifdef _KERNEL #define HAVE_INLINE_FFS diff --git a/sys/i386/isa/npx.c b/sys/i386/isa/npx.c index dff0f5c..bc42bbb 100644 --- a/sys/i386/isa/npx.c +++ b/sys/i386/isa/npx.c @@ -559,8 +559,7 @@ SYSINIT(npxinitstate, SI_SUB_DRIVERS, SI_ORDER_ANY, npxinitstate, NULL); * Free coprocessor (if we have it). */ void -npxexit(td) - struct thread *td; +npxexit(struct thread *td) { critical_enter(); @@ -590,7 +589,7 @@ npxexit(td) } int -npxformat() +npxformat(void) { if (!hw_float) @@ -970,7 +969,7 @@ npxresume(union savefpu *addr) } void -npxdrop() +npxdrop(void) { struct thread *td; @@ -1203,8 +1202,7 @@ fpu_clean_state(void) #endif /* CPU_ENABLE_SSE */ static void -fpurstor(addr) - union savefpu *addr; +fpurstor(union savefpu *addr) { #ifdef CPU_ENABLE_SSE |