diff options
author | kib <kib@FreeBSD.org> | 2009-07-29 19:38:33 +0000 |
---|---|---|
committer | kib <kib@FreeBSD.org> | 2009-07-29 19:38:33 +0000 |
commit | eafde3dc55d5265075b824d9b12983759e0e2c50 (patch) | |
tree | 164a73e1c29a2d69202d21542e07e762e70ed7aa /sys/i386 | |
parent | 0991e073e63d90ed3dac07a1ffbcb25463bd1cba (diff) | |
download | FreeBSD-src-eafde3dc55d5265075b824d9b12983759e0e2c50.zip FreeBSD-src-eafde3dc55d5265075b824d9b12983759e0e2c50.tar.gz |
Fix XEN build breakage, by implementing pmap_invalidate_cache_range()
and using it when appropriate. Merge analogue of the r195836
optimization to XEN.
Approved by: re (kensmith)
Diffstat (limited to 'sys/i386')
-rw-r--r-- | sys/i386/xen/pmap.c | 104 |
1 files changed, 88 insertions, 16 deletions
diff --git a/sys/i386/xen/pmap.c b/sys/i386/xen/pmap.c index a7c1b2a..9dc077f 100644 --- a/sys/i386/xen/pmap.c +++ b/sys/i386/xen/pmap.c @@ -121,6 +121,7 @@ __FBSDID("$FreeBSD$"); #include <sys/msgbuf.h> #include <sys/mutex.h> #include <sys/proc.h> +#include <sys/sf_buf.h> #include <sys/sx.h> #include <sys/vmmeter.h> #include <sys/sched.h> @@ -759,7 +760,7 @@ pmap_init(void) * Determine the appropriate bits to set in a PTE or PDE for a specified * caching mode. */ -static int +int pmap_cache_bits(int mode, boolean_t is_pde) { int pat_flag, pat_index, cache_bits; @@ -988,6 +989,40 @@ pmap_invalidate_cache(void) } #endif /* !SMP */ +void +pmap_invalidate_cache_range(vm_offset_t sva, vm_offset_t eva) +{ + + KASSERT((sva & PAGE_MASK) == 0, + ("pmap_invalidate_cache_range: sva not page-aligned")); + KASSERT((eva & PAGE_MASK) == 0, + ("pmap_invalidate_cache_range: eva not page-aligned")); + + if (cpu_feature & CPUID_SS) + ; /* If "Self Snoop" is supported, do nothing. */ + else if (cpu_feature & CPUID_CLFSH) { + + /* + * Otherwise, do per-cache line flush. Use the mfence + * instruction to insure that previous stores are + * included in the write-back. The processor + * propagates flush to other processors in the cache + * coherence domain. + */ + mfence(); + for (; eva < sva; eva += cpu_clflush_line_size) + clflush(eva); + mfence(); + } else { + + /* + * No targeted cache flush methods are supported by CPU, + * globally invalidate cache as a last resort. + */ + pmap_invalidate_cache(); + } +} + /* * Are we current address space or kernel? N.B. We return FALSE when * a pmap's page table is in use because a kernel thread is borrowing @@ -3865,7 +3900,8 @@ pmap_clear_reference(vm_page_t m) void * pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) { - vm_offset_t va, tmpva, offset; + vm_offset_t va, offset; + vm_size_t tmpsize; offset = pa & PAGE_MASK; size = roundup(offset + size, PAGE_SIZE); @@ -3878,14 +3914,10 @@ pmap_mapdev_attr(vm_paddr_t pa, vm_size_t size, int mode) if (!va) panic("pmap_mapdev: Couldn't alloc kernel virtual memory"); - for (tmpva = va; size > 0; ) { - pmap_kenter_attr(tmpva, pa, mode); - size -= PAGE_SIZE; - tmpva += PAGE_SIZE; - pa += PAGE_SIZE; - } - pmap_invalidate_range(kernel_pmap, va, tmpva); - pmap_invalidate_cache(); + for (tmpsize = 0; tmpsize < size; tmpsize += PAGE_SIZE) + pmap_kenter_attr(va + tmpsize, pa + tmpsize, mode); + pmap_invalidate_range(kernel_pmap, va, va + tmpsize); + pmap_invalidate_cache_range(va, va + size); return ((void *)(va + offset)); } @@ -3927,16 +3959,49 @@ pmap_unmapdev(vm_offset_t va, vm_size_t size) void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) { + struct sysmaps *sysmaps; + vm_offset_t sva, eva; m->md.pat_mode = ma; + if ((m->flags & PG_FICTITIOUS) != 0) + return; /* * If "m" is a normal page, flush it from the cache. + * See pmap_invalidate_cache_range(). + * + * First, try to find an existing mapping of the page by sf + * buffer. sf_buf_invalidate_cache() modifies mapping and + * flushes the cache. */ - if ((m->flags & PG_FICTITIOUS) == 0) { - /* If "Self Snoop" is supported, do nothing. */ - if (!(cpu_feature & CPUID_SS)) - pmap_invalidate_cache(); + if (sf_buf_invalidate_cache(m)) + return; + + /* + * If page is not mapped by sf buffer, but CPU does not + * support self snoop, map the page transient and do + * invalidation. In the worst case, whole cache is flushed by + * pmap_invalidate_cache_range(). + */ + if ((cpu_feature & (CPUID_SS|CPUID_CLFSH)) == CPUID_CLFSH) { + sysmaps = &sysmaps_pcpu[PCPU_GET(cpuid)]; + mtx_lock(&sysmaps->lock); + if (*sysmaps->CMAP2) + panic("pmap_page_set_memattr: CMAP2 busy"); + sched_pin(); + PT_SET_MA(sysmaps->CADDR2, PG_V | PG_RW | + xpmap_ptom(VM_PAGE_TO_PHYS(m)) | PG_A | PG_M | + pmap_cache_bits(m->md.pat_mode, 0)); + invlcaddr(sysmaps->CADDR2); + sva = (vm_offset_t)sysmaps->CADDR2; + eva = sva + PAGE_SIZE; + } else + sva = eva = 0; /* gcc */ + pmap_invalidate_cache_range(sva, eva); + if (sva != 0) { + PT_SET_MA(sysmaps->CADDR2, 0); + sched_unpin(); + mtx_unlock(&sysmaps->lock); } } @@ -3950,6 +4015,7 @@ pmap_change_attr(va, size, mode) pt_entry_t *pte; u_int opte, npte; pd_entry_t *pde; + boolean_t changed; base = trunc_page(va); offset = va & PAGE_MASK; @@ -3971,6 +4037,8 @@ pmap_change_attr(va, size, mode) return (EINVAL); } + changed = FALSE; + /* * Ok, all the pages exist and are 4k, so run through them updating * their cache mode. @@ -3988,6 +4056,8 @@ pmap_change_attr(va, size, mode) npte |= pmap_cache_bits(mode, 0); PT_SET_VA_MA(pte, npte, TRUE); } while (npte != opte && (*pte != npte)); + if (npte != opte) + changed = TRUE; tmpva += PAGE_SIZE; size -= PAGE_SIZE; } @@ -3996,8 +4066,10 @@ pmap_change_attr(va, size, mode) * Flush CPU caches to make sure any data isn't cached that shouldn't * be, etc. */ - pmap_invalidate_range(kernel_pmap, base, tmpva); - pmap_invalidate_cache(); + if (changed) { + pmap_invalidate_range(kernel_pmap, base, tmpva); + pmap_invalidate_cache_range(base, tmpva); + } return (0); } |