From 29d37118d7d41f1e2a517987a8d568df28c133bc Mon Sep 17 00:00:00 2001 From: skra Date: Tue, 24 Nov 2015 13:57:41 +0000 Subject: Flush all kernel mappings from TLB(s) in time when they are cleared. Replace tlb_flush_local() by tlb_flush() as even not global mappings could be fetched to TLB(s) on other cores by speculative table walk. From OS point of view, it was not a problem as either such mappings were not used anymore or they were flushed from TLB(s) when reused. However, from hardware point of view, it was a problem. Not flushed mappings could be a target for speculative reads or prefetches (which might be quite aggresive on ARM cores). As speculative read can fill cacheline, it can cause a real problem, when physical page is reused, but mapped with different memory attributes. Anyhow, it's good to have only valid mappings in TLB(s). Approved by: kib (mentor) --- sys/arm/arm/pmap-v6-new.c | 27 +++++++++++++-------------- sys/arm/include/sf_buf.h | 4 ++++ 2 files changed, 17 insertions(+), 14 deletions(-) (limited to 'sys/arm') diff --git a/sys/arm/arm/pmap-v6-new.c b/sys/arm/arm/pmap-v6-new.c index d553eb4..8b7676e 100644 --- a/sys/arm/arm/pmap-v6-new.c +++ b/sys/arm/arm/pmap-v6-new.c @@ -1541,12 +1541,12 @@ pmap_pt2pg_zero(vm_page_t m) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(pa, PTE2_AP_KRW, m->md.pat_mode)); - tlb_flush_local((vm_offset_t)sysmaps->CADDR2); /* Even VM_ALLOC_ZERO request is only advisory. */ if ((m->flags & PG_ZERO) == 0) pagezero(sysmaps->CADDR2); pte2_sync_range((pt2_entry_t *)sysmaps->CADDR2, PAGE_SIZE); pte2_clear(sysmaps->CMAP2); + tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); @@ -5470,7 +5470,6 @@ pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) struct sysmaps *sysmaps; vm_memattr_t oma; vm_paddr_t pa; - vm_offset_t va; oma = m->md.pat_mode; m->md.pat_mode = ma; @@ -5502,10 +5501,9 @@ pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma) if (*sysmaps->CMAP2) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(pa, PTE2_AP_KRW, ma)); - va = (vm_offset_t)sysmaps->CADDR2; - tlb_flush_local(va); - dcache_wbinv_poc(va, pa, PAGE_SIZE); + dcache_wbinv_poc((vm_offset_t)sysmaps->CADDR2, pa, PAGE_SIZE); pte2_clear(sysmaps->CMAP2); + tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -5594,9 +5592,9 @@ pmap_zero_page(vm_page_t m) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, m->md.pat_mode)); - tlb_flush_local((vm_offset_t)sysmaps->CADDR2); pagezero(sysmaps->CADDR2); pte2_clear(sysmaps->CMAP2); + tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -5619,12 +5617,12 @@ pmap_zero_page_area(vm_page_t m, int off, int size) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, m->md.pat_mode)); - tlb_flush_local((vm_offset_t)sysmaps->CADDR2); if (off == 0 && size == PAGE_SIZE) pagezero(sysmaps->CADDR2); else bzero(sysmaps->CADDR2 + off, size); pte2_clear(sysmaps->CMAP2); + tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -5644,9 +5642,9 @@ pmap_zero_page_idle(vm_page_t m) sched_pin(); pte2_store(CMAP3, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, m->md.pat_mode)); - tlb_flush_local((vm_offset_t)CADDR3); pagezero(CADDR3); pte2_clear(CMAP3); + tlb_flush((vm_offset_t)CADDR3); sched_unpin(); } @@ -5670,13 +5668,13 @@ pmap_copy_page(vm_page_t src, vm_page_t dst) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP1, PTE2_KERN_NG(VM_PAGE_TO_PHYS(src), PTE2_AP_KR | PTE2_NM, src->md.pat_mode)); - tlb_flush_local((vm_offset_t)sysmaps->CADDR1); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(dst), PTE2_AP_KRW, dst->md.pat_mode)); - tlb_flush_local((vm_offset_t)sysmaps->CADDR2); bcopy(sysmaps->CADDR1, sysmaps->CADDR2, PAGE_SIZE); pte2_clear(sysmaps->CMAP1); + tlb_flush((vm_offset_t)sysmaps->CADDR1); pte2_clear(sysmaps->CMAP2); + tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -5721,7 +5719,9 @@ pmap_copy_pages(vm_page_t ma[], vm_offset_t a_offset, vm_page_t mb[], xfersize -= cnt; } pte2_clear(sysmaps->CMAP1); + tlb_flush((vm_offset_t)sysmaps->CADDR1); pte2_clear(sysmaps->CMAP2); + tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -5740,8 +5740,6 @@ pmap_quick_enter_page(vm_page_t m) pte2_store(pte2p, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, pmap_page_get_memattr(m))); - tlb_flush_local(qmap_addr); - return (qmap_addr); } @@ -5758,6 +5756,7 @@ pmap_quick_remove_page(vm_offset_t addr) KASSERT(pte2_load(pte2p) != 0, ("%s: PTE2 not in use", __func__)); pte2_clear(pte2p); + tlb_flush(qmap_addr); critical_exit(); } @@ -6059,9 +6058,9 @@ pmap_dcache_wb_pou(vm_paddr_t pa, vm_size_t size, vm_memattr_t ma) if (*sysmaps->CMAP3) panic("%s: CMAP3 busy", __func__); pte2_store(sysmaps->CMAP3, PTE2_KERN_NG(pa, PTE2_AP_KRW, ma)); - tlb_flush_local((vm_offset_t)sysmaps->CADDR3); dcache_wb_pou((vm_offset_t)sysmaps->CADDR3 + (pa & PAGE_MASK), size); pte2_clear(sysmaps->CMAP3); + tlb_flush((vm_offset_t)sysmaps->CADDR3); sched_unpin(); mtx_unlock(&sysmaps->lock); } @@ -6313,13 +6312,13 @@ pmap_zero_page_check(vm_page_t m) panic("%s: CMAP2 busy", __func__); pte2_store(sysmaps->CMAP2, PTE2_KERN_NG(VM_PAGE_TO_PHYS(m), PTE2_AP_KRW, m->md.pat_mode)); - tlb_flush_local((vm_offset_t)sysmaps->CADDR2); end = (uint32_t*)(sysmaps->CADDR2 + PAGE_SIZE); for (p = (uint32_t*)sysmaps->CADDR2; p < end; p++) if (*p != 0) panic("%s: page %p not zero, va: %p", __func__, m, sysmaps->CADDR2); pte2_clear(sysmaps->CMAP2); + tlb_flush((vm_offset_t)sysmaps->CADDR2); sched_unpin(); mtx_unlock(&sysmaps->lock); } diff --git a/sys/arm/include/sf_buf.h b/sys/arm/include/sf_buf.h index 75d5722..eba8488 100644 --- a/sys/arm/include/sf_buf.h +++ b/sys/arm/include/sf_buf.h @@ -44,7 +44,11 @@ static inline int sf_buf_unmap(struct sf_buf *sf) { +#ifdef ARM_NEW_PMAP + pmap_qremove(sf->kva, 1); +#else pmap_kremove(sf->kva); +#endif return (1); } #endif /* !_MACHINE_SF_BUF_H_ */ -- cgit v1.1