diff options
author | marcel <marcel@FreeBSD.org> | 2008-03-30 23:09:14 +0000 |
---|---|---|
committer | marcel <marcel@FreeBSD.org> | 2008-03-30 23:09:14 +0000 |
commit | f4a93f082812d15c72cba54be4552decdf5f0854 (patch) | |
tree | aac37116f631bc8c53402968c1f26d527be88784 /sys/ia64 | |
parent | 39170f049ac46e82c52894edc2a720b83030deaa (diff) | |
download | FreeBSD-src-f4a93f082812d15c72cba54be4552decdf5f0854.zip FreeBSD-src-f4a93f082812d15c72cba54be4552decdf5f0854.tar.gz |
Better implement I-cache invalidation. The previous implementation
was a kluge. This implementation matches the behaviour on powerpc
and sparc64.
While on the subject, make sure to invalidate the I-cache after
loading a kernel module.
MFC after: 2 weeks
Diffstat (limited to 'sys/ia64')
-rw-r--r-- | sys/ia64/ia64/elf_machdep.c | 4 | ||||
-rw-r--r-- | sys/ia64/ia64/machdep.c | 23 | ||||
-rw-r--r-- | sys/ia64/ia64/pmap.c | 121 | ||||
-rw-r--r-- | sys/ia64/include/md_var.h | 1 |
4 files changed, 60 insertions, 89 deletions
diff --git a/sys/ia64/ia64/elf_machdep.c b/sys/ia64/ia64/elf_machdep.c index 4c74630..a6248ca 100644 --- a/sys/ia64/ia64/elf_machdep.c +++ b/sys/ia64/ia64/elf_machdep.c @@ -296,6 +296,10 @@ elf_cpu_load_file(linker_file_t lf) ++ph; } + /* Invalidate the I-cache, but not for the kernel itself. */ + if (lf->id != 1) + ia64_invalidate_icache((uintptr_t)lf->address, lf->size); + return (0); } diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index 4bc743a..149c704 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -100,9 +100,6 @@ __FBSDID("$FreeBSD$"); #include <i386/include/specialreg.h> -/* XXX fc.i kluge (quick fix) */ -extern int ia64_icache_sync_kluge; - u_int64_t processor_frequency; u_int64_t bus_frequency; u_int64_t itc_frequency; @@ -124,6 +121,8 @@ struct fpswa_iface *fpswa_iface; u_int64_t ia64_pal_base; u_int64_t ia64_port_base; +static int ia64_inval_icache_needed; + char machine[] = MACHINE; SYSCTL_STRING(_hw, HW_MACHINE, machine, CTLFLAG_RD, machine, 0, ""); @@ -221,8 +220,7 @@ identifycpu(void) } break; case 0x20: - /* XXX fc.i kluge (quick fix) */ - ia64_icache_sync_kluge = 1; + ia64_inval_icache_needed = 1; family_name = "Itanium 2"; switch (model) { @@ -1513,3 +1511,18 @@ ia64_highfp_save(struct thread *td) KASSERT(thr == td, ("Inconsistent high FP state")); return (1); } + +void +ia64_invalidate_icache(vm_offset_t va, vm_offset_t sz) +{ + vm_offset_t lim; + + if (!ia64_inval_icache_needed) + return; + + lim = va + sz; + while (va < lim) { + __asm __volatile("fc.i %0" :: "r"(va)); + va += 32; /* XXX */ + } +} diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index 0f51240..4c9f280 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -118,9 +118,6 @@ __FBSDID("$FreeBSD$"); /* XXX move to a header. */ extern uint64_t ia64_gateway_page[]; -/* XXX fc.i kluge (quick fix) */ -int ia64_icache_sync_kluge; - MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures"); #ifndef PMAP_SHPGPERPROC @@ -135,6 +132,7 @@ MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures"); #define pmap_accessed(lpte) ((lpte)->pte & PTE_ACCESSED) #define pmap_dirty(lpte) ((lpte)->pte & PTE_DIRTY) +#define pmap_exec(lpte) ((lpte)->pte & PTE_AR_RX) #define pmap_managed(lpte) ((lpte)->pte & PTE_MANAGED) #define pmap_ppn(lpte) ((lpte)->pte & PTE_PPN_MASK) #define pmap_present(lpte) ((lpte)->pte & PTE_PRESENT) @@ -549,53 +547,6 @@ pmap_init(void) * Manipulate TLBs for a pmap ***************************************************/ -#if 0 -static __inline void -pmap_invalidate_page_locally(void *arg) -{ - vm_offset_t va = (uintptr_t)arg; - struct ia64_lpte *pte; - - pte = (struct ia64_lpte *)ia64_thash(va); - if (pte->tag == ia64_ttag(va)) - pte->tag = 1UL << 63; - ia64_ptc_l(va, PAGE_SHIFT << 2); -} - -#ifdef SMP -static void -pmap_invalidate_page_1(void *arg) -{ - void **args = arg; - pmap_t oldpmap; - - critical_enter(); - oldpmap = pmap_switch(args[0]); - pmap_invalidate_page_locally(args[1]); - pmap_switch(oldpmap); - critical_exit(); -} -#endif - -static void -pmap_invalidate_page(pmap_t pmap, vm_offset_t va) -{ - - KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)), - ("invalidating TLB for non-current pmap")); - -#ifdef SMP - if (mp_ncpus > 1) { - void *args[2]; - args[0] = pmap; - args[1] = (void *)va; - smp_rendezvous(NULL, pmap_invalidate_page_1, NULL, args); - } else -#endif - pmap_invalidate_page_locally((void *)va); -} -#endif /* 0 */ - static void pmap_invalidate_page(pmap_t pmap, vm_offset_t va) { @@ -1186,7 +1137,6 @@ static void pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa, boolean_t wired, boolean_t managed) { - vm_offset_t lim; pte->pte &= PTE_PROT_MASK | PTE_PL_MASK | PTE_AR_MASK | PTE_ED; pte->pte |= PTE_PRESENT | PTE_MA_WB; @@ -1197,15 +1147,6 @@ pmap_set_pte(struct ia64_lpte *pte, vm_offset_t va, vm_offset_t pa, pte->itir = PAGE_SHIFT << 2; pte->tag = ia64_ttag(va); - - /* XXX fc.i kluge (quick fix) */ - if (ia64_icache_sync_kluge) { - lim = va + PAGE_SIZE; - while (va < lim) { - __asm __volatile("fc.i %0" :: "r"(va)); - va += 32; - } - } } /* @@ -1521,34 +1462,36 @@ pmap_protect(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, vm_prot_t prot) vm_page_lock_queues(); PMAP_LOCK(pmap); oldpmap = pmap_switch(pmap); - while (sva < eva) { - /* - * If page is invalid, skip this page - */ + for ( ; sva < eva; sva += PAGE_SIZE) { + /* If page is invalid, skip this page */ pte = pmap_find_vhpt(sva); - if (pte == NULL) { - sva += PAGE_SIZE; + if (pte == NULL) + continue; + + /* If there's no change, skip it too */ + if (pmap_prot(pte) == prot) continue; - } - if (pmap_prot(pte) != prot) { - if (pmap_managed(pte)) { - vm_offset_t pa = pmap_ppn(pte); - vm_page_t m = PHYS_TO_VM_PAGE(pa); - if (pmap_dirty(pte)) { - vm_page_dirty(m); - pmap_clear_dirty(pte); - } - if (pmap_accessed(pte)) { - vm_page_flag_set(m, PG_REFERENCED); - pmap_clear_accessed(pte); - } + if (pmap_managed(pte)) { + vm_offset_t pa = pmap_ppn(pte); + vm_page_t m = PHYS_TO_VM_PAGE(pa); + + if (pmap_dirty(pte)) { + vm_page_dirty(m); + pmap_clear_dirty(pte); + } + + if (pmap_accessed(pte)) { + vm_page_flag_set(m, PG_REFERENCED); + pmap_clear_accessed(pte); } - pmap_pte_prot(pmap, pte, prot); - pmap_invalidate_page(pmap, sva); } - sva += PAGE_SIZE; + if (prot & VM_PROT_EXECUTE) + ia64_invalidate_icache(sva, PAGE_SIZE); + + pmap_pte_prot(pmap, pte, prot); + pmap_invalidate_page(pmap, sva); } vm_page_unlock_queues(); pmap_switch(oldpmap); @@ -1576,7 +1519,7 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, vm_offset_t opa; struct ia64_lpte origpte; struct ia64_lpte *pte; - boolean_t managed; + boolean_t icache_inval, managed; vm_page_lock_queues(); PMAP_LOCK(pmap); @@ -1609,6 +1552,8 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, managed = FALSE; pa = VM_PAGE_TO_PHYS(m); + icache_inval = (prot & VM_PROT_EXECUTE) ? TRUE : FALSE; + /* * Mapping has not changed, must be protection or wiring change. */ @@ -1628,10 +1573,14 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_prot_t access, vm_page_t m, /* * We might be turning off write access to the page, - * so we go ahead and sense modify status. + * so we go ahead and sense modify status. Otherwise, + * we can avoid I-cache invalidation if the page + * already allowed execution. */ if (managed && pmap_dirty(&origpte)) vm_page_dirty(m); + else if (pmap_exec(&origpte)) + icache_inval = FALSE; pmap_invalidate_page(pmap, va); goto validate; @@ -1672,6 +1621,10 @@ validate: pmap_pte_prot(pmap, pte, prot); pmap_set_pte(pte, va, pa, wired, managed); + /* Invalidate the I-cache when needed. */ + if (icache_inval) + ia64_invalidate_icache(va, PAGE_SIZE); + if ((prot & VM_PROT_WRITE) != 0) vm_page_flag_set(m, PG_WRITEABLE); vm_page_unlock_queues(); diff --git a/sys/ia64/include/md_var.h b/sys/ia64/include/md_var.h index 0520e57..5fc86fa 100644 --- a/sys/ia64/include/md_var.h +++ b/sys/ia64/include/md_var.h @@ -88,6 +88,7 @@ uint64_t ia64_get_hcdp(void); int ia64_highfp_drop(struct thread *); int ia64_highfp_save(struct thread *); struct ia64_init_return ia64_init(void); +void ia64_invalidate_icache(vm_offset_t, vm_size_t); void ia64_probe_sapics(void); void interrupt(struct trapframe *); void map_gateway_page(void); |