From 93b3a3577a591307940bc50af949b9c35d3ad2f6 Mon Sep 17 00:00:00 2001 From: alc Date: Sun, 25 Jun 2006 19:07:01 +0000 Subject: Synchronize accesses to the PTEG table. Add many lock assertions. Tested by: grehan@ --- sys/powerpc/aim/mmu_oea.c | 32 +++++++++++++++++++++++++++++--- sys/powerpc/powerpc/mmu_oea.c | 32 +++++++++++++++++++++++++++++--- 2 files changed, 58 insertions(+), 6 deletions(-) (limited to 'sys/powerpc') diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index 8cd8c2e..a50fed4 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -434,6 +434,7 @@ static __inline void moea_attr_clear(vm_page_t m, int ptebit) { + mtx_assert(&vm_page_queue_mtx, MA_OWNED); m->md.mdpg_attrs &= ~ptebit; } @@ -448,6 +449,7 @@ static __inline void moea_attr_save(vm_page_t m, int ptebit) { + mtx_assert(&vm_page_queue_mtx, MA_OWNED); m->md.mdpg_attrs |= ptebit; } @@ -471,6 +473,9 @@ moea_pte_match(struct pte *pt, u_int sr, vm_offset_t va, int which) static __inline void moea_pte_create(struct pte *pt, u_int sr, vm_offset_t va, u_int pte_lo) { + + mtx_assert(&moea_table_mutex, MA_OWNED); + /* * Construct a PTE. Default to IMB initially. Valid bit only gets * set when the real pte is set in memory. @@ -486,6 +491,7 @@ static __inline void moea_pte_synch(struct pte *pt, struct pte *pvo_pt) { + mtx_assert(&moea_table_mutex, MA_OWNED); pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF | PTE_CHG); } @@ -493,6 +499,8 @@ static __inline void moea_pte_clear(struct pte *pt, vm_offset_t va, int ptebit) { + mtx_assert(&moea_table_mutex, MA_OWNED); + /* * As shown in Section 7.6.3.2.3 */ @@ -507,6 +515,7 @@ static __inline void moea_pte_set(struct pte *pt, struct pte *pvo_pt) { + mtx_assert(&moea_table_mutex, MA_OWNED); pvo_pt->pte_hi |= PTE_VALID; /* @@ -525,6 +534,7 @@ static __inline void moea_pte_unset(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) { + mtx_assert(&moea_table_mutex, MA_OWNED); pvo_pt->pte_hi &= ~PTE_VALID; /* @@ -766,7 +776,8 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) * Initialize the lock that synchronizes access to the pteg and pvo * tables. */ - mtx_init(&moea_table_mutex, "pmap table", NULL, MTX_DEF); + mtx_init(&moea_table_mutex, "pmap table", NULL, MTX_DEF | + MTX_RECURSE); /* * Allocate the message buffer. @@ -1461,8 +1472,10 @@ moea_page_protect(mmu_t mmu, vm_page_t m, vm_prot_t prot) pt = moea_pvo_to_pte(pvo, -1); pvo->pvo_pte.pte_lo &= ~PTE_PP; pvo->pvo_pte.pte_lo |= PTE_BR; - if (pt != NULL) + if (pt != NULL) { moea_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); + } PMAP_UNLOCK(pmap); MOEA_PVO_CHECK(pvo); /* sanity check */ } @@ -1616,8 +1629,10 @@ moea_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva, /* * If the PVO is in the page table, update that pte as well. */ - if (pt != NULL) + if (pt != NULL) { moea_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); + } } vm_page_unlock_queues(); PMAP_UNLOCK(pm); @@ -1795,6 +1810,7 @@ moea_rkva_alloc(mmu_t mmu) panic("moea_kva_alloc: moea_pvo_to_pte failed"); moea_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); PVO_PTEGIDX_CLR(pvo); moea_pte_overflow++; @@ -1818,6 +1834,7 @@ moea_pa_map(struct pvo_entry *pvo, vm_offset_t pa, struct pte *saved_pt, if (pt != NULL) { moea_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); PVO_PTEGIDX_CLR(pvo); moea_pte_overflow++; } @@ -1845,6 +1862,7 @@ moea_pa_unmap(struct pvo_entry *pvo, struct pte *saved_pt, int *depth_p) if (pt != NULL) { moea_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); PVO_PTEGIDX_CLR(pvo); moea_pte_overflow++; } @@ -2003,6 +2021,7 @@ moea_pvo_remove(struct pvo_entry *pvo, int pteidx) pt = moea_pvo_to_pte(pvo, pteidx); if (pt != NULL) { moea_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); PVO_PTEGIDX_CLR(pvo); } else { moea_pte_overflow--; @@ -2104,6 +2123,7 @@ moea_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) } pt = &moea_pteg_table[pteidx >> 3].pt[pteidx & 7]; + mtx_lock(&moea_table_mutex); if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { panic("moea_pvo_to_pte: pvo %p has valid pte in pvo but no " @@ -2127,6 +2147,7 @@ moea_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) "pte %p in moea_pteg_table", pvo, pt); } + mtx_assert(&moea_table_mutex, MA_OWNED); return (pt); } @@ -2135,6 +2156,7 @@ moea_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) "moea_pteg_table but valid in pvo", pvo, pt); } + mtx_unlock(&moea_table_mutex); return (NULL); } @@ -2267,6 +2289,8 @@ moea_pte_insert(u_int ptegidx, struct pte *pvo_pt) struct pte *pt; int i; + mtx_assert(&moea_table_mutex, MA_OWNED); + /* * First try primary hash. */ @@ -2337,6 +2361,7 @@ moea_query_bit(vm_page_t m, int ptebit) pt = moea_pvo_to_pte(pvo, -1); if (pt != NULL) { moea_pte_synch(pt, &pvo->pvo_pte); + mtx_unlock(&moea_table_mutex); if (pvo->pvo_pte.pte_lo & ptebit) { moea_attr_save(m, ptebit); MOEA_PVO_CHECK(pvo); /* sanity check */ @@ -2385,6 +2410,7 @@ moea_clear_bit(vm_page_t m, int ptebit, int *origbit) count++; moea_pte_clear(pt, PVO_VADDR(pvo), ptebit); } + mtx_unlock(&moea_table_mutex); } rv |= pvo->pvo_pte.pte_lo; pvo->pvo_pte.pte_lo &= ~ptebit; diff --git a/sys/powerpc/powerpc/mmu_oea.c b/sys/powerpc/powerpc/mmu_oea.c index 8cd8c2e..a50fed4 100644 --- a/sys/powerpc/powerpc/mmu_oea.c +++ b/sys/powerpc/powerpc/mmu_oea.c @@ -434,6 +434,7 @@ static __inline void moea_attr_clear(vm_page_t m, int ptebit) { + mtx_assert(&vm_page_queue_mtx, MA_OWNED); m->md.mdpg_attrs &= ~ptebit; } @@ -448,6 +449,7 @@ static __inline void moea_attr_save(vm_page_t m, int ptebit) { + mtx_assert(&vm_page_queue_mtx, MA_OWNED); m->md.mdpg_attrs |= ptebit; } @@ -471,6 +473,9 @@ moea_pte_match(struct pte *pt, u_int sr, vm_offset_t va, int which) static __inline void moea_pte_create(struct pte *pt, u_int sr, vm_offset_t va, u_int pte_lo) { + + mtx_assert(&moea_table_mutex, MA_OWNED); + /* * Construct a PTE. Default to IMB initially. Valid bit only gets * set when the real pte is set in memory. @@ -486,6 +491,7 @@ static __inline void moea_pte_synch(struct pte *pt, struct pte *pvo_pt) { + mtx_assert(&moea_table_mutex, MA_OWNED); pvo_pt->pte_lo |= pt->pte_lo & (PTE_REF | PTE_CHG); } @@ -493,6 +499,8 @@ static __inline void moea_pte_clear(struct pte *pt, vm_offset_t va, int ptebit) { + mtx_assert(&moea_table_mutex, MA_OWNED); + /* * As shown in Section 7.6.3.2.3 */ @@ -507,6 +515,7 @@ static __inline void moea_pte_set(struct pte *pt, struct pte *pvo_pt) { + mtx_assert(&moea_table_mutex, MA_OWNED); pvo_pt->pte_hi |= PTE_VALID; /* @@ -525,6 +534,7 @@ static __inline void moea_pte_unset(struct pte *pt, struct pte *pvo_pt, vm_offset_t va) { + mtx_assert(&moea_table_mutex, MA_OWNED); pvo_pt->pte_hi &= ~PTE_VALID; /* @@ -766,7 +776,8 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) * Initialize the lock that synchronizes access to the pteg and pvo * tables. */ - mtx_init(&moea_table_mutex, "pmap table", NULL, MTX_DEF); + mtx_init(&moea_table_mutex, "pmap table", NULL, MTX_DEF | + MTX_RECURSE); /* * Allocate the message buffer. @@ -1461,8 +1472,10 @@ moea_page_protect(mmu_t mmu, vm_page_t m, vm_prot_t prot) pt = moea_pvo_to_pte(pvo, -1); pvo->pvo_pte.pte_lo &= ~PTE_PP; pvo->pvo_pte.pte_lo |= PTE_BR; - if (pt != NULL) + if (pt != NULL) { moea_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); + } PMAP_UNLOCK(pmap); MOEA_PVO_CHECK(pvo); /* sanity check */ } @@ -1616,8 +1629,10 @@ moea_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva, /* * If the PVO is in the page table, update that pte as well. */ - if (pt != NULL) + if (pt != NULL) { moea_pte_change(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); + } } vm_page_unlock_queues(); PMAP_UNLOCK(pm); @@ -1795,6 +1810,7 @@ moea_rkva_alloc(mmu_t mmu) panic("moea_kva_alloc: moea_pvo_to_pte failed"); moea_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); PVO_PTEGIDX_CLR(pvo); moea_pte_overflow++; @@ -1818,6 +1834,7 @@ moea_pa_map(struct pvo_entry *pvo, vm_offset_t pa, struct pte *saved_pt, if (pt != NULL) { moea_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); PVO_PTEGIDX_CLR(pvo); moea_pte_overflow++; } @@ -1845,6 +1862,7 @@ moea_pa_unmap(struct pvo_entry *pvo, struct pte *saved_pt, int *depth_p) if (pt != NULL) { moea_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); PVO_PTEGIDX_CLR(pvo); moea_pte_overflow++; } @@ -2003,6 +2021,7 @@ moea_pvo_remove(struct pvo_entry *pvo, int pteidx) pt = moea_pvo_to_pte(pvo, pteidx); if (pt != NULL) { moea_pte_unset(pt, &pvo->pvo_pte, pvo->pvo_vaddr); + mtx_unlock(&moea_table_mutex); PVO_PTEGIDX_CLR(pvo); } else { moea_pte_overflow--; @@ -2104,6 +2123,7 @@ moea_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) } pt = &moea_pteg_table[pteidx >> 3].pt[pteidx & 7]; + mtx_lock(&moea_table_mutex); if ((pvo->pvo_pte.pte_hi & PTE_VALID) && !PVO_PTEGIDX_ISSET(pvo)) { panic("moea_pvo_to_pte: pvo %p has valid pte in pvo but no " @@ -2127,6 +2147,7 @@ moea_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) "pte %p in moea_pteg_table", pvo, pt); } + mtx_assert(&moea_table_mutex, MA_OWNED); return (pt); } @@ -2135,6 +2156,7 @@ moea_pvo_to_pte(const struct pvo_entry *pvo, int pteidx) "moea_pteg_table but valid in pvo", pvo, pt); } + mtx_unlock(&moea_table_mutex); return (NULL); } @@ -2267,6 +2289,8 @@ moea_pte_insert(u_int ptegidx, struct pte *pvo_pt) struct pte *pt; int i; + mtx_assert(&moea_table_mutex, MA_OWNED); + /* * First try primary hash. */ @@ -2337,6 +2361,7 @@ moea_query_bit(vm_page_t m, int ptebit) pt = moea_pvo_to_pte(pvo, -1); if (pt != NULL) { moea_pte_synch(pt, &pvo->pvo_pte); + mtx_unlock(&moea_table_mutex); if (pvo->pvo_pte.pte_lo & ptebit) { moea_attr_save(m, ptebit); MOEA_PVO_CHECK(pvo); /* sanity check */ @@ -2385,6 +2410,7 @@ moea_clear_bit(vm_page_t m, int ptebit, int *origbit) count++; moea_pte_clear(pt, PVO_VADDR(pvo), ptebit); } + mtx_unlock(&moea_table_mutex); } rv |= pvo->pvo_pte.pte_lo; pvo->pvo_pte.pte_lo &= ~ptebit; -- cgit v1.1