diff options
-rw-r--r-- | sys/alpha/alpha/pmap.c | 43 | ||||
-rw-r--r-- | sys/amd64/amd64/pmap.c | 14 | ||||
-rw-r--r-- | sys/arm/arm/pmap.c | 11 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 14 | ||||
-rw-r--r-- | sys/ia64/ia64/pmap.c | 4 | ||||
-rw-r--r-- | sys/powerpc/aim/mmu_oea.c | 8 | ||||
-rw-r--r-- | sys/powerpc/powerpc/mmu_oea.c | 8 | ||||
-rw-r--r-- | sys/powerpc/powerpc/pmap.c | 8 | ||||
-rw-r--r-- | sys/sparc64/sparc64/pmap.c | 8 | ||||
-rw-r--r-- | sys/vm/vm_fault.c | 10 | ||||
-rw-r--r-- | sys/vm/vm_map.c | 13 |
11 files changed, 107 insertions, 34 deletions
diff --git a/sys/alpha/alpha/pmap.c b/sys/alpha/alpha/pmap.c index 789a93a..41a5558 100644 --- a/sys/alpha/alpha/pmap.c +++ b/sys/alpha/alpha/pmap.c @@ -153,6 +153,7 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> +#include <sys/malloc.h> #include <sys/proc.h> #include <sys/msgbuf.h> #include <sys/vmmeter.h> @@ -334,7 +335,7 @@ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, static vm_page_t pmap_allocpte(pmap_t pmap, vm_offset_t va); -static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex); +static vm_page_t _pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags); static int _pmap_unwire_pte_hold(pmap_t pmap, vm_offset_t va, vm_page_t m); static int pmap_unuse_pt(pmap_t, vm_offset_t, vm_page_t); #ifdef SMP @@ -1038,24 +1039,28 @@ pmap_pinit(pmap) * mapped correctly. */ static vm_page_t -_pmap_allocpte(pmap, ptepindex) - pmap_t pmap; - unsigned ptepindex; +_pmap_allocpte(pmap_t pmap, unsigned ptepindex, int flags) { pt_entry_t* pte; vm_offset_t ptepa; vm_page_t m; + KASSERT((flags & (M_NOWAIT | M_WAITOK)) == M_NOWAIT || + (flags & (M_NOWAIT | M_WAITOK)) == M_WAITOK, + ("_pmap_allocpte: flags is neither M_NOWAIT nor M_WAITOK")); + /* * Find or fabricate a new pagetable page */ if ((m = vm_page_alloc(NULL, ptepindex, VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL) { - PMAP_UNLOCK(pmap); - vm_page_unlock_queues(); - VM_WAIT; - vm_page_lock_queues(); - PMAP_LOCK(pmap); + if (flags & M_WAITOK) { + PMAP_UNLOCK(pmap); + vm_page_unlock_queues(); + VM_WAIT; + vm_page_lock_queues(); + PMAP_LOCK(pmap); + } /* * Indicate the need to retry. While waiting, the page table @@ -1082,7 +1087,8 @@ _pmap_allocpte(pmap, ptepindex) pt_entry_t* l1pte = &pmap->pm_lev1[l1index]; pt_entry_t* l2map; if (!pmap_pte_v(l1pte)) { - if (_pmap_allocpte(pmap, NUSERLEV3MAPS + l1index) == NULL) { + if (_pmap_allocpte(pmap, NUSERLEV3MAPS + l1index, + flags) == NULL) { --m->wire_count; vm_page_free(m); return (NULL); @@ -1146,7 +1152,7 @@ retry: * Here if the pte page isn't mapped, or if it has been * deallocated. */ - m = _pmap_allocpte(pmap, ptepindex); + m = _pmap_allocpte(pmap, ptepindex, M_WAITOK); if (m == NULL) goto retry; } @@ -1842,9 +1848,20 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) } mpte->wire_count++; } else { - mpte = _pmap_allocpte(pmap, ptepindex); - if (mpte == NULL) + mpte = _pmap_allocpte(pmap, ptepindex, + M_NOWAIT); + if (mpte == NULL) { + PMAP_UNLOCK(pmap); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); + VM_WAIT; + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + PMAP_LOCK(pmap); goto retry; + } } } } else { diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index a19c721..e089d62 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -2053,9 +2053,19 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) mpte->wire_count++; } else { mpte = _pmap_allocpte(pmap, ptepindex, - M_WAITOK); - if (mpte == NULL) + M_NOWAIT); + if (mpte == NULL) { + PMAP_UNLOCK(pmap); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); + VM_WAIT; + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + PMAP_LOCK(pmap); goto retry; + } } } } else { diff --git a/sys/arm/arm/pmap.c b/sys/arm/arm/pmap.c index f9dca09..9159787 100644 --- a/sys/arm/arm/pmap.c +++ b/sys/arm/arm/pmap.c @@ -3417,8 +3417,19 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot, vm_page_t pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); + mtx_lock(&Giant); pmap_enter(pmap, va, m, VM_PROT_READ|VM_PROT_EXECUTE, FALSE); pmap_dcache_wbinv_all(pmap); /* XXX: shouldn't be needed */ + mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index d16bf02..67ac6f6 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -2094,9 +2094,19 @@ retry: mpte->wire_count++; } else { mpte = _pmap_allocpte(pmap, ptepindex, - M_WAITOK); - if (mpte == NULL) + M_NOWAIT); + if (mpte == NULL) { + PMAP_UNLOCK(pmap); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); + VM_WAIT; + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + PMAP_LOCK(pmap); goto retry; + } } } } else { diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c index da1bcf1..a2feed7 100644 --- a/sys/ia64/ia64/pmap.c +++ b/sys/ia64/ia64/pmap.c @@ -1631,9 +1631,13 @@ pmap_enter_quick(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_page_t mpte) while ((pte = pmap_find_pte(va)) == NULL) { pmap_install(oldpmap); PMAP_UNLOCK(pmap); + vm_page_busy(m); vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); VM_WAIT; + VM_OBJECT_LOCK(m->object); vm_page_lock_queues(); + vm_page_wakeup(m); PMAP_LOCK(pmap); oldpmap = pmap_install(pmap); } diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index 5398d61..136e412 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -1048,9 +1048,17 @@ vm_page_t pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); mtx_lock(&Giant); pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE); mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/powerpc/powerpc/mmu_oea.c b/sys/powerpc/powerpc/mmu_oea.c index 5398d61..136e412 100644 --- a/sys/powerpc/powerpc/mmu_oea.c +++ b/sys/powerpc/powerpc/mmu_oea.c @@ -1048,9 +1048,17 @@ vm_page_t pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); mtx_lock(&Giant); pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE); mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/powerpc/powerpc/pmap.c b/sys/powerpc/powerpc/pmap.c index 5398d61..136e412 100644 --- a/sys/powerpc/powerpc/pmap.c +++ b/sys/powerpc/powerpc/pmap.c @@ -1048,9 +1048,17 @@ vm_page_t pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); mtx_lock(&Giant); pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE); mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/sparc64/sparc64/pmap.c b/sys/sparc64/sparc64/pmap.c index 18cd0f9..fa964ab 100644 --- a/sys/sparc64/sparc64/pmap.c +++ b/sys/sparc64/sparc64/pmap.c @@ -1401,9 +1401,17 @@ vm_page_t pmap_enter_quick(pmap_t pm, vm_offset_t va, vm_page_t m, vm_page_t mpte) { + vm_page_lock_queues(); + vm_page_busy(m); + vm_page_unlock_queues(); + VM_OBJECT_UNLOCK(m->object); mtx_lock(&Giant); pmap_enter(pm, va, m, VM_PROT_READ | VM_PROT_EXECUTE, FALSE); mtx_unlock(&Giant); + VM_OBJECT_LOCK(m->object); + vm_page_lock_queues(); + vm_page_wakeup(m); + vm_page_unlock_queues(); return (NULL); } diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 4bd82f4..6fac17c 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -973,23 +973,17 @@ vm_fault_prefault(pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry) VM_OBJECT_UNLOCK(lobject); break; } - vm_page_lock_queues(); if (((m->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL) && (m->busy == 0) && (m->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { if ((m->queue - m->pc) == PQ_CACHE) { + vm_page_lock_queues(); vm_page_deactivate(m); + vm_page_unlock_queues(); } - vm_page_busy(m); - vm_page_unlock_queues(); - VM_OBJECT_UNLOCK(lobject); mpte = pmap_enter_quick(pmap, addr, m, mpte); - VM_OBJECT_LOCK(lobject); - vm_page_lock_queues(); - vm_page_wakeup(m); } - vm_page_unlock_queues(); VM_OBJECT_UNLOCK(lobject); } } diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index 3788a4f..46ed849 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -1417,22 +1417,17 @@ vm_map_pmap_enter(vm_map_t map, vm_offset_t addr, vm_prot_t prot, cnt.v_free_count < cnt.v_free_reserved) { break; } - vm_page_lock_queues(); if ((p->valid & VM_PAGE_BITS_ALL) == VM_PAGE_BITS_ALL && (p->busy == 0) && (p->flags & (PG_BUSY | PG_FICTITIOUS)) == 0) { - if ((p->queue - p->pc) == PQ_CACHE) + if ((p->queue - p->pc) == PQ_CACHE) { + vm_page_lock_queues(); vm_page_deactivate(p); - vm_page_busy(p); - vm_page_unlock_queues(); - VM_OBJECT_UNLOCK(object); + vm_page_unlock_queues(); + } mpte = pmap_enter_quick(map->pmap, addr + ptoa(tmpidx), p, mpte); - VM_OBJECT_LOCK(object); - vm_page_lock_queues(); - vm_page_wakeup(p); } - vm_page_unlock_queues(); } unlock_return: VM_OBJECT_UNLOCK(object); |