diff options
-rw-r--r-- | sys/amd64/amd64/pmap.c | 906 | ||||
-rw-r--r-- | sys/amd64/include/pmap.h | 25 | ||||
-rw-r--r-- | sys/i386/i386/pmap.c | 906 | ||||
-rw-r--r-- | sys/i386/include/pmap.h | 25 | ||||
-rw-r--r-- | sys/kern/kern_exec.c | 4 | ||||
-rw-r--r-- | sys/kern/kern_exit.c | 5 | ||||
-rw-r--r-- | sys/vm/pmap.h | 10 | ||||
-rw-r--r-- | sys/vm/swap_pager.c | 29 | ||||
-rw-r--r-- | sys/vm/vm_fault.c | 19 | ||||
-rw-r--r-- | sys/vm/vm_map.c | 145 | ||||
-rw-r--r-- | sys/vm/vm_map.h | 3 | ||||
-rw-r--r-- | sys/vm/vm_mmap.c | 229 | ||||
-rw-r--r-- | sys/vm/vm_object.c | 7 | ||||
-rw-r--r-- | sys/vm/vm_page.c | 49 | ||||
-rw-r--r-- | sys/vm/vm_page.h | 10 | ||||
-rw-r--r-- | sys/vm/vm_pageout.c | 60 | ||||
-rw-r--r-- | sys/vm/vnode_pager.c | 8 |
17 files changed, 1250 insertions, 1190 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c index 97be59c..4998f78 100644 --- a/sys/amd64/amd64/pmap.c +++ b/sys/amd64/amd64/pmap.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.111 1996/07/28 20:31:27 dyson Exp $ + * $Id: pmap.c,v 1.113 1996/07/29 14:22:46 dyson Exp $ */ /* @@ -99,12 +99,6 @@ #define PMAP_DIAGNOSTIC #endif -#if !defined(SMALL_KERNEL) -#define PMAP_INLINE __inline -#else -#define PMAP_INLINE -#endif - static void init_pv_entries __P((int)); /* @@ -152,7 +146,7 @@ extern int cpu_class; * Data for the pv entry allocation mechanism */ static int pv_freelistcnt; -TAILQ_HEAD (,pv_entry) pv_freelist; +static pv_entry_t pv_freelist; static vm_offset_t pvva; static int npvvapg; @@ -161,6 +155,7 @@ static int npvvapg; */ pt_entry_t *CMAP1; static pt_entry_t *CMAP2, *ptmmap; +static pv_entry_t *pv_table; caddr_t CADDR1, ptvmmap; static caddr_t CADDR2; static pt_entry_t *msgbufmap; @@ -170,32 +165,32 @@ pt_entry_t *PMAP1; unsigned *PADDR1; static void free_pv_entry __P((pv_entry_t pv)); -static unsigned * get_ptbase __P((pmap_t pmap)); +static __inline unsigned * get_ptbase __P((pmap_t pmap)); static pv_entry_t get_pv_entry __P((void)); static void i386_protection_init __P((void)); static void pmap_alloc_pv_entry __P((void)); +static void pmap_changebit __P((vm_offset_t pa, int bit, boolean_t setem)); static int pmap_is_managed __P((vm_offset_t pa)); -static int pmap_remove_all __P((vm_offset_t pa)); -static vm_page_t pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, - vm_offset_t pa, vm_page_t mpte)); +static void pmap_remove_all __P((vm_offset_t pa)); +static void pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, + vm_offset_t pa)); static int pmap_remove_pte __P((struct pmap *pmap, unsigned *ptq, vm_offset_t sva)); static void pmap_remove_page __P((struct pmap *pmap, vm_offset_t va)); -static int pmap_remove_entry __P((struct pmap *pmap, pv_table_t *pv, +static __inline int pmap_remove_entry __P((struct pmap *pmap, pv_entry_t *pv, vm_offset_t va)); -static void pmap_insert_entry __P((pmap_t pmap, vm_offset_t va, +static boolean_t pmap_testbit __P((vm_offset_t pa, int bit)); +static __inline void pmap_insert_entry __P((pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_offset_t pa)); -static vm_page_t pmap_allocpte __P((pmap_t pmap, vm_offset_t va)); +static __inline vm_page_t pmap_allocpte __P((pmap_t pmap, vm_offset_t va)); -static int pmap_release_free_page __P((pmap_t pmap, vm_page_t p)); +static __inline int pmap_release_free_page __P((pmap_t pmap, vm_page_t p)); static vm_page_t _pmap_allocpte __P((pmap_t pmap, int ptepindex)); -unsigned * __pure pmap_pte_quick __P((pmap_t pmap, vm_offset_t va)); -int pmap_tcbit __P((vm_offset_t pa, int bit)); -static vm_page_t pmap_page_alloc __P((vm_object_t object, vm_pindex_t pindex)); -#define PDSTACKMAX 6 +#define VATRACK 4 +#define PDSTACKMAX 16 static vm_offset_t pdstack[PDSTACKMAX]; static int pdstackptr; @@ -245,7 +240,6 @@ pmap_bootstrap(firstaddr, loadaddr) kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + IdlePTD); kernel_pmap->pm_count = 1; - TAILQ_INIT(&kernel_pmap->pm_pvlist.pv_list); nkpt = NKPT; /* @@ -312,15 +306,10 @@ pmap_init(phys_start, phys_end) * Allocate memory for random pmap data structures. Includes the * pv_head_table. */ - s = (vm_size_t) (sizeof(pv_table_t) * npg); + s = (vm_size_t) (sizeof(struct pv_entry *) * npg); s = round_page(s); addr = (vm_offset_t) kmem_alloc(kernel_map, s); - pv_table = (pv_table_t *) addr; - for(i=0;i<npg;i++) { - pv_table[i].pv_list_count = 0; - TAILQ_INIT(&pv_table[i].pv_list); - } - TAILQ_INIT(&pv_freelist); + pv_table = (pv_entry_t *) addr; /* * init the pv free list @@ -383,7 +372,7 @@ pmap_nw_modified(pt_entry_t ptea) { * this routine defines the region(s) of memory that should * not be tested for the modified bit. */ -static PMAP_INLINE int +static __inline int pmap_track_modified( vm_offset_t va) { if ((va < clean_sva) || (va >= clean_eva)) return 1; @@ -395,7 +384,7 @@ pmap_track_modified( vm_offset_t va) { * The below are finer grained pmap_update routines. These eliminate * the gratuitious tlb flushes on non-i386 architectures. */ -static PMAP_INLINE void +static __inline void pmap_update_1pg( vm_offset_t va) { #if defined(I386_CPU) if (cpu_class == CPUCLASS_386) @@ -405,7 +394,7 @@ pmap_update_1pg( vm_offset_t va) { __asm __volatile(".byte 0xf,0x1,0x38": :"a" (va)); } -static PMAP_INLINE void +static __inline void pmap_update_2pg( vm_offset_t va1, vm_offset_t va2) { #if defined(I386_CPU) if (cpu_class == CPUCLASS_386) { @@ -443,7 +432,7 @@ get_ptbase(pmap) * with the given map/virtual_address pair. */ -unsigned * __pure +__inline unsigned * __pure pmap_pte(pmap, va) register pmap_t pmap; vm_offset_t va; @@ -459,27 +448,25 @@ pmap_pte(pmap, va) * the pv lists. This eliminates many coarse-grained * pmap_update calls. */ -unsigned * __pure +__inline unsigned * __pure pmap_pte_quick(pmap, va) register pmap_t pmap; vm_offset_t va; { - unsigned pde, newpf; + unsigned pde; if (pde = (unsigned) pmap->pm_pdir[va >> PDRSHIFT]) { unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME; /* are we current address space or kernel? */ if (pmap == kernel_pmap || frame == (((unsigned) PTDpde) & PG_FRAME)) { return (unsigned *) PTmap + i386_btop(va); } - newpf = pde & PG_FRAME; - if ( ((* (unsigned *) PMAP1) & PG_FRAME) != newpf) { - * (unsigned *) PMAP1 = newpf | PG_RW | PG_V; - pmap_update_1pg((vm_offset_t) PADDR1); - } + * (int *) PMAP1 = (pde & PG_FRAME) | PG_V | PG_RW; + pmap_update_1pg((vm_offset_t) PADDR1); return PADDR1 + ((unsigned) i386_btop(va) & (NPTEPG - 1)); } return (0); } + /* * Routine: pmap_extract @@ -504,7 +491,7 @@ pmap_extract(pmap, va) /* * determine if a page is managed (memory vs. device) */ -static PMAP_INLINE __pure int +static __inline __pure int pmap_is_managed(pa) vm_offset_t pa; { @@ -577,7 +564,7 @@ pmap_qremove(va, count) * note that in order for the mapping to take effect -- you * should do a pmap_update after doing the pmap_kenter... */ -PMAP_INLINE void +__inline void pmap_kenter(va, pa) vm_offset_t va; register vm_offset_t pa; @@ -596,7 +583,7 @@ pmap_kenter(va, pa) /* * remove a page from the kernel pagetables */ -PMAP_INLINE void +__inline void pmap_kremove(va) vm_offset_t va; { @@ -607,17 +594,80 @@ pmap_kremove(va) pmap_update_1pg(va); } -static vm_page_t -pmap_page_alloc(object, pindex) - vm_object_t object; - vm_pindex_t pindex; + +/*************************************************** + * Page table page management routines..... + ***************************************************/ + +/* + * This routine unholds page table pages, and if the hold count + * drops to zero, then it decrements the wire count. + */ +static __inline int +pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { + vm_page_unhold(m); + if (m->hold_count == 0) { + vm_offset_t pteva; + /* + * unmap the page table page + */ + pmap->pm_pdir[m->pindex] = 0; + --pmap->pm_stats.resident_count; + /* + * Do a pmap_update to make the invalidated mapping + * take effect immediately. + */ + pteva = UPT_MIN_ADDRESS + i386_ptob(m->pindex); + pmap_update_1pg(pteva); + /* + * If the page is finally unwired, simply free it. + */ + --m->wire_count; + if (m->wire_count == 0) { + vm_page_free_zero(m); + --cnt.v_wire_count; + } + return 1; + } + return 0; +} + +/* + * After removing a page table entry, this routine is used to + * conditionally free the page, and manage the hold/wire counts. + */ +int +pmap_unuse_pt(pmap, va, mpte) + pmap_t pmap; + vm_offset_t va; + vm_page_t mpte; { - vm_page_t m; - m = vm_page_alloc(object, pindex, VM_ALLOC_ZERO); - if (m == NULL) { - VM_WAIT; + if (va >= UPT_MIN_ADDRESS) + return 0; + + if (mpte == NULL) { + vm_offset_t ptepa; + ptepa = ((vm_offset_t) *pmap_pde(pmap, va)); +#if defined(PMAP_DIAGNOSTIC) + if (!ptepa) + panic("pmap_unuse_pt: pagetable page missing, va: 0x%x", va); +#endif + if (!ptepa) + return 0; + mpte = PHYS_TO_VM_PAGE(ptepa); } - return m; + +#if defined(PMAP_DIAGNOSTIC) + if (mpte->pindex != (va >> PDRSHIFT)) + panic("pmap_unuse_pt: pindex(0x%x) != va(0x%x)", + mpte->pindex, (va >> PDRSHIFT)); + + if (mpte->hold_count == 0) { + panic("pmap_unuse_pt: hold count < 0, va: 0x%x", va); + } +#endif + + return pmap_unwire_pte_hold(pmap, mpte); } /* @@ -651,12 +701,12 @@ pmap_pinit(pmap) * allocate the page directory page */ retry: - ptdpg = pmap_page_alloc( pmap->pm_pteobj, PTDPTDI); - if (ptdpg == NULL) + ptdpg = vm_page_alloc( pmap->pm_pteobj, PTDPTDI, VM_ALLOC_ZERO); + if (ptdpg == NULL) { + VM_WAIT; goto retry; - - ptdpg->wire_count = 1; - ++cnt.v_wire_count; + } + vm_page_wire(ptdpg); ptdpg->flags &= ~(PG_MAPPED|PG_BUSY); /* not mapped normally */ ptdpg->valid = VM_PAGE_BITS_ALL; @@ -672,7 +722,6 @@ retry: VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW; pmap->pm_count = 1; - TAILQ_INIT(&pmap->pm_pvlist.pv_list); } static int @@ -702,7 +751,31 @@ pmap_release_free_page(pmap, p) --pmap->pm_stats.resident_count; if (p->hold_count) { + int *kvap; + int i; +#if defined(PMAP_DIAGNOSTIC) panic("pmap_release: freeing held page table page"); +#else + printf("pmap_release: freeing held page table page:\n"); +#endif + kvap = (int *)vm_pager_map_page(p); + for(i=0;i<NPTEPG;i++) { + if (kvap[i]) { + printf("pte: 0x%x, index: %d\n", kvap[i],i); + } + } + vm_pager_unmap_page((vm_offset_t)kvap); + + /* + * HACK ALERT!!! + * If this failure happens, we must clear the page, because + * there is likely a mapping still valid. This condition + * is an error, but at least this zero operation will mitigate + * some Sig-11's or crashes, because this page is thought + * to be zero. This is a robustness fix, and not meant to + * be a long term work-around. + */ + pmap_zero_page(VM_PAGE_TO_PHYS(p)); } /* * Page directory pages need to have the kernel @@ -728,7 +801,7 @@ _pmap_allocpte(pmap, ptepindex) pmap_t pmap; int ptepindex; { - vm_offset_t ptepa; + vm_offset_t pteva, ptepa; vm_page_t m; /* @@ -737,9 +810,11 @@ _pmap_allocpte(pmap, ptepindex) retry: m = vm_page_lookup(pmap->pm_pteobj, ptepindex); if (m == NULL) { - m = pmap_page_alloc(pmap->pm_pteobj, ptepindex); - if (m == NULL) + m = vm_page_alloc(pmap->pm_pteobj, ptepindex, VM_ALLOC_ZERO); + if (m == NULL) { + VM_WAIT; goto retry; + } if ((m->flags & PG_ZERO) == 0) pmap_zero_page(VM_PAGE_TO_PHYS(m)); m->flags &= ~(PG_ZERO|PG_BUSY); @@ -752,16 +827,22 @@ retry: } } + /* + * mark the object writeable + */ + pmap->pm_pteobj->flags |= OBJ_WRITEABLE; + if (m->queue != PQ_NONE) { int s = splvm(); - vm_page_unqueue(m,1); + vm_page_unqueue(m); splx(s); } - if (m->wire_count == 0) - ++cnt.v_wire_count; - ++m->wire_count; - + if (m->hold_count == 0) { + if (m->wire_count == 0) + ++cnt.v_wire_count; + ++m->wire_count; + } /* * Increment the hold count for the page table page * (denoting a new mapping.) @@ -778,12 +859,14 @@ retry: ptepa = VM_PAGE_TO_PHYS(m); pmap->pm_pdir[ptepindex] = (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V); + pteva = UPT_MIN_ADDRESS + i386_ptob(ptepindex); + pmap_update_1pg(pteva); m->flags |= PG_MAPPED; return m; } -PMAP_INLINE static vm_page_t +static __inline vm_page_t pmap_allocpte(pmap, va) pmap_t pmap; vm_offset_t va; @@ -807,7 +890,7 @@ pmap_allocpte(pmap, va) * hold count, and activate it. */ if (ptepa) { - m = vm_page_lookup( pmap->pm_pteobj, ptepindex); + m = PHYS_TO_VM_PAGE(ptepa); ++m->hold_count; return m; } @@ -952,12 +1035,13 @@ pmap_reference(pmap) /* * free the pv_entry back to the free list */ -static PMAP_INLINE void +static __inline void free_pv_entry(pv) pv_entry_t pv; { ++pv_freelistcnt; - TAILQ_INSERT_HEAD(&pv_freelist, pv, pv_list); + pv->pv_next = pv_freelist; + pv_freelist = pv; } /* @@ -966,7 +1050,7 @@ free_pv_entry(pv) * the memory allocation is performed bypassing the malloc code * because of the possibility of allocations at interrupt time. */ -static PMAP_INLINE pv_entry_t +static __inline pv_entry_t get_pv_entry() { pv_entry_t tmp; @@ -974,16 +1058,15 @@ get_pv_entry() /* * get more pv_entry pages if needed */ - if (pv_freelistcnt < PV_FREELIST_MIN) { + if (pv_freelistcnt < PV_FREELIST_MIN || pv_freelist == 0) { pmap_alloc_pv_entry(); } - /* * get a pv_entry off of the free list */ --pv_freelistcnt; - tmp = TAILQ_FIRST(&pv_freelist); - TAILQ_REMOVE(&pv_freelist, tmp, pv_list); + tmp = pv_freelist; + pv_freelist = tmp->pv_next; return tmp; } @@ -1040,7 +1123,7 @@ pmap_alloc_pv_entry() } } } - if (TAILQ_FIRST(&pv_freelist) == NULL) + if (!pv_freelist) panic("get_pv_entry: cannot get a pv_entry_t"); } @@ -1067,107 +1150,62 @@ init_pv_entries(npg) } /* - * This routine unholds page table pages, and if the hold count - * drops to zero, then it decrements the wire count. + * If it is the first entry on the list, it is actually + * in the header and we must copy the following entry up + * to the header. Otherwise we must search the list for + * the entry. In either case we free the now unused entry. */ -static int -pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { - vm_page_unhold(m); - if (m->hold_count == 0) { - vm_offset_t pteva; - /* - * unmap the page table page - */ - pmap->pm_pdir[m->pindex] = 0; - --pmap->pm_stats.resident_count; - if ((((unsigned)pmap->pm_pdir[PTDPTDI]) & PG_FRAME) == - (((unsigned) PTDpde) & PG_FRAME)) { - /* - * Do a pmap_update to make the invalidated mapping - * take effect immediately. - */ - pteva = UPT_MIN_ADDRESS + i386_ptob(m->pindex); - pmap_update_1pg(pteva); - } - /* - * If the page is finally unwired, simply free it. - */ - --m->wire_count; - if (m->wire_count == 0) { - vm_page_free_zero(m); - --cnt.v_wire_count; +static __inline int +pmap_remove_entry(pmap, ppv, va) + struct pmap *pmap; + pv_entry_t *ppv; + vm_offset_t va; +{ + pv_entry_t npv; + int s; + + s = splvm(); + for (npv = *ppv; npv; (ppv = &npv->pv_next, npv = *ppv)) { + if (pmap == npv->pv_pmap && va == npv->pv_va) { + int rtval = pmap_unuse_pt(pmap, va, npv->pv_ptem); + *ppv = npv->pv_next; + free_pv_entry(npv); + splx(s); + return rtval; } - return 1; } + splx(s); return 0; } /* - * After removing a page table entry, this routine is used to - * conditionally free the page, and manage the hold/wire counts. + * Create a pv entry for page at pa for + * (pmap, va). */ -PMAP_INLINE int -pmap_unuse_pt(pmap, va, mpte) +static __inline void +pmap_insert_entry(pmap, va, mpte, pa) pmap_t pmap; vm_offset_t va; vm_page_t mpte; + vm_offset_t pa; { - int ptepindex; - if (va >= UPT_MIN_ADDRESS) - return 0; - if (mpte == NULL) { - ptepindex = (va >> PDRSHIFT); - mpte = vm_page_lookup( pmap->pm_pteobj, ptepindex); - } - - return pmap_unwire_pte_hold(pmap, mpte); -} - -/* - * If it is the first entry on the list, it is actually - * in the header and we must copy the following entry up - * to the header. Otherwise we must search the list for - * the entry. In either case we free the now unused entry. - */ -static int -pmap_remove_entry(pmap, ppv, va) - struct pmap *pmap; - pv_table_t *ppv; - vm_offset_t va; -{ - pv_entry_t pv; - int rtval; int s; + pv_entry_t *ppv, pv; s = splvm(); - if (ppv->pv_list_count < pmap->pm_stats.resident_count) { - for (pv = TAILQ_FIRST(&ppv->pv_list); - pv; - pv = TAILQ_NEXT(pv, pv_list)) { - if (pmap == pv->pv_pmap && va == pv->pv_va) - break; - } - } else { - for (pv = TAILQ_FIRST(&pmap->pm_pvlist.pv_list); - pv; - pv = TAILQ_NEXT(pv, pv_plist)) { - if (va == pv->pv_va) - break; - } - } + pv = get_pv_entry(); + pv->pv_va = va; + pv->pv_pmap = pmap; + pv->pv_ptem = mpte; - rtval = 0; - if (pv) { - rtval = pmap_unuse_pt(pmap, va, pv->pv_ptem); - TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); - --ppv->pv_list_count; - TAILQ_REMOVE(&pmap->pm_pvlist.pv_list, pv, pv_plist); - free_pv_entry(pv); - } - + ppv = pa_to_pvh(pa); + if (*ppv) + pv->pv_next = *ppv; + else + pv->pv_next = NULL; + *ppv = pv; splx(s); - return rtval; } /* @@ -1180,6 +1218,7 @@ pmap_remove_pte(pmap, ptq, va) vm_offset_t va; { unsigned oldpte; + pv_entry_t *ppv; oldpte = *ptq; *ptq = 0; @@ -1196,7 +1235,8 @@ pmap_remove_pte(pmap, ptq, va) if (pmap_track_modified(va)) PHYS_TO_VM_PAGE(oldpte)->dirty = VM_PAGE_BITS_ALL; } - return pmap_remove_entry(pmap, pa_to_pvh(oldpte), va); + ppv = pa_to_pvh(oldpte); + return pmap_remove_entry(pmap, ppv, va); } else { return pmap_unuse_pt(pmap, va, NULL); } @@ -1250,6 +1290,7 @@ pmap_remove(pmap, sva, eva) vm_offset_t sindex, eindex; vm_page_t mpte; int anyvalid; + vm_offset_t vachanged[VATRACK]; if (pmap == NULL) return; @@ -1274,7 +1315,6 @@ pmap_remove(pmap, sva, eva) sindex = i386_btop(sva); eindex = i386_btop(eva); - mpte = NULL; for (; sindex < eindex; sindex = pdnxt) { @@ -1291,6 +1331,19 @@ pmap_remove(pmap, sva, eva) if (ptpaddr == 0) continue; + if (sindex < i386_btop(UPT_MIN_ADDRESS)) { + /* + * get the vm_page_t for the page table page + */ + mpte = PHYS_TO_VM_PAGE(ptpaddr); + + /* + * if the pte isn't wired, just skip it. + */ + if (mpte->wire_count == 0) + continue; + } + /* * Limit our scan to either the end of the va represented * by the current page table page, or to the end of the @@ -1307,6 +1360,8 @@ pmap_remove(pmap, sva, eva) } va = i386_ptob(sindex); + if (anyvalid < VATRACK) + vachanged[anyvalid] = va; anyvalid++; if (pmap_remove_pte(pmap, ptbase + sindex, va)) @@ -1315,8 +1370,15 @@ pmap_remove(pmap, sva, eva) } if (anyvalid) { - pmap_update(); + if (anyvalid <= VATRACK) { + int i; + for(i=0;i<anyvalid;i++) + pmap_update_1pg(vachanged[i]); + } else { + pmap_update(); + } } + } /* @@ -1331,18 +1393,15 @@ pmap_remove(pmap, sva, eva) * inefficient because they iteratively called * pmap_remove (slow...) */ -static int +static void pmap_remove_all(pa) vm_offset_t pa; { - register pv_entry_t pv, npv; - pv_table_t *ppv; + register pv_entry_t pv, *ppv, npv; register unsigned *pte, tpte; vm_page_t m; - int nmodify; int s; - nmodify = 0; #if defined(PMAP_DIAGNOSTIC) /* * XXX this makes pmap_page_protect(NONE) illegal for non-managed @@ -1356,9 +1415,7 @@ pmap_remove_all(pa) s = splvm(); m = NULL; ppv = pa_to_pvh(pa); - for (pv = TAILQ_FIRST(&ppv->pv_list); - pv; - pv = npv) { + for (pv = *ppv; pv; pv=pv->pv_next) { pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (tpte = *pte) { pv->pv_pmap->pm_stats.resident_count--; @@ -1368,27 +1425,28 @@ pmap_remove_all(pa) /* * Update the vm_page_t clean and reference bits. */ - if ((tpte & (PG_M|PG_MANAGED)) == (PG_M|PG_MANAGED)) { + if (tpte & PG_M) { #if defined(PMAP_DIAGNOSTIC) if (pmap_nw_modified((pt_entry_t) tpte)) { printf("pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n", pv->pv_va, tpte); } #endif - if (pmap_track_modified(pv->pv_va)) - nmodify += 1; + if (pmap_track_modified(pv->pv_va)) { + if (m == NULL) + m = PHYS_TO_VM_PAGE(pa); + m->dirty = VM_PAGE_BITS_ALL; + } } } - TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist.pv_list, pv, pv_plist); + } - npv = TAILQ_NEXT(pv, pv_list); - TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); - --ppv->pv_list_count; + for (pv = *ppv; pv; pv = npv) { + npv = pv->pv_next; pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); free_pv_entry(pv); } - + *ppv = NULL; splx(s); - return nmodify; } /* @@ -1426,7 +1484,6 @@ pmap_protect(pmap, sva, eva, prot) sindex = i386_btop(sva); eindex = i386_btop(eva); - mpte = NULL; for (; sindex < eindex; sindex = pdnxt) { pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); @@ -1439,6 +1496,18 @@ pmap_protect(pmap, sva, eva, prot) if (ptpaddr == 0) continue; + /* + * Skip page ranges, where the page table page isn't wired. + * If the page table page is not wired, there are no page mappings + * there. + */ + if (sindex < i386_btop(UPT_MIN_ADDRESS)) { + mpte = PHYS_TO_VM_PAGE(ptpaddr); + + if (mpte->wire_count == 0) + continue; + } + if (pdnxt > eindex) { pdnxt = eindex; } @@ -1447,8 +1516,8 @@ pmap_protect(pmap, sva, eva, prot) unsigned pbits = ptbase[sindex]; - if ((pbits & (PG_RW|PG_V)) == (PG_RW|PG_V)) { - if ((pbits & (PG_M|PG_MANAGED)) == (PG_M|PG_MANAGED)) { + if (pbits & PG_RW) { + if (pbits & PG_M) { vm_offset_t sva = i386_ptob(sindex); if (pmap_track_modified(sva)) { vm_page_t m = PHYS_TO_VM_PAGE(pbits); @@ -1465,37 +1534,6 @@ pmap_protect(pmap, sva, eva, prot) } /* - * Create a pv entry for page at pa for - * (pmap, va). - */ -static void -pmap_insert_entry(pmap, va, mpte, pa) - pmap_t pmap; - vm_offset_t va; - vm_page_t mpte; - vm_offset_t pa; -{ - - int s; - pv_entry_t pv; - pv_table_t *ppv; - - s = splvm(); - pv = get_pv_entry(); - pv->pv_va = va; - pv->pv_pmap = pmap; - pv->pv_ptem = mpte; - - TAILQ_INSERT_TAIL(&pmap->pm_pvlist.pv_list, pv, pv_plist); - - ppv = pa_to_pvh(pa); - TAILQ_INSERT_TAIL(&ppv->pv_list, pv, pv_list); - ++ppv->pv_list_count; - - splx(s); -} - -/* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. @@ -1539,7 +1577,7 @@ pmap_enter(pmap, va, pa, prot, wired) if (va < UPT_MIN_ADDRESS) mpte = pmap_allocpte(pmap, va); - pte = pmap_pte(pmap, va); + pte = pmap_pte_quick(pmap, va); /* * Page Directory table entry not valid, we need a new PT page */ @@ -1555,7 +1593,7 @@ pmap_enter(pmap, va, pa, prot, wired) /* * Mapping has not changed, must be protection or wiring change. */ - if (origpte && (opa == pa)) { + if (opa == pa) { /* * Wiring change, just update stats. We don't worry about * wiring PT pages as they remain resident as long as there @@ -1578,10 +1616,12 @@ pmap_enter(pmap, va, pa, prot, wired) * so we go ahead and sense modify status. */ if (origpte & PG_MANAGED) { - if ((origpte & PG_M) && pmap_track_modified(va)) { - vm_page_t m; - m = PHYS_TO_VM_PAGE(pa); - m->dirty = VM_PAGE_BITS_ALL; + vm_page_t m; + if (origpte & PG_M) { + if (pmap_track_modified(va)) { + m = PHYS_TO_VM_PAGE(pa); + m->dirty = VM_PAGE_BITS_ALL; + } } pa |= PG_MANAGED; } @@ -1595,7 +1635,7 @@ pmap_enter(pmap, va, pa, prot, wired) * Mapping has changed, invalidate old range and fall through to * handle validating new mapping. */ - if (origpte) { + if (opa) { int err; err = pmap_remove_pte(pmap, pte, va); if (err) @@ -1652,49 +1692,22 @@ validate: * but is *MUCH* faster than pmap_enter... */ -static vm_page_t -pmap_enter_quick(pmap, va, pa, mpte) +static void +pmap_enter_quick(pmap, va, pa) register pmap_t pmap; vm_offset_t va; register vm_offset_t pa; - vm_page_t mpte; { register unsigned *pte; + vm_page_t mpte; + mpte = NULL; /* * In the case that a page table page is not * resident, we are creating it here. */ - if (va < UPT_MIN_ADDRESS) { - int ptepindex; - vm_offset_t ptepa; - - /* - * Calculate pagetable page index - */ - ptepindex = va >> PDRSHIFT; - if (mpte && (mpte->pindex == ptepindex)) { - ++mpte->hold_count; - } else { - /* - * Get the page directory entry - */ - ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex]; - - /* - * If the page table page is mapped, we just increment - * the hold count, and activate it. - */ - if (ptepa) { - mpte = vm_page_lookup( pmap->pm_pteobj, ptepindex); - ++mpte->hold_count; - } else { - mpte = _pmap_allocpte(pmap, ptepindex); - } - } - } else { - mpte = NULL; - } + if (va < UPT_MIN_ADDRESS) + mpte = pmap_allocpte(pmap, va); /* * This call to vtopte makes the assumption that we are @@ -1706,7 +1719,7 @@ pmap_enter_quick(pmap, va, pa, mpte) if (*pte) { if (mpte) pmap_unwire_pte_hold(pmap, mpte); - return NULL; + return; } /* @@ -1726,7 +1739,7 @@ pmap_enter_quick(pmap, va, pa, mpte) */ *pte = pa | PG_V | PG_U | PG_MANAGED; - return mpte; + return; } #define MAX_INIT_PT (96) @@ -1746,7 +1759,7 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) { vm_offset_t tmpidx; int psize; - vm_page_t p, mpte; + vm_page_t p; int objpgs; psize = i386_btop(size); @@ -1760,7 +1773,6 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) if (psize + pindex > object->size) psize = object->size - pindex; - mpte = NULL; /* * if we are processing a major portion of the object, then scan the * entire thing. @@ -1786,9 +1798,9 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) if (p->queue == PQ_CACHE) vm_page_deactivate(p); p->flags |= PG_BUSY; - mpte = pmap_enter_quick(pmap, + pmap_enter_quick(pmap, addr + i386_ptob(tmpidx), - VM_PAGE_TO_PHYS(p), mpte); + VM_PAGE_TO_PHYS(p)); p->flags |= PG_MAPPED; PAGE_WAKEUP(p); } @@ -1807,9 +1819,9 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) if (p->queue == PQ_CACHE) vm_page_deactivate(p); p->flags |= PG_BUSY; - mpte = pmap_enter_quick(pmap, + pmap_enter_quick(pmap, addr + i386_ptob(tmpidx), - VM_PAGE_TO_PHYS(p), mpte); + VM_PAGE_TO_PHYS(p)); p->flags |= PG_MAPPED; PAGE_WAKEUP(p); } @@ -1843,7 +1855,7 @@ pmap_prefault(pmap, addra, entry, object) vm_offset_t starta; vm_offset_t addr; vm_pindex_t pindex; - vm_page_t m, mpte; + vm_page_t m; if (entry->object.vm_object != object) return; @@ -1858,7 +1870,6 @@ pmap_prefault(pmap, addra, entry, object) starta = 0; } - mpte = NULL; for (i = 0; i < PAGEORDER_SIZE; i++) { vm_object_t lobject; unsigned *pte; @@ -1899,8 +1910,7 @@ pmap_prefault(pmap, addra, entry, object) vm_page_deactivate(m); } m->flags |= PG_BUSY; - mpte = pmap_enter_quick(pmap, addr, - VM_PAGE_TO_PHYS(m), mpte); + pmap_enter_quick(pmap, addr, VM_PAGE_TO_PHYS(m)); m->flags |= PG_MAPPED; PAGE_WAKEUP(m); } @@ -1939,6 +1949,8 @@ pmap_change_wiring(pmap, va, wired) pmap_pte_set_w(pte, wired); } + + /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len @@ -1975,19 +1987,16 @@ pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) unsigned *src_pte, *dst_pte; vm_page_t dstmpte, srcmpte; vm_offset_t srcptepaddr; - int ptepindex; if (addr >= UPT_MIN_ADDRESS) panic("pmap_copy: invalid to pmap_copy page tables\n"); - pdnxt = ((addr + PAGE_SIZE*NPTEPG) & ~(PAGE_SIZE*NPTEPG - 1)); - ptepindex = addr >> PDRSHIFT; - - srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[ptepindex]; - if (srcptepaddr == 0) + srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[addr >> PDRSHIFT]; + if (srcptepaddr == 0) { continue; + } - srcmpte = vm_page_lookup(src_pmap->pm_pteobj, ptepindex); + srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); if (srcmpte->hold_count == 0) continue; @@ -2011,14 +2020,12 @@ pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) dstmpte = pmap_allocpte(dst_pmap, addr); if ((*dst_pte == 0) && (ptetemp = *src_pte)) { /* - * Clear the modified and - * accessed (referenced) bits - * during the copy. + * Simply clear the modified and accessed (referenced) + * bits. */ *dst_pte = ptetemp & ~(PG_M|PG_A); dst_pmap->pm_stats.resident_count++; - pmap_insert_entry(dst_pmap, addr, - dstmpte, + pmap_insert_entry(dst_pmap, addr, dstmpte, (ptetemp & PG_FRAME)); } else { pmap_unwire_pte_hold(dst_pmap, dstmpte); @@ -2119,8 +2126,7 @@ pmap_page_exists(pmap, pa) pmap_t pmap; vm_offset_t pa; { - register pv_entry_t pv; - pv_table_t *ppv; + register pv_entry_t *ppv, pv; int s; if (!pmap_is_managed(pa)) @@ -2132,9 +2138,7 @@ pmap_page_exists(pmap, pa) /* * Not found, check current mappings returning immediately if found. */ - for (pv = TAILQ_FIRST(&ppv->pv_list); - pv; - pv = TAILQ_NEXT(pv, pv_list)) { + for (pv = *ppv; pv; pv = pv->pv_next) { if (pv->pv_pmap == pmap) { splx(s); return TRUE; @@ -2144,72 +2148,149 @@ pmap_page_exists(pmap, pa) return (FALSE); } -#define PMAP_REMOVE_PAGES_CURPROC_ONLY /* - * Remove all pages from specified address space - * this aids process exit speeds. Also, this code - * is special cased for current process only. + * pmap_testbit tests bits in pte's + * note that the testbit/changebit routines are inline, + * and a lot of things compile-time evaluate. */ -void -pmap_remove_pages(pmap, sva, eva) - pmap_t pmap; - vm_offset_t sva, eva; +static __inline boolean_t +pmap_testbit(pa, bit) + register vm_offset_t pa; + int bit; { - unsigned *pte, tpte; - pv_table_t *ppv; - pv_entry_t pv, npv; + register pv_entry_t *ppv, pv; + unsigned *pte; int s; -#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY - if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap)) { - printf("warning: pmap_remove_pages called with non-current pmap\n"); - return; - } -#endif + if (!pmap_is_managed(pa)) + return FALSE; + + ppv = pa_to_pvh(pa); + if (*ppv == NULL) + return FALSE; s = splvm(); + /* + * Not found, check current mappings returning immediately if found. + */ + for (pv = *ppv ;pv; pv = pv->pv_next) { - for(pv = TAILQ_FIRST(&pmap->pm_pvlist.pv_list); - pv; - pv = npv) { + /* + * if the bit being tested is the modified bit, then + * mark clean_map and ptes as never + * modified. + */ + if (bit & (PG_A|PG_M)) { + if (!pmap_track_modified(pv->pv_va)) + continue; + } - if (pv->pv_va >= eva || pv->pv_va < sva) { - npv = TAILQ_NEXT(pv, pv_plist); + if (!pv->pv_pmap) { +#if defined(PMAP_DIAGNOSTIC) + printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va); +#endif continue; } - -#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY - pte = (unsigned *)vtopte(pv->pv_va); -#else pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); -#endif - tpte = *pte; - *pte = 0; - - if (tpte) { - pv->pv_pmap->pm_stats.resident_count--; - if (tpte & PG_W) - pv->pv_pmap->pm_stats.wired_count--; - /* - * Update the vm_page_t clean and reference bits. - */ - if (tpte & PG_M) { - PHYS_TO_VM_PAGE(tpte)->dirty = VM_PAGE_BITS_ALL; - } + if (pte == NULL) + continue; + if (*pte & bit) { + splx(s); + return TRUE; } + } + splx(s); + return (FALSE); +} - npv = TAILQ_NEXT(pv, pv_plist); - TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist.pv_list, pv, pv_plist); +/* + * this routine is used to modify bits in ptes + */ +static __inline void +pmap_changebit(pa, bit, setem) + vm_offset_t pa; + int bit; + boolean_t setem; +{ + register pv_entry_t pv, *ppv; + register unsigned *pte; + vm_offset_t va; + int changed; + int s; - ppv = pa_to_pvh(tpte); - --ppv->pv_list_count; - TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); + if (!pmap_is_managed(pa)) + return; - pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); - free_pv_entry(pv); + s = splvm(); + changed = 0; + ppv = pa_to_pvh(pa); + /* + * Loop over all current mappings setting/clearing as appropos If + * setting RO do we need to clear the VAC? + */ + for ( pv = *ppv; pv; pv = pv->pv_next) { + va = pv->pv_va; + + /* + * don't write protect pager mappings + */ + if (!setem && (bit == PG_RW)) { + if (va >= clean_sva && va < clean_eva) + continue; + } + if (!pv->pv_pmap) { +#if defined(PMAP_DIAGNOSTIC) + printf("Null pmap (cb) at va: 0x%lx\n", va); +#endif + continue; + } + + pte = pmap_pte_quick(pv->pv_pmap, va); + if (pte == NULL) + continue; + if (setem) { + *(int *)pte |= bit; + changed = 1; + } else { + vm_offset_t pbits = *(vm_offset_t *)pte; + if (pbits & bit) + changed = 1; + if (bit == PG_RW) { + if (pbits & PG_M) { + vm_page_t m; + vm_offset_t pa = pbits & PG_FRAME; + m = PHYS_TO_VM_PAGE(pa); + m->dirty = VM_PAGE_BITS_ALL; + } + *(int *)pte = pbits & ~(PG_M|PG_RW); + } else { + *(int *)pte = pbits & ~bit; + } + } } - pmap_update(); splx(s); + if (changed) + pmap_update(); +} + +/* + * pmap_page_protect: + * + * Lower the permission for all mappings to a given page. + */ +void +pmap_page_protect(phys, prot) + vm_offset_t phys; + vm_prot_t prot; +{ + if ((prot & VM_PROT_WRITE) == 0) { + if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { + pmap_changebit(phys, PG_RW, FALSE); + } else { + pmap_remove_all(phys); + pmap_update(); + } + } } vm_offset_t @@ -2220,107 +2301,150 @@ pmap_phys_address(ppn) } /* - * pmap_tcbit: + * pmap_is_referenced: + * + * Return whether or not the specified physical page was referenced + * by any physical maps. + */ +boolean_t +pmap_is_referenced(vm_offset_t pa) +{ + register pv_entry_t *ppv, pv, lpv; + unsigned *pte; + int s; + + if (!pmap_is_managed(pa)) + return FALSE; + + ppv = pa_to_pvh(pa); + + s = splvm(); + /* + * Not found, check current mappings returning immediately if found. + */ + for (lpv = NULL, pv = *ppv ;pv; lpv = pv, pv = pv->pv_next) { + /* + * if the bit being tested is the modified bit, then + * mark clean_map and ptes as never + * modified. + */ + if (!pmap_track_modified(pv->pv_va)) + continue; + if (!pv->pv_pmap) { + continue; + } + pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); + if (pte == NULL) + continue; + if ((int) *pte & PG_A) { + if (lpv) { + lpv->pv_next = pv->pv_next; + pv->pv_next = *ppv; + *ppv = pv; + } + splx(s); + return TRUE; + } + } + splx(s); + return (FALSE); +} + +/* + * pmap_ts_referenced: * - * Return the count of bits for a page, clearing all of them. + * Return the count of reference bits for a page, clearing all of them. * */ int -pmap_tcbit(vm_offset_t pa, int bit) +pmap_ts_referenced(vm_offset_t pa) { - register pv_entry_t pv, npv; - pv_table_t *ppv; + register pv_entry_t *ppv, pv; unsigned *pte; int s; int rtval = 0; + vm_offset_t vachanged[VATRACK]; + + if (!pmap_is_managed(pa)) + return FALSE; s = splvm(); ppv = pa_to_pvh(pa); + + if (*ppv == NULL) { + splx(s); + return 0; + } + /* * Not found, check current mappings returning immediately if found. */ - for (pv = TAILQ_FIRST(&ppv->pv_list); - pv; - pv = npv) { - npv = TAILQ_NEXT(pv, pv_list); + for (pv = *ppv ;pv; pv = pv->pv_next) { /* * if the bit being tested is the modified bit, then * mark clean_map and ptes as never * modified. */ - if (((bit & PG_M) != 0) - && !pmap_track_modified(pv->pv_va)) + if (!pmap_track_modified(pv->pv_va)) continue; + if (!pv->pv_pmap) { + continue; + } pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (pte == NULL) continue; - - if ((rtval == 0) && (*pte & bit)) { - rtval = 1; + if (*pte & PG_A) { + if (rtval < VATRACK) + vachanged[rtval] = pv->pv_va; + rtval++; + *pte &= ~PG_A; } - *pte &= ~bit; } splx(s); - if (rtval) - pmap_update(); + if (rtval) { + if (rtval <= VATRACK) { + int i; + for(i=0;i<rtval;i++) + pmap_update_1pg(vachanged[i]); + } else { + pmap_update(); + } + } return (rtval); } /* - * pmap_tc_modified: + * pmap_is_modified: * - * Return the count of modified bits for a page, clearing all of them. - * + * Return whether or not the specified physical page was modified + * in any physical maps. */ -int -pmap_tc_modified(vm_page_t m) +boolean_t +pmap_is_modified(vm_offset_t pa) { - int rtval; - - rtval = pmap_tcbit(VM_PAGE_TO_PHYS(m), PG_M); - if (rtval) - m->dirty = VM_PAGE_BITS_ALL; - - return rtval; + return pmap_testbit((pa), PG_M); } /* - * pmap_tc_referenced: - * - * Return the count of referenced bits for a page, clearing all of them. - * + * Clear the modify bits on the specified physical page. */ -int -pmap_tc_referenced(vm_offset_t pa) +void +pmap_clear_modify(vm_offset_t pa) { - if (!pmap_is_managed(pa)) - return 0; - return pmap_tcbit(pa, PG_A); + pmap_changebit((pa), PG_M, FALSE); } /* - * pmap_page_protect: + * pmap_clear_reference: * - * Lower the permission for all mappings to a given page. + * Clear the reference bit on the specified physical page. */ void -pmap_page_protect(m, prot) - vm_page_t m; - vm_prot_t prot; +pmap_clear_reference(vm_offset_t pa) { - if ((prot & VM_PROT_WRITE) == 0) { - if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { - if ((m->flags & PG_FICTITIOUS) == 0) - pmap_tcbit(VM_PAGE_TO_PHYS(m), PG_RW); - } else { - if (pmap_remove_all(VM_PAGE_TO_PHYS(m))) { - m->dirty = VM_PAGE_BITS_ALL; - } - pmap_update(); - } - } + pmap_changebit((pa), PG_A, FALSE); } /* @@ -2401,7 +2525,7 @@ pmap_mincore(pmap, addr) unsigned *ptep, pte; int val = 0; - ptep = pmap_pte_quick(pmap, addr); + ptep = pmap_pte(pmap, addr); if (ptep == 0) { return 0; } @@ -2420,10 +2544,8 @@ pmap_mincore(pmap, addr) * Modified by someone */ else if (PHYS_TO_VM_PAGE(pa)->dirty || - pmap_tcbit(pa, PG_M)) { + pmap_is_modified(pa)) val |= MINCORE_MODIFIED_OTHER; - PHYS_TO_VM_PAGE(pa)->dirty = VM_PAGE_BITS_ALL; - } /* * Referenced by us */ @@ -2434,7 +2556,7 @@ pmap_mincore(pmap, addr) * Referenced by someone */ else if ((PHYS_TO_VM_PAGE(pa)->flags & PG_REFERENCED) || - pmap_tcbit(pa, PG_A)) + pmap_is_referenced(pa)) val |= MINCORE_REFERENCED_OTHER; } return val; @@ -2470,7 +2592,7 @@ pmap_pid_dump(int pid) { } return npte; } - pte = pmap_pte_quick( pmap, va); + pte = pmap_pte( pmap, va); if (pte && pmap_pte_v(pte)) { vm_offset_t pa; vm_page_t m; @@ -2519,7 +2641,7 @@ pads(pm) continue; if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) continue; - ptep = pmap_pte_quick(pm, va); + ptep = pmap_pte(pm, va); if (pmap_pte_v(ptep)) printf("%x:%x ", va, *(int *) ptep); }; @@ -2533,9 +2655,7 @@ pmap_pvdump(pa) register pv_entry_t pv; printf("pa %x", pa); - for (pv = TAILQ_FIRST(pa_to_pvh(pa)); - pv; - pv = TAILQ_NEXT(pv, pv_list)) { + for (pv = pa_to_pvh(pa); pv; pv = pv->pv_next) { #ifdef used_to_be printf(" -> pmap %x, va %x, flags %x", pv->pv_pmap, pv->pv_va, pv->pv_flags); diff --git a/sys/amd64/include/pmap.h b/sys/amd64/include/pmap.h index b53db62..64b682c 100644 --- a/sys/amd64/include/pmap.h +++ b/sys/amd64/include/pmap.h @@ -42,13 +42,12 @@ * * from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90 * from: @(#)pmap.h 7.4 (Berkeley) 5/12/91 - * $Id: pmap.h,v 1.40 1996/06/08 11:21:19 bde Exp $ + * $Id: pmap.h,v 1.41 1996/07/27 03:23:32 dyson Exp $ */ #ifndef _MACHINE_PMAP_H_ #define _MACHINE_PMAP_H_ - /* * Page-directory and page-table entires follow this format, with a few * of the fields not present here and there, depending on a lot of things. @@ -114,9 +113,6 @@ #define ISA_HOLE_LENGTH (0x100000-ISA_HOLE_START) #ifndef LOCORE - -#include <sys/queue.h> - typedef unsigned int *pd_entry_t; typedef unsigned int *pt_entry_t; @@ -162,24 +158,17 @@ pmap_kextract(vm_offset_t va) } #endif -struct vm_page; - /* * Pmap stuff */ -struct pv_entry; -typedef struct { - int pv_list_count; - TAILQ_HEAD(,pv_entry) pv_list; -} pv_table_t; struct pmap { pd_entry_t *pm_pdir; /* KVA of page directory */ vm_object_t pm_pteobj; /* Container for pte's */ - pv_table_t pm_pvlist; /* list of mappings in pmap */ - int pm_count; /* reference count */ + short pm_dref; /* page directory ref count */ + short pm_count; /* pmap reference count */ struct pmap_statistics pm_stats; /* pmap statistics */ - struct vm_page *pm_ptphint; /* pmap ptp hint */ + struct vm_map *pm_map; /* map that owns this pmap */ }; typedef struct pmap *pmap_t; @@ -188,16 +177,14 @@ typedef struct pmap *pmap_t; extern pmap_t kernel_pmap; #endif - /* * For each vm_page_t, there is a list of all currently valid virtual * mappings of that page. An entry is a pv_entry_t, the list is pv_table. */ typedef struct pv_entry { + struct pv_entry *pv_next; /* next pv_entry */ pmap_t pv_pmap; /* pmap where mapping lies */ vm_offset_t pv_va; /* virtual address for mapping */ - TAILQ_ENTRY(pv_entry) pv_list; - TAILQ_ENTRY(pv_entry) pv_plist; vm_page_t pv_ptem; /* VM page for pte */ } *pv_entry_t; @@ -213,7 +200,7 @@ extern pt_entry_t *CMAP1; extern vm_offset_t avail_end; extern vm_offset_t avail_start; extern vm_offset_t phys_avail[]; -pv_table_t *pv_table; +extern pv_entry_t *pv_table; /* array of entries, one per page */ extern vm_offset_t virtual_avail; extern vm_offset_t virtual_end; diff --git a/sys/i386/i386/pmap.c b/sys/i386/i386/pmap.c index 97be59c..4998f78 100644 --- a/sys/i386/i386/pmap.c +++ b/sys/i386/i386/pmap.c @@ -39,7 +39,7 @@ * SUCH DAMAGE. * * from: @(#)pmap.c 7.7 (Berkeley) 5/12/91 - * $Id: pmap.c,v 1.111 1996/07/28 20:31:27 dyson Exp $ + * $Id: pmap.c,v 1.113 1996/07/29 14:22:46 dyson Exp $ */ /* @@ -99,12 +99,6 @@ #define PMAP_DIAGNOSTIC #endif -#if !defined(SMALL_KERNEL) -#define PMAP_INLINE __inline -#else -#define PMAP_INLINE -#endif - static void init_pv_entries __P((int)); /* @@ -152,7 +146,7 @@ extern int cpu_class; * Data for the pv entry allocation mechanism */ static int pv_freelistcnt; -TAILQ_HEAD (,pv_entry) pv_freelist; +static pv_entry_t pv_freelist; static vm_offset_t pvva; static int npvvapg; @@ -161,6 +155,7 @@ static int npvvapg; */ pt_entry_t *CMAP1; static pt_entry_t *CMAP2, *ptmmap; +static pv_entry_t *pv_table; caddr_t CADDR1, ptvmmap; static caddr_t CADDR2; static pt_entry_t *msgbufmap; @@ -170,32 +165,32 @@ pt_entry_t *PMAP1; unsigned *PADDR1; static void free_pv_entry __P((pv_entry_t pv)); -static unsigned * get_ptbase __P((pmap_t pmap)); +static __inline unsigned * get_ptbase __P((pmap_t pmap)); static pv_entry_t get_pv_entry __P((void)); static void i386_protection_init __P((void)); static void pmap_alloc_pv_entry __P((void)); +static void pmap_changebit __P((vm_offset_t pa, int bit, boolean_t setem)); static int pmap_is_managed __P((vm_offset_t pa)); -static int pmap_remove_all __P((vm_offset_t pa)); -static vm_page_t pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, - vm_offset_t pa, vm_page_t mpte)); +static void pmap_remove_all __P((vm_offset_t pa)); +static void pmap_enter_quick __P((pmap_t pmap, vm_offset_t va, + vm_offset_t pa)); static int pmap_remove_pte __P((struct pmap *pmap, unsigned *ptq, vm_offset_t sva)); static void pmap_remove_page __P((struct pmap *pmap, vm_offset_t va)); -static int pmap_remove_entry __P((struct pmap *pmap, pv_table_t *pv, +static __inline int pmap_remove_entry __P((struct pmap *pmap, pv_entry_t *pv, vm_offset_t va)); -static void pmap_insert_entry __P((pmap_t pmap, vm_offset_t va, +static boolean_t pmap_testbit __P((vm_offset_t pa, int bit)); +static __inline void pmap_insert_entry __P((pmap_t pmap, vm_offset_t va, vm_page_t mpte, vm_offset_t pa)); -static vm_page_t pmap_allocpte __P((pmap_t pmap, vm_offset_t va)); +static __inline vm_page_t pmap_allocpte __P((pmap_t pmap, vm_offset_t va)); -static int pmap_release_free_page __P((pmap_t pmap, vm_page_t p)); +static __inline int pmap_release_free_page __P((pmap_t pmap, vm_page_t p)); static vm_page_t _pmap_allocpte __P((pmap_t pmap, int ptepindex)); -unsigned * __pure pmap_pte_quick __P((pmap_t pmap, vm_offset_t va)); -int pmap_tcbit __P((vm_offset_t pa, int bit)); -static vm_page_t pmap_page_alloc __P((vm_object_t object, vm_pindex_t pindex)); -#define PDSTACKMAX 6 +#define VATRACK 4 +#define PDSTACKMAX 16 static vm_offset_t pdstack[PDSTACKMAX]; static int pdstackptr; @@ -245,7 +240,6 @@ pmap_bootstrap(firstaddr, loadaddr) kernel_pmap->pm_pdir = (pd_entry_t *) (KERNBASE + IdlePTD); kernel_pmap->pm_count = 1; - TAILQ_INIT(&kernel_pmap->pm_pvlist.pv_list); nkpt = NKPT; /* @@ -312,15 +306,10 @@ pmap_init(phys_start, phys_end) * Allocate memory for random pmap data structures. Includes the * pv_head_table. */ - s = (vm_size_t) (sizeof(pv_table_t) * npg); + s = (vm_size_t) (sizeof(struct pv_entry *) * npg); s = round_page(s); addr = (vm_offset_t) kmem_alloc(kernel_map, s); - pv_table = (pv_table_t *) addr; - for(i=0;i<npg;i++) { - pv_table[i].pv_list_count = 0; - TAILQ_INIT(&pv_table[i].pv_list); - } - TAILQ_INIT(&pv_freelist); + pv_table = (pv_entry_t *) addr; /* * init the pv free list @@ -383,7 +372,7 @@ pmap_nw_modified(pt_entry_t ptea) { * this routine defines the region(s) of memory that should * not be tested for the modified bit. */ -static PMAP_INLINE int +static __inline int pmap_track_modified( vm_offset_t va) { if ((va < clean_sva) || (va >= clean_eva)) return 1; @@ -395,7 +384,7 @@ pmap_track_modified( vm_offset_t va) { * The below are finer grained pmap_update routines. These eliminate * the gratuitious tlb flushes on non-i386 architectures. */ -static PMAP_INLINE void +static __inline void pmap_update_1pg( vm_offset_t va) { #if defined(I386_CPU) if (cpu_class == CPUCLASS_386) @@ -405,7 +394,7 @@ pmap_update_1pg( vm_offset_t va) { __asm __volatile(".byte 0xf,0x1,0x38": :"a" (va)); } -static PMAP_INLINE void +static __inline void pmap_update_2pg( vm_offset_t va1, vm_offset_t va2) { #if defined(I386_CPU) if (cpu_class == CPUCLASS_386) { @@ -443,7 +432,7 @@ get_ptbase(pmap) * with the given map/virtual_address pair. */ -unsigned * __pure +__inline unsigned * __pure pmap_pte(pmap, va) register pmap_t pmap; vm_offset_t va; @@ -459,27 +448,25 @@ pmap_pte(pmap, va) * the pv lists. This eliminates many coarse-grained * pmap_update calls. */ -unsigned * __pure +__inline unsigned * __pure pmap_pte_quick(pmap, va) register pmap_t pmap; vm_offset_t va; { - unsigned pde, newpf; + unsigned pde; if (pde = (unsigned) pmap->pm_pdir[va >> PDRSHIFT]) { unsigned frame = (unsigned) pmap->pm_pdir[PTDPTDI] & PG_FRAME; /* are we current address space or kernel? */ if (pmap == kernel_pmap || frame == (((unsigned) PTDpde) & PG_FRAME)) { return (unsigned *) PTmap + i386_btop(va); } - newpf = pde & PG_FRAME; - if ( ((* (unsigned *) PMAP1) & PG_FRAME) != newpf) { - * (unsigned *) PMAP1 = newpf | PG_RW | PG_V; - pmap_update_1pg((vm_offset_t) PADDR1); - } + * (int *) PMAP1 = (pde & PG_FRAME) | PG_V | PG_RW; + pmap_update_1pg((vm_offset_t) PADDR1); return PADDR1 + ((unsigned) i386_btop(va) & (NPTEPG - 1)); } return (0); } + /* * Routine: pmap_extract @@ -504,7 +491,7 @@ pmap_extract(pmap, va) /* * determine if a page is managed (memory vs. device) */ -static PMAP_INLINE __pure int +static __inline __pure int pmap_is_managed(pa) vm_offset_t pa; { @@ -577,7 +564,7 @@ pmap_qremove(va, count) * note that in order for the mapping to take effect -- you * should do a pmap_update after doing the pmap_kenter... */ -PMAP_INLINE void +__inline void pmap_kenter(va, pa) vm_offset_t va; register vm_offset_t pa; @@ -596,7 +583,7 @@ pmap_kenter(va, pa) /* * remove a page from the kernel pagetables */ -PMAP_INLINE void +__inline void pmap_kremove(va) vm_offset_t va; { @@ -607,17 +594,80 @@ pmap_kremove(va) pmap_update_1pg(va); } -static vm_page_t -pmap_page_alloc(object, pindex) - vm_object_t object; - vm_pindex_t pindex; + +/*************************************************** + * Page table page management routines..... + ***************************************************/ + +/* + * This routine unholds page table pages, and if the hold count + * drops to zero, then it decrements the wire count. + */ +static __inline int +pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { + vm_page_unhold(m); + if (m->hold_count == 0) { + vm_offset_t pteva; + /* + * unmap the page table page + */ + pmap->pm_pdir[m->pindex] = 0; + --pmap->pm_stats.resident_count; + /* + * Do a pmap_update to make the invalidated mapping + * take effect immediately. + */ + pteva = UPT_MIN_ADDRESS + i386_ptob(m->pindex); + pmap_update_1pg(pteva); + /* + * If the page is finally unwired, simply free it. + */ + --m->wire_count; + if (m->wire_count == 0) { + vm_page_free_zero(m); + --cnt.v_wire_count; + } + return 1; + } + return 0; +} + +/* + * After removing a page table entry, this routine is used to + * conditionally free the page, and manage the hold/wire counts. + */ +int +pmap_unuse_pt(pmap, va, mpte) + pmap_t pmap; + vm_offset_t va; + vm_page_t mpte; { - vm_page_t m; - m = vm_page_alloc(object, pindex, VM_ALLOC_ZERO); - if (m == NULL) { - VM_WAIT; + if (va >= UPT_MIN_ADDRESS) + return 0; + + if (mpte == NULL) { + vm_offset_t ptepa; + ptepa = ((vm_offset_t) *pmap_pde(pmap, va)); +#if defined(PMAP_DIAGNOSTIC) + if (!ptepa) + panic("pmap_unuse_pt: pagetable page missing, va: 0x%x", va); +#endif + if (!ptepa) + return 0; + mpte = PHYS_TO_VM_PAGE(ptepa); } - return m; + +#if defined(PMAP_DIAGNOSTIC) + if (mpte->pindex != (va >> PDRSHIFT)) + panic("pmap_unuse_pt: pindex(0x%x) != va(0x%x)", + mpte->pindex, (va >> PDRSHIFT)); + + if (mpte->hold_count == 0) { + panic("pmap_unuse_pt: hold count < 0, va: 0x%x", va); + } +#endif + + return pmap_unwire_pte_hold(pmap, mpte); } /* @@ -651,12 +701,12 @@ pmap_pinit(pmap) * allocate the page directory page */ retry: - ptdpg = pmap_page_alloc( pmap->pm_pteobj, PTDPTDI); - if (ptdpg == NULL) + ptdpg = vm_page_alloc( pmap->pm_pteobj, PTDPTDI, VM_ALLOC_ZERO); + if (ptdpg == NULL) { + VM_WAIT; goto retry; - - ptdpg->wire_count = 1; - ++cnt.v_wire_count; + } + vm_page_wire(ptdpg); ptdpg->flags &= ~(PG_MAPPED|PG_BUSY); /* not mapped normally */ ptdpg->valid = VM_PAGE_BITS_ALL; @@ -672,7 +722,6 @@ retry: VM_PAGE_TO_PHYS(ptdpg) | PG_V | PG_RW; pmap->pm_count = 1; - TAILQ_INIT(&pmap->pm_pvlist.pv_list); } static int @@ -702,7 +751,31 @@ pmap_release_free_page(pmap, p) --pmap->pm_stats.resident_count; if (p->hold_count) { + int *kvap; + int i; +#if defined(PMAP_DIAGNOSTIC) panic("pmap_release: freeing held page table page"); +#else + printf("pmap_release: freeing held page table page:\n"); +#endif + kvap = (int *)vm_pager_map_page(p); + for(i=0;i<NPTEPG;i++) { + if (kvap[i]) { + printf("pte: 0x%x, index: %d\n", kvap[i],i); + } + } + vm_pager_unmap_page((vm_offset_t)kvap); + + /* + * HACK ALERT!!! + * If this failure happens, we must clear the page, because + * there is likely a mapping still valid. This condition + * is an error, but at least this zero operation will mitigate + * some Sig-11's or crashes, because this page is thought + * to be zero. This is a robustness fix, and not meant to + * be a long term work-around. + */ + pmap_zero_page(VM_PAGE_TO_PHYS(p)); } /* * Page directory pages need to have the kernel @@ -728,7 +801,7 @@ _pmap_allocpte(pmap, ptepindex) pmap_t pmap; int ptepindex; { - vm_offset_t ptepa; + vm_offset_t pteva, ptepa; vm_page_t m; /* @@ -737,9 +810,11 @@ _pmap_allocpte(pmap, ptepindex) retry: m = vm_page_lookup(pmap->pm_pteobj, ptepindex); if (m == NULL) { - m = pmap_page_alloc(pmap->pm_pteobj, ptepindex); - if (m == NULL) + m = vm_page_alloc(pmap->pm_pteobj, ptepindex, VM_ALLOC_ZERO); + if (m == NULL) { + VM_WAIT; goto retry; + } if ((m->flags & PG_ZERO) == 0) pmap_zero_page(VM_PAGE_TO_PHYS(m)); m->flags &= ~(PG_ZERO|PG_BUSY); @@ -752,16 +827,22 @@ retry: } } + /* + * mark the object writeable + */ + pmap->pm_pteobj->flags |= OBJ_WRITEABLE; + if (m->queue != PQ_NONE) { int s = splvm(); - vm_page_unqueue(m,1); + vm_page_unqueue(m); splx(s); } - if (m->wire_count == 0) - ++cnt.v_wire_count; - ++m->wire_count; - + if (m->hold_count == 0) { + if (m->wire_count == 0) + ++cnt.v_wire_count; + ++m->wire_count; + } /* * Increment the hold count for the page table page * (denoting a new mapping.) @@ -778,12 +859,14 @@ retry: ptepa = VM_PAGE_TO_PHYS(m); pmap->pm_pdir[ptepindex] = (pd_entry_t) (ptepa | PG_U | PG_RW | PG_V); + pteva = UPT_MIN_ADDRESS + i386_ptob(ptepindex); + pmap_update_1pg(pteva); m->flags |= PG_MAPPED; return m; } -PMAP_INLINE static vm_page_t +static __inline vm_page_t pmap_allocpte(pmap, va) pmap_t pmap; vm_offset_t va; @@ -807,7 +890,7 @@ pmap_allocpte(pmap, va) * hold count, and activate it. */ if (ptepa) { - m = vm_page_lookup( pmap->pm_pteobj, ptepindex); + m = PHYS_TO_VM_PAGE(ptepa); ++m->hold_count; return m; } @@ -952,12 +1035,13 @@ pmap_reference(pmap) /* * free the pv_entry back to the free list */ -static PMAP_INLINE void +static __inline void free_pv_entry(pv) pv_entry_t pv; { ++pv_freelistcnt; - TAILQ_INSERT_HEAD(&pv_freelist, pv, pv_list); + pv->pv_next = pv_freelist; + pv_freelist = pv; } /* @@ -966,7 +1050,7 @@ free_pv_entry(pv) * the memory allocation is performed bypassing the malloc code * because of the possibility of allocations at interrupt time. */ -static PMAP_INLINE pv_entry_t +static __inline pv_entry_t get_pv_entry() { pv_entry_t tmp; @@ -974,16 +1058,15 @@ get_pv_entry() /* * get more pv_entry pages if needed */ - if (pv_freelistcnt < PV_FREELIST_MIN) { + if (pv_freelistcnt < PV_FREELIST_MIN || pv_freelist == 0) { pmap_alloc_pv_entry(); } - /* * get a pv_entry off of the free list */ --pv_freelistcnt; - tmp = TAILQ_FIRST(&pv_freelist); - TAILQ_REMOVE(&pv_freelist, tmp, pv_list); + tmp = pv_freelist; + pv_freelist = tmp->pv_next; return tmp; } @@ -1040,7 +1123,7 @@ pmap_alloc_pv_entry() } } } - if (TAILQ_FIRST(&pv_freelist) == NULL) + if (!pv_freelist) panic("get_pv_entry: cannot get a pv_entry_t"); } @@ -1067,107 +1150,62 @@ init_pv_entries(npg) } /* - * This routine unholds page table pages, and if the hold count - * drops to zero, then it decrements the wire count. + * If it is the first entry on the list, it is actually + * in the header and we must copy the following entry up + * to the header. Otherwise we must search the list for + * the entry. In either case we free the now unused entry. */ -static int -pmap_unwire_pte_hold(pmap_t pmap, vm_page_t m) { - vm_page_unhold(m); - if (m->hold_count == 0) { - vm_offset_t pteva; - /* - * unmap the page table page - */ - pmap->pm_pdir[m->pindex] = 0; - --pmap->pm_stats.resident_count; - if ((((unsigned)pmap->pm_pdir[PTDPTDI]) & PG_FRAME) == - (((unsigned) PTDpde) & PG_FRAME)) { - /* - * Do a pmap_update to make the invalidated mapping - * take effect immediately. - */ - pteva = UPT_MIN_ADDRESS + i386_ptob(m->pindex); - pmap_update_1pg(pteva); - } - /* - * If the page is finally unwired, simply free it. - */ - --m->wire_count; - if (m->wire_count == 0) { - vm_page_free_zero(m); - --cnt.v_wire_count; +static __inline int +pmap_remove_entry(pmap, ppv, va) + struct pmap *pmap; + pv_entry_t *ppv; + vm_offset_t va; +{ + pv_entry_t npv; + int s; + + s = splvm(); + for (npv = *ppv; npv; (ppv = &npv->pv_next, npv = *ppv)) { + if (pmap == npv->pv_pmap && va == npv->pv_va) { + int rtval = pmap_unuse_pt(pmap, va, npv->pv_ptem); + *ppv = npv->pv_next; + free_pv_entry(npv); + splx(s); + return rtval; } - return 1; } + splx(s); return 0; } /* - * After removing a page table entry, this routine is used to - * conditionally free the page, and manage the hold/wire counts. + * Create a pv entry for page at pa for + * (pmap, va). */ -PMAP_INLINE int -pmap_unuse_pt(pmap, va, mpte) +static __inline void +pmap_insert_entry(pmap, va, mpte, pa) pmap_t pmap; vm_offset_t va; vm_page_t mpte; + vm_offset_t pa; { - int ptepindex; - if (va >= UPT_MIN_ADDRESS) - return 0; - if (mpte == NULL) { - ptepindex = (va >> PDRSHIFT); - mpte = vm_page_lookup( pmap->pm_pteobj, ptepindex); - } - - return pmap_unwire_pte_hold(pmap, mpte); -} - -/* - * If it is the first entry on the list, it is actually - * in the header and we must copy the following entry up - * to the header. Otherwise we must search the list for - * the entry. In either case we free the now unused entry. - */ -static int -pmap_remove_entry(pmap, ppv, va) - struct pmap *pmap; - pv_table_t *ppv; - vm_offset_t va; -{ - pv_entry_t pv; - int rtval; int s; + pv_entry_t *ppv, pv; s = splvm(); - if (ppv->pv_list_count < pmap->pm_stats.resident_count) { - for (pv = TAILQ_FIRST(&ppv->pv_list); - pv; - pv = TAILQ_NEXT(pv, pv_list)) { - if (pmap == pv->pv_pmap && va == pv->pv_va) - break; - } - } else { - for (pv = TAILQ_FIRST(&pmap->pm_pvlist.pv_list); - pv; - pv = TAILQ_NEXT(pv, pv_plist)) { - if (va == pv->pv_va) - break; - } - } + pv = get_pv_entry(); + pv->pv_va = va; + pv->pv_pmap = pmap; + pv->pv_ptem = mpte; - rtval = 0; - if (pv) { - rtval = pmap_unuse_pt(pmap, va, pv->pv_ptem); - TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); - --ppv->pv_list_count; - TAILQ_REMOVE(&pmap->pm_pvlist.pv_list, pv, pv_plist); - free_pv_entry(pv); - } - + ppv = pa_to_pvh(pa); + if (*ppv) + pv->pv_next = *ppv; + else + pv->pv_next = NULL; + *ppv = pv; splx(s); - return rtval; } /* @@ -1180,6 +1218,7 @@ pmap_remove_pte(pmap, ptq, va) vm_offset_t va; { unsigned oldpte; + pv_entry_t *ppv; oldpte = *ptq; *ptq = 0; @@ -1196,7 +1235,8 @@ pmap_remove_pte(pmap, ptq, va) if (pmap_track_modified(va)) PHYS_TO_VM_PAGE(oldpte)->dirty = VM_PAGE_BITS_ALL; } - return pmap_remove_entry(pmap, pa_to_pvh(oldpte), va); + ppv = pa_to_pvh(oldpte); + return pmap_remove_entry(pmap, ppv, va); } else { return pmap_unuse_pt(pmap, va, NULL); } @@ -1250,6 +1290,7 @@ pmap_remove(pmap, sva, eva) vm_offset_t sindex, eindex; vm_page_t mpte; int anyvalid; + vm_offset_t vachanged[VATRACK]; if (pmap == NULL) return; @@ -1274,7 +1315,6 @@ pmap_remove(pmap, sva, eva) sindex = i386_btop(sva); eindex = i386_btop(eva); - mpte = NULL; for (; sindex < eindex; sindex = pdnxt) { @@ -1291,6 +1331,19 @@ pmap_remove(pmap, sva, eva) if (ptpaddr == 0) continue; + if (sindex < i386_btop(UPT_MIN_ADDRESS)) { + /* + * get the vm_page_t for the page table page + */ + mpte = PHYS_TO_VM_PAGE(ptpaddr); + + /* + * if the pte isn't wired, just skip it. + */ + if (mpte->wire_count == 0) + continue; + } + /* * Limit our scan to either the end of the va represented * by the current page table page, or to the end of the @@ -1307,6 +1360,8 @@ pmap_remove(pmap, sva, eva) } va = i386_ptob(sindex); + if (anyvalid < VATRACK) + vachanged[anyvalid] = va; anyvalid++; if (pmap_remove_pte(pmap, ptbase + sindex, va)) @@ -1315,8 +1370,15 @@ pmap_remove(pmap, sva, eva) } if (anyvalid) { - pmap_update(); + if (anyvalid <= VATRACK) { + int i; + for(i=0;i<anyvalid;i++) + pmap_update_1pg(vachanged[i]); + } else { + pmap_update(); + } } + } /* @@ -1331,18 +1393,15 @@ pmap_remove(pmap, sva, eva) * inefficient because they iteratively called * pmap_remove (slow...) */ -static int +static void pmap_remove_all(pa) vm_offset_t pa; { - register pv_entry_t pv, npv; - pv_table_t *ppv; + register pv_entry_t pv, *ppv, npv; register unsigned *pte, tpte; vm_page_t m; - int nmodify; int s; - nmodify = 0; #if defined(PMAP_DIAGNOSTIC) /* * XXX this makes pmap_page_protect(NONE) illegal for non-managed @@ -1356,9 +1415,7 @@ pmap_remove_all(pa) s = splvm(); m = NULL; ppv = pa_to_pvh(pa); - for (pv = TAILQ_FIRST(&ppv->pv_list); - pv; - pv = npv) { + for (pv = *ppv; pv; pv=pv->pv_next) { pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (tpte = *pte) { pv->pv_pmap->pm_stats.resident_count--; @@ -1368,27 +1425,28 @@ pmap_remove_all(pa) /* * Update the vm_page_t clean and reference bits. */ - if ((tpte & (PG_M|PG_MANAGED)) == (PG_M|PG_MANAGED)) { + if (tpte & PG_M) { #if defined(PMAP_DIAGNOSTIC) if (pmap_nw_modified((pt_entry_t) tpte)) { printf("pmap_remove_all: modified page not writable: va: 0x%lx, pte: 0x%lx\n", pv->pv_va, tpte); } #endif - if (pmap_track_modified(pv->pv_va)) - nmodify += 1; + if (pmap_track_modified(pv->pv_va)) { + if (m == NULL) + m = PHYS_TO_VM_PAGE(pa); + m->dirty = VM_PAGE_BITS_ALL; + } } } - TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist.pv_list, pv, pv_plist); + } - npv = TAILQ_NEXT(pv, pv_list); - TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); - --ppv->pv_list_count; + for (pv = *ppv; pv; pv = npv) { + npv = pv->pv_next; pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); free_pv_entry(pv); } - + *ppv = NULL; splx(s); - return nmodify; } /* @@ -1426,7 +1484,6 @@ pmap_protect(pmap, sva, eva, prot) sindex = i386_btop(sva); eindex = i386_btop(eva); - mpte = NULL; for (; sindex < eindex; sindex = pdnxt) { pdnxt = ((sindex + NPTEPG) & ~(NPTEPG - 1)); @@ -1439,6 +1496,18 @@ pmap_protect(pmap, sva, eva, prot) if (ptpaddr == 0) continue; + /* + * Skip page ranges, where the page table page isn't wired. + * If the page table page is not wired, there are no page mappings + * there. + */ + if (sindex < i386_btop(UPT_MIN_ADDRESS)) { + mpte = PHYS_TO_VM_PAGE(ptpaddr); + + if (mpte->wire_count == 0) + continue; + } + if (pdnxt > eindex) { pdnxt = eindex; } @@ -1447,8 +1516,8 @@ pmap_protect(pmap, sva, eva, prot) unsigned pbits = ptbase[sindex]; - if ((pbits & (PG_RW|PG_V)) == (PG_RW|PG_V)) { - if ((pbits & (PG_M|PG_MANAGED)) == (PG_M|PG_MANAGED)) { + if (pbits & PG_RW) { + if (pbits & PG_M) { vm_offset_t sva = i386_ptob(sindex); if (pmap_track_modified(sva)) { vm_page_t m = PHYS_TO_VM_PAGE(pbits); @@ -1465,37 +1534,6 @@ pmap_protect(pmap, sva, eva, prot) } /* - * Create a pv entry for page at pa for - * (pmap, va). - */ -static void -pmap_insert_entry(pmap, va, mpte, pa) - pmap_t pmap; - vm_offset_t va; - vm_page_t mpte; - vm_offset_t pa; -{ - - int s; - pv_entry_t pv; - pv_table_t *ppv; - - s = splvm(); - pv = get_pv_entry(); - pv->pv_va = va; - pv->pv_pmap = pmap; - pv->pv_ptem = mpte; - - TAILQ_INSERT_TAIL(&pmap->pm_pvlist.pv_list, pv, pv_plist); - - ppv = pa_to_pvh(pa); - TAILQ_INSERT_TAIL(&ppv->pv_list, pv, pv_list); - ++ppv->pv_list_count; - - splx(s); -} - -/* * Insert the given physical page (p) at * the specified virtual address (v) in the * target physical map with the protection requested. @@ -1539,7 +1577,7 @@ pmap_enter(pmap, va, pa, prot, wired) if (va < UPT_MIN_ADDRESS) mpte = pmap_allocpte(pmap, va); - pte = pmap_pte(pmap, va); + pte = pmap_pte_quick(pmap, va); /* * Page Directory table entry not valid, we need a new PT page */ @@ -1555,7 +1593,7 @@ pmap_enter(pmap, va, pa, prot, wired) /* * Mapping has not changed, must be protection or wiring change. */ - if (origpte && (opa == pa)) { + if (opa == pa) { /* * Wiring change, just update stats. We don't worry about * wiring PT pages as they remain resident as long as there @@ -1578,10 +1616,12 @@ pmap_enter(pmap, va, pa, prot, wired) * so we go ahead and sense modify status. */ if (origpte & PG_MANAGED) { - if ((origpte & PG_M) && pmap_track_modified(va)) { - vm_page_t m; - m = PHYS_TO_VM_PAGE(pa); - m->dirty = VM_PAGE_BITS_ALL; + vm_page_t m; + if (origpte & PG_M) { + if (pmap_track_modified(va)) { + m = PHYS_TO_VM_PAGE(pa); + m->dirty = VM_PAGE_BITS_ALL; + } } pa |= PG_MANAGED; } @@ -1595,7 +1635,7 @@ pmap_enter(pmap, va, pa, prot, wired) * Mapping has changed, invalidate old range and fall through to * handle validating new mapping. */ - if (origpte) { + if (opa) { int err; err = pmap_remove_pte(pmap, pte, va); if (err) @@ -1652,49 +1692,22 @@ validate: * but is *MUCH* faster than pmap_enter... */ -static vm_page_t -pmap_enter_quick(pmap, va, pa, mpte) +static void +pmap_enter_quick(pmap, va, pa) register pmap_t pmap; vm_offset_t va; register vm_offset_t pa; - vm_page_t mpte; { register unsigned *pte; + vm_page_t mpte; + mpte = NULL; /* * In the case that a page table page is not * resident, we are creating it here. */ - if (va < UPT_MIN_ADDRESS) { - int ptepindex; - vm_offset_t ptepa; - - /* - * Calculate pagetable page index - */ - ptepindex = va >> PDRSHIFT; - if (mpte && (mpte->pindex == ptepindex)) { - ++mpte->hold_count; - } else { - /* - * Get the page directory entry - */ - ptepa = (vm_offset_t) pmap->pm_pdir[ptepindex]; - - /* - * If the page table page is mapped, we just increment - * the hold count, and activate it. - */ - if (ptepa) { - mpte = vm_page_lookup( pmap->pm_pteobj, ptepindex); - ++mpte->hold_count; - } else { - mpte = _pmap_allocpte(pmap, ptepindex); - } - } - } else { - mpte = NULL; - } + if (va < UPT_MIN_ADDRESS) + mpte = pmap_allocpte(pmap, va); /* * This call to vtopte makes the assumption that we are @@ -1706,7 +1719,7 @@ pmap_enter_quick(pmap, va, pa, mpte) if (*pte) { if (mpte) pmap_unwire_pte_hold(pmap, mpte); - return NULL; + return; } /* @@ -1726,7 +1739,7 @@ pmap_enter_quick(pmap, va, pa, mpte) */ *pte = pa | PG_V | PG_U | PG_MANAGED; - return mpte; + return; } #define MAX_INIT_PT (96) @@ -1746,7 +1759,7 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) { vm_offset_t tmpidx; int psize; - vm_page_t p, mpte; + vm_page_t p; int objpgs; psize = i386_btop(size); @@ -1760,7 +1773,6 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) if (psize + pindex > object->size) psize = object->size - pindex; - mpte = NULL; /* * if we are processing a major portion of the object, then scan the * entire thing. @@ -1786,9 +1798,9 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) if (p->queue == PQ_CACHE) vm_page_deactivate(p); p->flags |= PG_BUSY; - mpte = pmap_enter_quick(pmap, + pmap_enter_quick(pmap, addr + i386_ptob(tmpidx), - VM_PAGE_TO_PHYS(p), mpte); + VM_PAGE_TO_PHYS(p)); p->flags |= PG_MAPPED; PAGE_WAKEUP(p); } @@ -1807,9 +1819,9 @@ pmap_object_init_pt(pmap, addr, object, pindex, size, limit) if (p->queue == PQ_CACHE) vm_page_deactivate(p); p->flags |= PG_BUSY; - mpte = pmap_enter_quick(pmap, + pmap_enter_quick(pmap, addr + i386_ptob(tmpidx), - VM_PAGE_TO_PHYS(p), mpte); + VM_PAGE_TO_PHYS(p)); p->flags |= PG_MAPPED; PAGE_WAKEUP(p); } @@ -1843,7 +1855,7 @@ pmap_prefault(pmap, addra, entry, object) vm_offset_t starta; vm_offset_t addr; vm_pindex_t pindex; - vm_page_t m, mpte; + vm_page_t m; if (entry->object.vm_object != object) return; @@ -1858,7 +1870,6 @@ pmap_prefault(pmap, addra, entry, object) starta = 0; } - mpte = NULL; for (i = 0; i < PAGEORDER_SIZE; i++) { vm_object_t lobject; unsigned *pte; @@ -1899,8 +1910,7 @@ pmap_prefault(pmap, addra, entry, object) vm_page_deactivate(m); } m->flags |= PG_BUSY; - mpte = pmap_enter_quick(pmap, addr, - VM_PAGE_TO_PHYS(m), mpte); + pmap_enter_quick(pmap, addr, VM_PAGE_TO_PHYS(m)); m->flags |= PG_MAPPED; PAGE_WAKEUP(m); } @@ -1939,6 +1949,8 @@ pmap_change_wiring(pmap, va, wired) pmap_pte_set_w(pte, wired); } + + /* * Copy the range specified by src_addr/len * from the source map to the range dst_addr/len @@ -1975,19 +1987,16 @@ pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) unsigned *src_pte, *dst_pte; vm_page_t dstmpte, srcmpte; vm_offset_t srcptepaddr; - int ptepindex; if (addr >= UPT_MIN_ADDRESS) panic("pmap_copy: invalid to pmap_copy page tables\n"); - pdnxt = ((addr + PAGE_SIZE*NPTEPG) & ~(PAGE_SIZE*NPTEPG - 1)); - ptepindex = addr >> PDRSHIFT; - - srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[ptepindex]; - if (srcptepaddr == 0) + srcptepaddr = (vm_offset_t) src_pmap->pm_pdir[addr >> PDRSHIFT]; + if (srcptepaddr == 0) { continue; + } - srcmpte = vm_page_lookup(src_pmap->pm_pteobj, ptepindex); + srcmpte = PHYS_TO_VM_PAGE(srcptepaddr); if (srcmpte->hold_count == 0) continue; @@ -2011,14 +2020,12 @@ pmap_copy(dst_pmap, src_pmap, dst_addr, len, src_addr) dstmpte = pmap_allocpte(dst_pmap, addr); if ((*dst_pte == 0) && (ptetemp = *src_pte)) { /* - * Clear the modified and - * accessed (referenced) bits - * during the copy. + * Simply clear the modified and accessed (referenced) + * bits. */ *dst_pte = ptetemp & ~(PG_M|PG_A); dst_pmap->pm_stats.resident_count++; - pmap_insert_entry(dst_pmap, addr, - dstmpte, + pmap_insert_entry(dst_pmap, addr, dstmpte, (ptetemp & PG_FRAME)); } else { pmap_unwire_pte_hold(dst_pmap, dstmpte); @@ -2119,8 +2126,7 @@ pmap_page_exists(pmap, pa) pmap_t pmap; vm_offset_t pa; { - register pv_entry_t pv; - pv_table_t *ppv; + register pv_entry_t *ppv, pv; int s; if (!pmap_is_managed(pa)) @@ -2132,9 +2138,7 @@ pmap_page_exists(pmap, pa) /* * Not found, check current mappings returning immediately if found. */ - for (pv = TAILQ_FIRST(&ppv->pv_list); - pv; - pv = TAILQ_NEXT(pv, pv_list)) { + for (pv = *ppv; pv; pv = pv->pv_next) { if (pv->pv_pmap == pmap) { splx(s); return TRUE; @@ -2144,72 +2148,149 @@ pmap_page_exists(pmap, pa) return (FALSE); } -#define PMAP_REMOVE_PAGES_CURPROC_ONLY /* - * Remove all pages from specified address space - * this aids process exit speeds. Also, this code - * is special cased for current process only. + * pmap_testbit tests bits in pte's + * note that the testbit/changebit routines are inline, + * and a lot of things compile-time evaluate. */ -void -pmap_remove_pages(pmap, sva, eva) - pmap_t pmap; - vm_offset_t sva, eva; +static __inline boolean_t +pmap_testbit(pa, bit) + register vm_offset_t pa; + int bit; { - unsigned *pte, tpte; - pv_table_t *ppv; - pv_entry_t pv, npv; + register pv_entry_t *ppv, pv; + unsigned *pte; int s; -#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY - if (!curproc || (pmap != &curproc->p_vmspace->vm_pmap)) { - printf("warning: pmap_remove_pages called with non-current pmap\n"); - return; - } -#endif + if (!pmap_is_managed(pa)) + return FALSE; + + ppv = pa_to_pvh(pa); + if (*ppv == NULL) + return FALSE; s = splvm(); + /* + * Not found, check current mappings returning immediately if found. + */ + for (pv = *ppv ;pv; pv = pv->pv_next) { - for(pv = TAILQ_FIRST(&pmap->pm_pvlist.pv_list); - pv; - pv = npv) { + /* + * if the bit being tested is the modified bit, then + * mark clean_map and ptes as never + * modified. + */ + if (bit & (PG_A|PG_M)) { + if (!pmap_track_modified(pv->pv_va)) + continue; + } - if (pv->pv_va >= eva || pv->pv_va < sva) { - npv = TAILQ_NEXT(pv, pv_plist); + if (!pv->pv_pmap) { +#if defined(PMAP_DIAGNOSTIC) + printf("Null pmap (tb) at va: 0x%lx\n", pv->pv_va); +#endif continue; } - -#ifdef PMAP_REMOVE_PAGES_CURPROC_ONLY - pte = (unsigned *)vtopte(pv->pv_va); -#else pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); -#endif - tpte = *pte; - *pte = 0; - - if (tpte) { - pv->pv_pmap->pm_stats.resident_count--; - if (tpte & PG_W) - pv->pv_pmap->pm_stats.wired_count--; - /* - * Update the vm_page_t clean and reference bits. - */ - if (tpte & PG_M) { - PHYS_TO_VM_PAGE(tpte)->dirty = VM_PAGE_BITS_ALL; - } + if (pte == NULL) + continue; + if (*pte & bit) { + splx(s); + return TRUE; } + } + splx(s); + return (FALSE); +} - npv = TAILQ_NEXT(pv, pv_plist); - TAILQ_REMOVE(&pv->pv_pmap->pm_pvlist.pv_list, pv, pv_plist); +/* + * this routine is used to modify bits in ptes + */ +static __inline void +pmap_changebit(pa, bit, setem) + vm_offset_t pa; + int bit; + boolean_t setem; +{ + register pv_entry_t pv, *ppv; + register unsigned *pte; + vm_offset_t va; + int changed; + int s; - ppv = pa_to_pvh(tpte); - --ppv->pv_list_count; - TAILQ_REMOVE(&ppv->pv_list, pv, pv_list); + if (!pmap_is_managed(pa)) + return; - pmap_unuse_pt(pv->pv_pmap, pv->pv_va, pv->pv_ptem); - free_pv_entry(pv); + s = splvm(); + changed = 0; + ppv = pa_to_pvh(pa); + /* + * Loop over all current mappings setting/clearing as appropos If + * setting RO do we need to clear the VAC? + */ + for ( pv = *ppv; pv; pv = pv->pv_next) { + va = pv->pv_va; + + /* + * don't write protect pager mappings + */ + if (!setem && (bit == PG_RW)) { + if (va >= clean_sva && va < clean_eva) + continue; + } + if (!pv->pv_pmap) { +#if defined(PMAP_DIAGNOSTIC) + printf("Null pmap (cb) at va: 0x%lx\n", va); +#endif + continue; + } + + pte = pmap_pte_quick(pv->pv_pmap, va); + if (pte == NULL) + continue; + if (setem) { + *(int *)pte |= bit; + changed = 1; + } else { + vm_offset_t pbits = *(vm_offset_t *)pte; + if (pbits & bit) + changed = 1; + if (bit == PG_RW) { + if (pbits & PG_M) { + vm_page_t m; + vm_offset_t pa = pbits & PG_FRAME; + m = PHYS_TO_VM_PAGE(pa); + m->dirty = VM_PAGE_BITS_ALL; + } + *(int *)pte = pbits & ~(PG_M|PG_RW); + } else { + *(int *)pte = pbits & ~bit; + } + } } - pmap_update(); splx(s); + if (changed) + pmap_update(); +} + +/* + * pmap_page_protect: + * + * Lower the permission for all mappings to a given page. + */ +void +pmap_page_protect(phys, prot) + vm_offset_t phys; + vm_prot_t prot; +{ + if ((prot & VM_PROT_WRITE) == 0) { + if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { + pmap_changebit(phys, PG_RW, FALSE); + } else { + pmap_remove_all(phys); + pmap_update(); + } + } } vm_offset_t @@ -2220,107 +2301,150 @@ pmap_phys_address(ppn) } /* - * pmap_tcbit: + * pmap_is_referenced: + * + * Return whether or not the specified physical page was referenced + * by any physical maps. + */ +boolean_t +pmap_is_referenced(vm_offset_t pa) +{ + register pv_entry_t *ppv, pv, lpv; + unsigned *pte; + int s; + + if (!pmap_is_managed(pa)) + return FALSE; + + ppv = pa_to_pvh(pa); + + s = splvm(); + /* + * Not found, check current mappings returning immediately if found. + */ + for (lpv = NULL, pv = *ppv ;pv; lpv = pv, pv = pv->pv_next) { + /* + * if the bit being tested is the modified bit, then + * mark clean_map and ptes as never + * modified. + */ + if (!pmap_track_modified(pv->pv_va)) + continue; + if (!pv->pv_pmap) { + continue; + } + pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); + if (pte == NULL) + continue; + if ((int) *pte & PG_A) { + if (lpv) { + lpv->pv_next = pv->pv_next; + pv->pv_next = *ppv; + *ppv = pv; + } + splx(s); + return TRUE; + } + } + splx(s); + return (FALSE); +} + +/* + * pmap_ts_referenced: * - * Return the count of bits for a page, clearing all of them. + * Return the count of reference bits for a page, clearing all of them. * */ int -pmap_tcbit(vm_offset_t pa, int bit) +pmap_ts_referenced(vm_offset_t pa) { - register pv_entry_t pv, npv; - pv_table_t *ppv; + register pv_entry_t *ppv, pv; unsigned *pte; int s; int rtval = 0; + vm_offset_t vachanged[VATRACK]; + + if (!pmap_is_managed(pa)) + return FALSE; s = splvm(); ppv = pa_to_pvh(pa); + + if (*ppv == NULL) { + splx(s); + return 0; + } + /* * Not found, check current mappings returning immediately if found. */ - for (pv = TAILQ_FIRST(&ppv->pv_list); - pv; - pv = npv) { - npv = TAILQ_NEXT(pv, pv_list); + for (pv = *ppv ;pv; pv = pv->pv_next) { /* * if the bit being tested is the modified bit, then * mark clean_map and ptes as never * modified. */ - if (((bit & PG_M) != 0) - && !pmap_track_modified(pv->pv_va)) + if (!pmap_track_modified(pv->pv_va)) continue; + if (!pv->pv_pmap) { + continue; + } pte = pmap_pte_quick(pv->pv_pmap, pv->pv_va); if (pte == NULL) continue; - - if ((rtval == 0) && (*pte & bit)) { - rtval = 1; + if (*pte & PG_A) { + if (rtval < VATRACK) + vachanged[rtval] = pv->pv_va; + rtval++; + *pte &= ~PG_A; } - *pte &= ~bit; } splx(s); - if (rtval) - pmap_update(); + if (rtval) { + if (rtval <= VATRACK) { + int i; + for(i=0;i<rtval;i++) + pmap_update_1pg(vachanged[i]); + } else { + pmap_update(); + } + } return (rtval); } /* - * pmap_tc_modified: + * pmap_is_modified: * - * Return the count of modified bits for a page, clearing all of them. - * + * Return whether or not the specified physical page was modified + * in any physical maps. */ -int -pmap_tc_modified(vm_page_t m) +boolean_t +pmap_is_modified(vm_offset_t pa) { - int rtval; - - rtval = pmap_tcbit(VM_PAGE_TO_PHYS(m), PG_M); - if (rtval) - m->dirty = VM_PAGE_BITS_ALL; - - return rtval; + return pmap_testbit((pa), PG_M); } /* - * pmap_tc_referenced: - * - * Return the count of referenced bits for a page, clearing all of them. - * + * Clear the modify bits on the specified physical page. */ -int -pmap_tc_referenced(vm_offset_t pa) +void +pmap_clear_modify(vm_offset_t pa) { - if (!pmap_is_managed(pa)) - return 0; - return pmap_tcbit(pa, PG_A); + pmap_changebit((pa), PG_M, FALSE); } /* - * pmap_page_protect: + * pmap_clear_reference: * - * Lower the permission for all mappings to a given page. + * Clear the reference bit on the specified physical page. */ void -pmap_page_protect(m, prot) - vm_page_t m; - vm_prot_t prot; +pmap_clear_reference(vm_offset_t pa) { - if ((prot & VM_PROT_WRITE) == 0) { - if (prot & (VM_PROT_READ | VM_PROT_EXECUTE)) { - if ((m->flags & PG_FICTITIOUS) == 0) - pmap_tcbit(VM_PAGE_TO_PHYS(m), PG_RW); - } else { - if (pmap_remove_all(VM_PAGE_TO_PHYS(m))) { - m->dirty = VM_PAGE_BITS_ALL; - } - pmap_update(); - } - } + pmap_changebit((pa), PG_A, FALSE); } /* @@ -2401,7 +2525,7 @@ pmap_mincore(pmap, addr) unsigned *ptep, pte; int val = 0; - ptep = pmap_pte_quick(pmap, addr); + ptep = pmap_pte(pmap, addr); if (ptep == 0) { return 0; } @@ -2420,10 +2544,8 @@ pmap_mincore(pmap, addr) * Modified by someone */ else if (PHYS_TO_VM_PAGE(pa)->dirty || - pmap_tcbit(pa, PG_M)) { + pmap_is_modified(pa)) val |= MINCORE_MODIFIED_OTHER; - PHYS_TO_VM_PAGE(pa)->dirty = VM_PAGE_BITS_ALL; - } /* * Referenced by us */ @@ -2434,7 +2556,7 @@ pmap_mincore(pmap, addr) * Referenced by someone */ else if ((PHYS_TO_VM_PAGE(pa)->flags & PG_REFERENCED) || - pmap_tcbit(pa, PG_A)) + pmap_is_referenced(pa)) val |= MINCORE_REFERENCED_OTHER; } return val; @@ -2470,7 +2592,7 @@ pmap_pid_dump(int pid) { } return npte; } - pte = pmap_pte_quick( pmap, va); + pte = pmap_pte( pmap, va); if (pte && pmap_pte_v(pte)) { vm_offset_t pa; vm_page_t m; @@ -2519,7 +2641,7 @@ pads(pm) continue; if (pm != kernel_pmap && va > UPT_MAX_ADDRESS) continue; - ptep = pmap_pte_quick(pm, va); + ptep = pmap_pte(pm, va); if (pmap_pte_v(ptep)) printf("%x:%x ", va, *(int *) ptep); }; @@ -2533,9 +2655,7 @@ pmap_pvdump(pa) register pv_entry_t pv; printf("pa %x", pa); - for (pv = TAILQ_FIRST(pa_to_pvh(pa)); - pv; - pv = TAILQ_NEXT(pv, pv_list)) { + for (pv = pa_to_pvh(pa); pv; pv = pv->pv_next) { #ifdef used_to_be printf(" -> pmap %x, va %x, flags %x", pv->pv_pmap, pv->pv_va, pv->pv_flags); diff --git a/sys/i386/include/pmap.h b/sys/i386/include/pmap.h index b53db62..64b682c 100644 --- a/sys/i386/include/pmap.h +++ b/sys/i386/include/pmap.h @@ -42,13 +42,12 @@ * * from: hp300: @(#)pmap.h 7.2 (Berkeley) 12/16/90 * from: @(#)pmap.h 7.4 (Berkeley) 5/12/91 - * $Id: pmap.h,v 1.40 1996/06/08 11:21:19 bde Exp $ + * $Id: pmap.h,v 1.41 1996/07/27 03:23:32 dyson Exp $ */ #ifndef _MACHINE_PMAP_H_ #define _MACHINE_PMAP_H_ - /* * Page-directory and page-table entires follow this format, with a few * of the fields not present here and there, depending on a lot of things. @@ -114,9 +113,6 @@ #define ISA_HOLE_LENGTH (0x100000-ISA_HOLE_START) #ifndef LOCORE - -#include <sys/queue.h> - typedef unsigned int *pd_entry_t; typedef unsigned int *pt_entry_t; @@ -162,24 +158,17 @@ pmap_kextract(vm_offset_t va) } #endif -struct vm_page; - /* * Pmap stuff */ -struct pv_entry; -typedef struct { - int pv_list_count; - TAILQ_HEAD(,pv_entry) pv_list; -} pv_table_t; struct pmap { pd_entry_t *pm_pdir; /* KVA of page directory */ vm_object_t pm_pteobj; /* Container for pte's */ - pv_table_t pm_pvlist; /* list of mappings in pmap */ - int pm_count; /* reference count */ + short pm_dref; /* page directory ref count */ + short pm_count; /* pmap reference count */ struct pmap_statistics pm_stats; /* pmap statistics */ - struct vm_page *pm_ptphint; /* pmap ptp hint */ + struct vm_map *pm_map; /* map that owns this pmap */ }; typedef struct pmap *pmap_t; @@ -188,16 +177,14 @@ typedef struct pmap *pmap_t; extern pmap_t kernel_pmap; #endif - /* * For each vm_page_t, there is a list of all currently valid virtual * mappings of that page. An entry is a pv_entry_t, the list is pv_table. */ typedef struct pv_entry { + struct pv_entry *pv_next; /* next pv_entry */ pmap_t pv_pmap; /* pmap where mapping lies */ vm_offset_t pv_va; /* virtual address for mapping */ - TAILQ_ENTRY(pv_entry) pv_list; - TAILQ_ENTRY(pv_entry) pv_plist; vm_page_t pv_ptem; /* VM page for pte */ } *pv_entry_t; @@ -213,7 +200,7 @@ extern pt_entry_t *CMAP1; extern vm_offset_t avail_end; extern vm_offset_t avail_start; extern vm_offset_t phys_avail[]; -pv_table_t *pv_table; +extern pv_entry_t *pv_table; /* array of entries, one per page */ extern vm_offset_t virtual_avail; extern vm_offset_t virtual_end; diff --git a/sys/kern/kern_exec.c b/sys/kern/kern_exec.c index 7d3a0eb..0fabc69 100644 --- a/sys/kern/kern_exec.c +++ b/sys/kern/kern_exec.c @@ -23,7 +23,7 @@ * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF * SUCH DAMAGE. * - * $Id: kern_exec.c,v 1.44 1996/07/12 04:11:37 bde Exp $ + * $Id: kern_exec.c,v 1.45 1996/07/27 03:23:41 dyson Exp $ */ #include <sys/param.h> @@ -367,7 +367,7 @@ exec_new_vmspace(imgp) /* Blow away entire process VM */ if (vmspace->vm_shm) shmexit(imgp->proc); - vm_map_remove_userspace(&vmspace->vm_map); + vm_map_remove(&vmspace->vm_map, 0, USRSTACK); /* Allocate a new stack */ error = vm_map_find(&vmspace->vm_map, NULL, 0, (vm_offset_t *)&stack_addr, diff --git a/sys/kern/kern_exit.c b/sys/kern/kern_exit.c index 86398d7..0746e58 100644 --- a/sys/kern/kern_exit.c +++ b/sys/kern/kern_exit.c @@ -36,7 +36,7 @@ * SUCH DAMAGE. * * @(#)kern_exit.c 8.7 (Berkeley) 2/12/94 - * $Id: kern_exit.c,v 1.33 1996/06/12 05:07:28 gpalmer Exp $ + * $Id: kern_exit.c,v 1.34 1996/07/27 03:23:42 dyson Exp $ */ #include "opt_ktrace.h" @@ -156,7 +156,8 @@ exit1(p, rv) * may be mapped within that space also. */ if (vm->vm_refcnt == 1) - vm_map_remove_userspace(&vm->vm_map); + (void) vm_map_remove(&vm->vm_map, VM_MIN_ADDRESS, + VM_MAXUSER_ADDRESS); if (SESS_LEADER(p)) { register struct session *sp = p->p_session; diff --git a/sys/vm/pmap.h b/sys/vm/pmap.h index cc5f6f0..b8eb6fc 100644 --- a/sys/vm/pmap.h +++ b/sys/vm/pmap.h @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: pmap.h,v 1.11 1996/06/17 03:35:34 dyson Exp $ + * $Id: pmap.h,v 1.13 1996/07/27 04:22:12 dyson Exp $ */ /* @@ -99,8 +99,9 @@ void pmap_enter __P((pmap_t, vm_offset_t, vm_offset_t, vm_prot_t, vm_offset_t pmap_extract __P((pmap_t, vm_offset_t)); void pmap_growkernel __P((vm_offset_t)); void pmap_init __P((vm_offset_t, vm_offset_t)); -int pmap_tc_modified __P((vm_page_t m)); -int pmap_tc_referenced __P((vm_offset_t pa)); +boolean_t pmap_is_modified __P((vm_offset_t pa)); +boolean_t pmap_is_referenced __P((vm_offset_t pa)); +boolean_t pmap_ts_referenced __P((vm_offset_t pa)); void pmap_kenter __P((vm_offset_t, vm_offset_t)); void pmap_kremove __P((vm_offset_t)); vm_offset_t pmap_map __P((vm_offset_t, vm_offset_t, vm_offset_t, int)); @@ -108,7 +109,7 @@ void pmap_object_init_pt __P((pmap_t pmap, vm_offset_t addr, vm_object_t object, vm_pindex_t pindex, vm_offset_t size, int pagelimit)); boolean_t pmap_page_exists __P((pmap_t, vm_offset_t)); -void pmap_page_protect __P((vm_page_t, vm_prot_t)); +void pmap_page_protect __P((vm_offset_t, vm_prot_t)); void pmap_pageable __P((pmap_t, vm_offset_t, vm_offset_t, boolean_t)); vm_offset_t pmap_phys_address __P((int)); @@ -120,7 +121,6 @@ void pmap_qremove __P((vm_offset_t, int)); void pmap_reference __P((pmap_t)); void pmap_release __P((pmap_t)); void pmap_remove __P((pmap_t, vm_offset_t, vm_offset_t)); -void pmap_remove_pages __P((pmap_t, vm_offset_t, vm_offset_t)); void pmap_zero_page __P((vm_offset_t)); void pmap_prefault __P((pmap_t pmap, vm_offset_t addra, vm_map_entry_t entry, vm_object_t object)); diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c index b68195c..b378cd2 100644 --- a/sys/vm/swap_pager.c +++ b/sys/vm/swap_pager.c @@ -39,7 +39,7 @@ * from: Utah $Hdr: swap_pager.c 1.4 91/04/30$ * * @(#)swap_pager.c 8.9 (Berkeley) 3/21/94 - * $Id: swap_pager.c,v 1.68 1996/06/10 04:58:48 dyson Exp $ + * $Id: swap_pager.c,v 1.69 1996/07/27 03:23:51 dyson Exp $ */ /* @@ -1078,7 +1078,7 @@ swap_pager_getpages(object, m, count, reqpage) pagedaemon_wakeup(); swap_pager_needflags &= ~(SWAP_FREE_NEEDED|SWAP_FREE_NEEDED_BY_PAGEOUT); if (rv == VM_PAGER_OK) { - pmap_tc_modified(m[reqpage]); + pmap_clear_modify(VM_PAGE_TO_PHYS(m[reqpage])); m[reqpage]->valid = VM_PAGE_BITS_ALL; m[reqpage]->dirty = 0; } @@ -1092,7 +1092,7 @@ swap_pager_getpages(object, m, count, reqpage) */ if (rv == VM_PAGER_OK) { for (i = 0; i < count; i++) { - pmap_tc_modified(m[i]); + pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); m[i]->dirty = 0; m[i]->flags &= ~PG_ZERO; if (i != reqpage) { @@ -1469,7 +1469,7 @@ retryfree: if (rv == VM_PAGER_OK) { for (i = 0; i < count; i++) { if (rtvals[i] == VM_PAGER_OK) { - pmap_tc_modified(m[i]); + pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); m[i]->dirty = 0; /* * optimization, if a page has been read @@ -1477,7 +1477,7 @@ retryfree: */ if ((m[i]->queue != PQ_ACTIVE) && ((m[i]->flags & (PG_WANTED|PG_REFERENCED)) || - pmap_tc_referenced(VM_PAGE_TO_PHYS(m[i])))) { + pmap_is_referenced(VM_PAGE_TO_PHYS(m[i])))) { vm_page_activate(m[i]); } } @@ -1580,21 +1580,12 @@ swap_pager_finish(spc) (u_long) VM_PAGE_TO_PHYS(spc->spc_m[i])); } } else { - int pagewanted = 0; for (i = 0; i < spc->spc_count; i++) { - if (spc->spc_m[i]->flags & (PG_WANTED | PG_REFERENCED)) { - pagewanted = 1; - break; - } - } - for (i = 0; i < spc->spc_count; i++) { - pmap_tc_modified(spc->spc_m[i]); + pmap_clear_modify(VM_PAGE_TO_PHYS(spc->spc_m[i])); spc->spc_m[i]->dirty = 0; - if (pagewanted) { - if (spc->spc_m[i]->queue != PQ_ACTIVE) - vm_page_activate(spc->spc_m[i]); - spc->spc_m[i]->flags |= PG_REFERENCED; - } + if ((spc->spc_m[i]->queue != PQ_ACTIVE) && + ((spc->spc_m[i]->flags & PG_WANTED) || pmap_is_referenced(VM_PAGE_TO_PHYS(spc->spc_m[i])))) + vm_page_activate(spc->spc_m[i]); } } @@ -1634,7 +1625,9 @@ swap_pager_iodone(bp) if (bp->b_vp) pbrelvp(bp); +/* if (bp->b_flags & B_WANTED) +*/ wakeup(bp); if (bp->b_rcred != NOCRED) diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 66965d4..43b4158 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -66,7 +66,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_fault.c,v 1.54 1996/07/27 03:23:52 dyson Exp $ + * $Id: vm_fault.c,v 1.55 1996/07/28 01:14:01 dyson Exp $ */ /* @@ -103,6 +103,10 @@ int vm_fault_additional_pages __P((vm_page_t, int, int, vm_page_t *, int *)); #define VM_FAULT_READ_BEHIND 3 #define VM_FAULT_READ (VM_FAULT_READ_AHEAD+VM_FAULT_READ_BEHIND+1) +int vm_fault_free_1; +int vm_fault_copy_save_1; +int vm_fault_copy_save_2; + /* * vm_fault: * @@ -278,7 +282,7 @@ RetryFault:; } queue = m->queue; - vm_page_unqueue(m,0); + vm_page_unqueue_nowakeup(m); /* * Mark page busy for other processes, and the pagedaemon. @@ -561,6 +565,7 @@ readrest: first_m = m; m->dirty = VM_PAGE_BITS_ALL; m = NULL; + ++vm_fault_copy_save_1; } else { /* * Oh, well, lets copy it. @@ -634,6 +639,7 @@ readrest: PAGE_WAKEUP(m); vm_page_free(m); m = NULL; + ++vm_fault_free_1; tm->dirty = VM_PAGE_BITS_ALL; first_m->dirty = VM_PAGE_BITS_ALL; } @@ -645,6 +651,7 @@ readrest: vm_page_rename(m, other_object, other_pindex); m->dirty = VM_PAGE_BITS_ALL; m->valid = VM_PAGE_BITS_ALL; + ++vm_fault_copy_save_2; } } } @@ -653,9 +660,9 @@ readrest: if (m) { if (m->queue != PQ_ACTIVE) vm_page_activate(m); - /* - * We no longer need the old page or object. - */ + /* + * We no longer need the old page or object. + */ PAGE_WAKEUP(m); } @@ -1084,7 +1091,7 @@ vm_fault_additional_pages(m, rbehind, rahead, marray, reqpage) endpindex = pindex + (rahead + 1); if (endpindex > object->size) endpindex = object->size; - while (tpindex < endpindex) { + while (tpindex < endpindex) { if ( vm_page_lookup(object, tpindex)) { break; } diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index 31455b4..0c5996a 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_map.c,v 1.52 1996/07/07 03:27:41 davidg Exp $ + * $Id: vm_map.c,v 1.53 1996/07/27 03:23:56 dyson Exp $ */ /* @@ -172,8 +172,6 @@ static void vm_map_entry_unwire __P((vm_map_t, vm_map_entry_t)); static void vm_map_copy_entry __P((vm_map_t, vm_map_t, vm_map_entry_t, vm_map_entry_t)); static void vm_map_simplify_entry __P((vm_map_t, vm_map_entry_t)); -static __pure int vm_map_simplify_okay __P((vm_map_entry_t entry1, - vm_map_entry_t entry2)); void vm_map_startup() @@ -232,6 +230,7 @@ vmspace_alloc(min, max, pageable) vm_map_init(&vm->vm_map, min, max, pageable); pmap_pinit(&vm->vm_pmap); vm->vm_map.pmap = &vm->vm_pmap; /* XXX */ + vm->vm_pmap.pm_map = &vm->vm_map; vm->vm_refcnt = 1; return (vm); } @@ -635,8 +634,8 @@ vm_map_insert(map, object, offset, start, end, prot, max, cow) return (KERN_NO_SPACE); if ((prev_entry != &map->header) && - (object == NULL) && (prev_entry->end == start) && + ((object == NULL) || (prev_entry->object.vm_object == object)) && (prev_entry->is_a_map == FALSE) && (prev_entry->is_sub_map == FALSE) && (prev_entry->inheritance == VM_INHERIT_DEFAULT) && @@ -649,22 +648,24 @@ vm_map_insert(map, object, offset, start, end, prot, max, cow) * See if we can avoid creating a new entry by extending one of our * neighbors. */ - if (vm_object_coalesce(prev_entry->object.vm_object, - OFF_TO_IDX(prev_entry->offset), - (vm_size_t) (prev_entry->end - - prev_entry->start), - (vm_size_t) (end - prev_entry->end))) { + if (object == NULL) { + if (vm_object_coalesce(prev_entry->object.vm_object, + OFF_TO_IDX(prev_entry->offset), + (vm_size_t) (prev_entry->end + - prev_entry->start), + (vm_size_t) (end - prev_entry->end))) { - /* - * Coalesced the two objects - can extend the - * previous map entry to include the new - * range. - */ - map->size += (end - prev_entry->end); - prev_entry->end = end; - prev_object = prev_entry->object.vm_object; - default_pager_convert_to_swapq(prev_object); - return (KERN_SUCCESS); + /* + * Coalesced the two objects - can extend the + * previous map entry to include the new + * range. + */ + map->size += (end - prev_entry->end); + prev_entry->end = end; + prev_object = prev_entry->object.vm_object; + default_pager_convert_to_swapq(prev_object); + return (KERN_SUCCESS); + } } } /* @@ -706,10 +707,9 @@ vm_map_insert(map, object, offset, start, end, prot, max, cow) /* * Update the free space hint */ - if (map->first_free == prev_entry) { - if (prev_entry->end == new_entry->start) - map->first_free = new_entry; - } + if ((map->first_free == prev_entry) && + (prev_entry->end >= new_entry->start)) + map->first_free = new_entry; default_pager_convert_to_swapq(object); return (KERN_SUCCESS); @@ -739,9 +739,8 @@ vm_map_findspace(map, start, length, addr) * at this address, we have to start after it. */ if (start == map->min_offset) { - if ((entry = map->first_free) != &map->header) { + if ((entry = map->first_free) != &map->header) start = entry->end; - } } else { vm_map_entry_t tmp; @@ -822,39 +821,12 @@ vm_map_find(map, object, offset, addr, length, find_space, prot, max, cow) return (result); } -static __pure int -vm_map_simplify_okay(entry1, entry2) - vm_map_entry_t entry1, entry2; -{ - if ((entry1->end != entry2->start) || - (entry1->object.vm_object != entry2->object.vm_object)) - return 0; - if (entry1->object.vm_object) { - if (entry1->object.vm_object->behavior != - entry2->object.vm_object->behavior) - return 0; - if (entry1->offset + (entry1->end - entry1->start) != - entry2->offset) - return 0; - } - if ((entry1->needs_copy != entry2->needs_copy) || - (entry1->copy_on_write != entry2->copy_on_write) || - (entry1->protection != entry2->protection) || - (entry1->max_protection != entry2->max_protection) || - (entry1->inheritance != entry2->inheritance) || - (entry1->is_sub_map != FALSE) || - (entry1->is_a_map != FALSE) || - (entry1->wired_count != 0) || - (entry2->is_sub_map != FALSE) || - (entry2->is_a_map != FALSE) || - (entry2->wired_count != 0)) - return 0; - - return 1; -} - /* * vm_map_simplify_entry: [ internal use only ] + * + * Simplify the given map entry by: + * removing extra sharing maps + * [XXX maybe later] merging with a neighbor */ static void vm_map_simplify_entry(map, entry) @@ -862,13 +834,34 @@ vm_map_simplify_entry(map, entry) vm_map_entry_t entry; { vm_map_entry_t next, prev; + vm_size_t nextsize, prevsize, esize; - if (entry->is_a_map || entry->is_sub_map || entry->wired_count) + /* + * If this entry corresponds to a sharing map, then see if we can + * remove the level of indirection. If it's not a sharing map, then it + * points to a VM object, so see if we can merge with either of our + * neighbors. + */ + + if (entry->is_sub_map || entry->is_a_map || entry->wired_count) return; prev = entry->prev; if (prev != &map->header) { - if ( vm_map_simplify_okay(prev, entry)) { + prevsize = prev->end - prev->start; + if ( (prev->end == entry->start) && + (prev->object.vm_object == entry->object.vm_object) && + (!prev->object.vm_object || (prev->object.vm_object->behavior == entry->object.vm_object->behavior)) && + (!prev->object.vm_object || + (prev->offset + prevsize == entry->offset)) && + (prev->needs_copy == entry->needs_copy) && + (prev->copy_on_write == entry->copy_on_write) && + (prev->protection == entry->protection) && + (prev->max_protection == entry->max_protection) && + (prev->inheritance == entry->inheritance) && + (prev->is_a_map == FALSE) && + (prev->is_sub_map == FALSE) && + (prev->wired_count == 0)) { if (map->first_free == prev) map->first_free = entry; if (map->hint == prev) @@ -884,7 +877,21 @@ vm_map_simplify_entry(map, entry) next = entry->next; if (next != &map->header) { - if ( vm_map_simplify_okay(entry, next)) { + nextsize = next->end - next->start; + esize = entry->end - entry->start; + if ((entry->end == next->start) && + (next->object.vm_object == entry->object.vm_object) && + (!next->object.vm_object || (next->object.vm_object->behavior == entry->object.vm_object->behavior)) && + (!entry->object.vm_object || + (entry->offset + esize == next->offset)) && + (next->needs_copy == entry->needs_copy) && + (next->copy_on_write == entry->copy_on_write) && + (next->protection == entry->protection) && + (next->max_protection == entry->max_protection) && + (next->inheritance == entry->inheritance) && + (next->is_a_map == FALSE) && + (next->is_sub_map == FALSE) && + (next->wired_count == 0)) { if (map->first_free == next) map->first_free = entry; if (map->hint == next) @@ -897,7 +904,6 @@ vm_map_simplify_entry(map, entry) } } } - /* * vm_map_clip_start: [ internal use only ] * @@ -1835,21 +1841,6 @@ vm_map_remove(map, start, end) } /* - * vm_map_remove_userspace: - * Removes the user portion of the address space. - */ -void -vm_map_remove_userspace(map) - register vm_map_t map; -{ - vm_map_lock(map); - pmap_remove_pages(map->pmap, VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS); - vm_map_delete(map, VM_MIN_ADDRESS, VM_MAXUSER_ADDRESS); - vm_map_unlock(map); - return; -} - -/* * vm_map_check_protection: * * Assert that the target map allows the specified @@ -2266,8 +2257,8 @@ RetryLookup:; lock_write_to_read(&share_map->lock); } - default_pager_convert_to_swapq(entry->object.vm_object); - + if (entry->object.vm_object != NULL) + default_pager_convert_to_swapq(entry->object.vm_object); /* * Return the object/offset from this entry. If the entry was * copy-on-write or empty, it has been fixed up. diff --git a/sys/vm/vm_map.h b/sys/vm/vm_map.h index a50fa62..fd04d87 100644 --- a/sys/vm/vm_map.h +++ b/sys/vm/vm_map.h @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_map.h,v 1.13 1996/05/19 07:36:48 dyson Exp $ + * $Id: vm_map.h,v 1.14 1996/07/27 03:23:59 dyson Exp $ */ /* @@ -233,7 +233,6 @@ int vm_map_clean __P((vm_map_t, vm_offset_t, vm_offset_t, boolean_t, boolean_t)) int vm_map_protect __P((vm_map_t, vm_offset_t, vm_offset_t, vm_prot_t, boolean_t)); void vm_map_reference __P((vm_map_t)); int vm_map_remove __P((vm_map_t, vm_offset_t, vm_offset_t)); -void vm_map_remove_userspace __P((vm_map_t)); void vm_map_simplify __P((vm_map_t, vm_offset_t)); void vm_map_startup __P((void)); int vm_map_submap __P((vm_map_t, vm_offset_t, vm_offset_t, vm_map_t)); diff --git a/sys/vm/vm_mmap.c b/sys/vm/vm_mmap.c index a1a48a9..047b39e 100644 --- a/sys/vm/vm_mmap.c +++ b/sys/vm/vm_mmap.c @@ -38,7 +38,7 @@ * from: Utah $Hdr: vm_mmap.c 1.6 91/10/21$ * * @(#)vm_mmap.c 8.4 (Berkeley) 1/12/94 - * $Id: vm_mmap.c,v 1.47 1996/07/27 17:21:41 dyson Exp $ + * $Id: vm_mmap.c,v 1.48 1996/07/28 02:54:09 davidg Exp $ */ /* @@ -72,7 +72,6 @@ #include <vm/vm_extern.h> #include <vm/vm_kern.h> #include <vm/vm_page.h> -#include <vm/default_pager.h> #ifndef _SYS_SYSPROTO_H_ struct sbrk_args { @@ -690,10 +689,10 @@ mincore(p, uap, retval) if (m) { mincoreinfo = MINCORE_INCORE; if (m->dirty || - pmap_tc_modified(m)) + pmap_is_modified(VM_PAGE_TO_PHYS(m))) mincoreinfo |= MINCORE_MODIFIED_OTHER; if ((m->flags & PG_REFERENCED) || - pmap_tc_referenced(VM_PAGE_TO_PHYS(m))) + pmap_is_referenced(VM_PAGE_TO_PHYS(m))) mincoreinfo |= MINCORE_REFERENCED_OTHER; } } @@ -845,7 +844,7 @@ vm_mmap(map, addr, size, prot, maxprot, flags, handle, foff) vm_ooffset_t foff; { boolean_t fitit; - vm_object_t object; + vm_object_t object, object2; struct vnode *vp = NULL; objtype_t type; int rv = KERN_SUCCESS; @@ -883,15 +882,12 @@ vm_mmap(map, addr, size, prot, maxprot, flags, handle, foff) * Lookup/allocate object. */ if (flags & MAP_ANON) { + type = OBJT_SWAP; /* * Unnamed anonymous regions always start at 0. */ - if (handle == 0) { + if (handle == 0) foff = 0; - type = OBJT_DEFAULT; - } else { - type = OBJT_SWAP; - } } else { vp = (struct vnode *) handle; if (vp->v_type == VCHR) { @@ -908,15 +904,9 @@ vm_mmap(map, addr, size, prot, maxprot, flags, handle, foff) type = OBJT_VNODE; } } - - if (type != OBJT_DEFAULT) { - object = vm_pager_allocate(type, handle, - OFF_TO_IDX(objsize), prot, foff); - if (object == NULL) - return (type == OBJT_DEVICE ? EINVAL : ENOMEM); - } else { - object = NULL; - } + object = vm_pager_allocate(type, handle, OFF_TO_IDX(objsize), prot, foff); + if (object == NULL) + return (type == OBJT_DEVICE ? EINVAL : ENOMEM); /* * Force device mappings to be shared. @@ -926,13 +916,30 @@ vm_mmap(map, addr, size, prot, maxprot, flags, handle, foff) flags |= MAP_SHARED; } + object2 = NULL; docow = 0; if ((flags & (MAP_ANON|MAP_SHARED)) == 0) { - docow = MAP_COPY_ON_WRITE|MAP_COPY_NEEDED; + docow = MAP_COPY_ON_WRITE; + if (objsize < size) { + object2 = vm_object_allocate( OBJT_DEFAULT, + OFF_TO_IDX(size - (foff & ~PAGE_MASK))); + object2->backing_object = object; + object2->backing_object_offset = foff; + TAILQ_INSERT_TAIL(&object->shadow_head, + object2, shadow_list); + ++object->shadow_count; + } else { + docow |= MAP_COPY_NEEDED; + } } - rv = vm_map_find(map, object, foff, addr, size, fitit, - prot, maxprot, docow); + if (object2) + rv = vm_map_find(map, object2, 0, addr, size, fitit, + prot, maxprot, docow); + else + rv = vm_map_find(map, object, foff, addr, size, fitit, + prot, maxprot, docow); + if (rv != KERN_SUCCESS) { /* @@ -940,7 +947,10 @@ vm_mmap(map, addr, size, prot, maxprot, flags, handle, foff) * object if it's an unnamed anonymous mapping * or named anonymous without other references. */ - vm_object_deallocate(object); + if (object2) + vm_object_deallocate(object2); + else + vm_object_deallocate(object); goto out; } @@ -975,176 +985,3 @@ out: return (EINVAL); } } - -#ifdef notyet -/* - * Efficient mapping of a .text+.data+.bss object - */ -int -vm_mapaout(map, baseaddr, vp, foff, textsize, datasize, bsssize, addr) - vm_map_t map; - vm_offset_t baseaddr; - struct vnode *vp; - vm_ooffset_t foff; - register vm_size_t textsize, datasize, bsssize; - vm_offset_t *addr; -{ - vm_object_t object; - int rv; - vm_pindex_t objpsize; - struct proc *p = curproc; - - vm_size_t totalsize; - vm_size_t textend; - struct vattr vat; - int error; - - textsize = round_page(textsize); - datasize = round_page(datasize); - bsssize = round_page(bsssize); - totalsize = textsize + datasize + bsssize; - - vm_map_lock(map); - /* - * If baseaddr == -1, then we need to search for space. Otherwise, - * we need to be loaded into a certain spot. - */ - if (baseaddr != (vm_offset_t) -1) { - if (vm_map_findspace(map, baseaddr, totalsize, addr)) { - goto outnomem; - } - - if(*addr != baseaddr) { - goto outnomem; - } - } else { - baseaddr = round_page(p->p_vmspace->vm_daddr + MAXDSIZ); - if (vm_map_findspace(map, baseaddr, totalsize, addr)) { - goto outnomem; - } - } - - if (foff & PAGE_MASK) { - vm_map_unlock(map); - return EINVAL; - } - - if ((vp->v_object != 0) && - ((((vm_object_t)vp->v_object)->flags & OBJ_DEAD) == 0)) { - object = vp->v_object; - vm_object_reference(object); - } else { - /* - * get the object size to allocate - */ - error = VOP_GETATTR(vp, &vat, p->p_ucred, p); - if (error) { - vm_map_unlock(map); - return error; - } - objpsize = OFF_TO_IDX(round_page(vat.va_size)); - /* - * Alloc/reference the object - */ - object = vm_pager_allocate(OBJT_VNODE, vp, - objpsize, VM_PROT_ALL, foff); - if (object == NULL) { - goto outnomem; - } - } - - /* - * Insert .text into the map - */ - textend = *addr + textsize; - rv = vm_map_insert(map, object, foff, - *addr, textend, - VM_PROT_READ|VM_PROT_EXECUTE, VM_PROT_ALL, - MAP_COPY_ON_WRITE|MAP_COPY_NEEDED); - if (rv != KERN_SUCCESS) { - vm_object_deallocate(object); - goto out; - } - - /* - * Insert .data into the map, if there is any to map. - */ - if (datasize != 0) { - object->ref_count++; - rv = vm_map_insert(map, object, foff + textsize, - textend, textend + datasize, - VM_PROT_ALL, VM_PROT_ALL, - MAP_COPY_ON_WRITE|MAP_COPY_NEEDED); - if (rv != KERN_SUCCESS) { - --object->ref_count; - vm_map_delete(map, *addr, textend); - goto out; - } - } - - /* - * Preload the page tables - */ - pmap_object_init_pt(map->pmap, *addr, - object, (vm_pindex_t) OFF_TO_IDX(foff), - textsize + datasize, 1); - - /* - * Get the space for bss. - */ - if (bsssize != 0) { - rv = vm_map_insert(map, NULL, 0, - textend + datasize, - *addr + totalsize, - VM_PROT_ALL, VM_PROT_ALL, 0); - } - if (rv != KERN_SUCCESS) { - vm_map_delete(map, *addr, textend + datasize + bsssize); - } - -out: - vm_map_unlock(map); - switch (rv) { - case KERN_SUCCESS: - return 0; - case KERN_INVALID_ADDRESS: - case KERN_NO_SPACE: - return ENOMEM; - case KERN_PROTECTION_FAILURE: - return EACCES; - default: - return EINVAL; - } -outnomem: - vm_map_unlock(map); - return ENOMEM; -} - - -int -mapaout(struct proc *p, struct mapaout_args *uap, int *retval) -{ - - register struct filedesc *fdp = p->p_fd; - struct file *fp; - struct vnode *vp; - int rtval; - - if (((unsigned) uap->fd) >= fdp->fd_nfiles || - (fp = fdp->fd_ofiles[uap->fd]) == NULL) - return (EBADF); - if (fp->f_type != DTYPE_VNODE) - return (EINVAL); - - vp = (struct vnode *) fp->f_data; - if ((vp->v_type != VREG) && (vp->v_type != VCHR)) - return (EINVAL); - - rtval = vm_mapaout( &p->p_vmspace->vm_map, - uap->addr, vp, uap->offset, - uap->textsize, uap->datasize, uap->bsssize, - (vm_offset_t *)retval); - - return rtval; -} -#endif diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 39c7ee0..496da96 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_object.c,v 1.76 1996/06/16 20:37:30 dyson Exp $ + * $Id: vm_object.c,v 1.77 1996/07/27 03:24:03 dyson Exp $ */ /* @@ -219,6 +219,7 @@ vm_object_allocate(type, size) result = (vm_object_t) malloc((u_long) sizeof *result, M_VMOBJ, M_WAITOK); + _vm_object_allocate(type, size, result); return (result); @@ -230,7 +231,7 @@ vm_object_allocate(type, size) * * Gets another reference to the given object. */ -void +inline void vm_object_reference(object) register vm_object_t object; { @@ -402,10 +403,8 @@ vm_object_terminate(object) * from paging queues. */ while ((p = TAILQ_FIRST(&object->memq)) != NULL) { -#if defined(DIAGNOSTIC) if (p->flags & PG_BUSY) printf("vm_object_terminate: freeing busy page\n"); -#endif PAGE_WAKEUP(p); vm_page_free(p); cnt.v_pfree++; diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c index 79dd930..4857d45 100644 --- a/sys/vm/vm_page.c +++ b/sys/vm/vm_page.c @@ -34,7 +34,7 @@ * SUCH DAMAGE. * * from: @(#)vm_page.c 7.4 (Berkeley) 5/7/91 - * $Id: vm_page.c,v 1.60 1996/06/26 05:39:25 dyson Exp $ + * $Id: vm_page.c,v 1.61 1996/07/27 03:24:05 dyson Exp $ */ /* @@ -385,7 +385,7 @@ vm_page_hash(object, pindex) * The object and page must be locked, and must be splhigh. */ -void +__inline void vm_page_insert(m, object, pindex) register vm_page_t m; register vm_object_t object; @@ -434,7 +434,7 @@ vm_page_insert(m, object, pindex) * The object and page must be locked, and at splhigh. */ -void +__inline void vm_page_remove(m) register vm_page_t m; { @@ -523,19 +523,34 @@ vm_page_rename(m, new_object, new_pindex) } /* + * vm_page_unqueue without any wakeup + */ +__inline void +vm_page_unqueue_nowakeup(m) + vm_page_t m; +{ + int queue = m->queue; + if (queue != PQ_NONE) { + m->queue = PQ_NONE; + TAILQ_REMOVE(vm_page_queues[queue].pl, m, pageq); + --(*vm_page_queues[queue].cnt); + } +} + + +/* * vm_page_unqueue must be called at splhigh(); */ __inline void -vm_page_unqueue(m, wakeup) +vm_page_unqueue(m) vm_page_t m; - int wakeup; { int queue = m->queue; if (queue != PQ_NONE) { m->queue = PQ_NONE; TAILQ_REMOVE(vm_page_queues[queue].pl, m, pageq); --(*vm_page_queues[queue].cnt); - if ((queue == PQ_CACHE) && wakeup) { + if (queue == PQ_CACHE) { if ((cnt.v_cache_count + cnt.v_free_count) < (cnt.v_free_reserved + cnt.v_cache_min)) pagedaemon_wakeup(); @@ -721,7 +736,7 @@ vm_page_activate(m) if (m->queue == PQ_CACHE) cnt.v_reactivated++; - vm_page_unqueue(m, 1); + vm_page_unqueue(m); if (m->wire_count == 0) { TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq); @@ -736,7 +751,7 @@ vm_page_activate(m) /* * helper routine for vm_page_free and vm_page_free_zero */ -__inline static int +static int vm_page_freechk_and_unqueue(m) vm_page_t m; { @@ -754,7 +769,7 @@ vm_page_freechk_and_unqueue(m) } vm_page_remove(m); - vm_page_unqueue(m,0); + vm_page_unqueue_nowakeup(m); if ((m->flags & PG_FICTITIOUS) != 0) { return 0; } @@ -773,7 +788,7 @@ vm_page_freechk_and_unqueue(m) /* * helper routine for vm_page_free and vm_page_free_zero */ -__inline static void +static __inline void vm_page_free_wakeup() { @@ -880,7 +895,7 @@ vm_page_wire(m) if (m->wire_count == 0) { s = splvm(); - vm_page_unqueue(m,1); + vm_page_unqueue(m); splx(s); cnt.v_wire_count++; } @@ -946,7 +961,7 @@ vm_page_deactivate(m) if (m->wire_count == 0 && m->hold_count == 0) { if (m->queue == PQ_CACHE) cnt.v_reactivated++; - vm_page_unqueue(m,1); + vm_page_unqueue(m); TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq); m->queue = PQ_INACTIVE; cnt.v_inactive_count++; @@ -977,7 +992,7 @@ vm_page_cache(m) panic("vm_page_cache: caching a dirty page, pindex: %d", m->pindex); } s = splvm(); - vm_page_unqueue(m,0); + vm_page_unqueue_nowakeup(m); TAILQ_INSERT_TAIL(&vm_page_queue_cache, m, pageq); m->queue = PQ_CACHE; cnt.v_cache_count++; @@ -1016,7 +1031,7 @@ vm_page_set_validclean(m, base, size) m->valid |= pagebits; m->dirty &= ~pagebits; if( base == 0 && size == PAGE_SIZE) - pmap_tc_modified(m); + pmap_clear_modify(VM_PAGE_TO_PHYS(m)); } /* @@ -1056,8 +1071,10 @@ void vm_page_test_dirty(m) vm_page_t m; { - if (m->dirty != VM_PAGE_BITS_ALL) - pmap_tc_modified(m); + if ((m->dirty != VM_PAGE_BITS_ALL) && + pmap_is_modified(VM_PAGE_TO_PHYS(m))) { + m->dirty = VM_PAGE_BITS_ALL; + } } /* diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h index 1680d4d..921aa31 100644 --- a/sys/vm/vm_page.h +++ b/sys/vm/vm_page.h @@ -61,7 +61,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_page.h,v 1.29 1996/06/26 05:39:25 dyson Exp $ + * $Id: vm_page.h,v 1.30 1996/07/27 03:24:06 dyson Exp $ */ /* @@ -220,7 +220,6 @@ extern vm_offset_t last_phys_addr; /* physical address for last_page */ (m)->flags &= ~PG_BUSY; \ if ((m)->flags & PG_WANTED) { \ (m)->flags &= ~PG_WANTED; \ - (m)->flags |= PG_REFERENCED; \ wakeup((caddr_t) (m)); \ } \ } @@ -252,7 +251,8 @@ void vm_page_rename __P((vm_page_t, vm_object_t, vm_pindex_t)); vm_offset_t vm_page_startup __P((vm_offset_t, vm_offset_t, vm_offset_t)); void vm_page_unwire __P((vm_page_t)); void vm_page_wire __P((vm_page_t)); -void vm_page_unqueue __P((vm_page_t, int)); +void vm_page_unqueue __P((vm_page_t)); +void vm_page_unqueue_nowakeup __P((vm_page_t)); void vm_page_set_validclean __P((vm_page_t, int, int)); void vm_page_set_invalid __P((vm_page_t, int, int)); static __inline boolean_t vm_page_zero_fill __P((vm_page_t)); @@ -292,11 +292,11 @@ vm_page_protect(vm_page_t mem, int prot) { if (prot == VM_PROT_NONE) { if (mem->flags & (PG_WRITEABLE|PG_MAPPED)) { - pmap_page_protect(mem, prot); + pmap_page_protect(VM_PAGE_TO_PHYS(mem), prot); mem->flags &= ~(PG_WRITEABLE|PG_MAPPED); } } else if ((prot == VM_PROT_READ) && (mem->flags & PG_WRITEABLE)) { - pmap_page_protect(mem, prot); + pmap_page_protect(VM_PAGE_TO_PHYS(mem), prot); mem->flags &= ~PG_WRITEABLE; } } diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c index 26df38c..d4db0bc 100644 --- a/sys/vm/vm_pageout.c +++ b/sys/vm/vm_pageout.c @@ -65,7 +65,7 @@ * any improvements or extensions that they make and grant Carnegie the * rights to redistribute these changes. * - * $Id: vm_pageout.c,v 1.81 1996/07/08 02:25:53 dyson Exp $ + * $Id: vm_pageout.c,v 1.83 1996/07/27 03:24:08 dyson Exp $ */ /* @@ -314,9 +314,12 @@ do_backward: } } + /* + * we allow reads during pageouts... + */ for (i = page_base; i < (page_base + pageout_count); i++) { mc[i]->flags |= PG_BUSY; - vm_page_protect(mc[i], VM_PROT_NONE); + vm_page_protect(mc[i], VM_PROT_READ); } return vm_pageout_flush(&mc[page_base], pageout_count, sync); @@ -356,7 +359,7 @@ vm_pageout_flush(mc, count, sync) * essentially lose the changes by pretending it * worked. */ - pmap_tc_modified(mt); + pmap_clear_modify(VM_PAGE_TO_PHYS(mt)); mt->dirty = 0; break; case VM_PAGER_ERROR: @@ -443,7 +446,7 @@ vm_pageout_object_deactivate_pages(map, object, desired, map_remove_only) continue; } - refcount = pmap_tc_referenced(VM_PAGE_TO_PHYS(p)); + refcount = pmap_ts_referenced(VM_PAGE_TO_PHYS(p)); if (refcount) { p->flags |= PG_REFERENCED; } else if (p->flags & PG_REFERENCED) { @@ -583,7 +586,7 @@ vm_pageout_scan() maxlaunder = (cnt.v_inactive_target > MAXLAUNDER) ? MAXLAUNDER : cnt.v_inactive_target; - +rescan0: maxscan = cnt.v_inactive_count; for( m = TAILQ_FIRST(&vm_page_queue_inactive); @@ -596,7 +599,7 @@ vm_pageout_scan() cnt.v_pdpages++; if (m->queue != PQ_INACTIVE) { - break; + goto rescan0; } next = TAILQ_NEXT(m, pageq); @@ -618,33 +621,32 @@ vm_pageout_scan() continue; } - if (m->valid != 0) { - if (m->object->ref_count == 0) { - m->flags &= ~PG_REFERENCED; - pmap_tc_referenced(VM_PAGE_TO_PHYS(m)); - } else if (((m->flags & PG_REFERENCED) == 0) && - pmap_tc_referenced(VM_PAGE_TO_PHYS(m))) { - vm_page_activate(m); - continue; - } + if (m->object->ref_count == 0) { + m->flags &= ~PG_REFERENCED; + pmap_clear_reference(VM_PAGE_TO_PHYS(m)); + } else if (((m->flags & PG_REFERENCED) == 0) && + pmap_ts_referenced(VM_PAGE_TO_PHYS(m))) { + vm_page_activate(m); + continue; + } - if ((m->flags & PG_REFERENCED) != 0) { - m->flags &= ~PG_REFERENCED; - pmap_tc_referenced(VM_PAGE_TO_PHYS(m)); - vm_page_activate(m); - continue; - } - if (m->dirty == 0) { - vm_page_test_dirty(m); - } else if (m->dirty != 0) { - m->dirty = VM_PAGE_BITS_ALL; - } - } + if ((m->flags & PG_REFERENCED) != 0) { + m->flags &= ~PG_REFERENCED; + pmap_clear_reference(VM_PAGE_TO_PHYS(m)); + vm_page_activate(m); + continue; + } + + if (m->dirty == 0) { + vm_page_test_dirty(m); + } else if (m->dirty != 0) { + m->dirty = VM_PAGE_BITS_ALL; + } if (m->valid == 0) { vm_page_protect(m, VM_PROT_NONE); vm_page_free(m); - ++cnt.v_dfree; + cnt.v_dfree++; ++pages_freed; } else if (m->dirty == 0) { vm_page_cache(m); @@ -786,7 +788,7 @@ vm_pageout_scan() if (m->flags & PG_REFERENCED) { refcount += 1; } - refcount += pmap_tc_referenced(VM_PAGE_TO_PHYS(m)); + refcount += pmap_ts_referenced(VM_PAGE_TO_PHYS(m)); if (refcount) { m->act_count += ACT_ADVANCE + refcount; if (m->act_count > ACT_MAX) diff --git a/sys/vm/vnode_pager.c b/sys/vm/vnode_pager.c index 3fe0ae3..3c9d21a 100644 --- a/sys/vm/vnode_pager.c +++ b/sys/vm/vnode_pager.c @@ -38,7 +38,7 @@ * SUCH DAMAGE. * * from: @(#)vnode_pager.c 7.5 (Berkeley) 4/20/91 - * $Id: vnode_pager.c,v 1.60 1996/05/03 21:01:54 phk Exp $ + * $Id: vnode_pager.c,v 1.61 1996/07/27 03:24:10 dyson Exp $ */ /* @@ -525,7 +525,7 @@ vnode_pager_input_smlfs(object, m) } } vm_pager_unmap_page(kva); - pmap_tc_modified(m); + pmap_clear_modify(VM_PAGE_TO_PHYS(m)); m->flags &= ~PG_ZERO; if (error) { return VM_PAGER_ERROR; @@ -588,7 +588,7 @@ vnode_pager_input_old(object, m) } vm_pager_unmap_page(kva); } - pmap_tc_modified(m); + pmap_clear_modify(VM_PAGE_TO_PHYS(m)); m->dirty = 0; m->flags &= ~PG_ZERO; return error ? VM_PAGER_ERROR : VM_PAGER_OK; @@ -808,7 +808,7 @@ vnode_pager_leaf_getpages(object, m, count, reqpage) relpbuf(bp); for (i = 0; i < count; i++) { - pmap_tc_modified(m[i]); + pmap_clear_modify(VM_PAGE_TO_PHYS(m[i])); m[i]->dirty = 0; m[i]->valid = VM_PAGE_BITS_ALL; m[i]->flags &= ~PG_ZERO; |