diff options
Diffstat (limited to 'sys/vm')
-rw-r--r-- | sys/vm/pmap.h | 2 | ||||
-rw-r--r-- | sys/vm/vm_extern.h | 2 | ||||
-rw-r--r-- | sys/vm/vm_fault.c | 62 | ||||
-rw-r--r-- | sys/vm/vm_map.c | 128 | ||||
-rw-r--r-- | sys/vm/vm_object.c | 72 | ||||
-rw-r--r-- | sys/vm/vm_object.h | 2 |
6 files changed, 159 insertions, 109 deletions
diff --git a/sys/vm/pmap.h b/sys/vm/pmap.h index 2e7b19d..d73babc 100644 --- a/sys/vm/pmap.h +++ b/sys/vm/pmap.h @@ -109,7 +109,6 @@ void pmap_advise(pmap_t pmap, vm_offset_t sva, vm_offset_t eva, int advice); void pmap_align_superpage(vm_object_t, vm_ooffset_t, vm_offset_t *, vm_size_t); -void pmap_change_wiring(pmap_t, vm_offset_t, boolean_t); void pmap_clear_modify(vm_page_t m); void pmap_copy(pmap_t, pmap_t, vm_offset_t, vm_size_t, vm_offset_t); void pmap_copy_page(vm_page_t, vm_page_t); @@ -149,6 +148,7 @@ void pmap_remove_pages(pmap_t); void pmap_remove_write(vm_page_t m); void pmap_sync_icache(pmap_t, vm_offset_t, vm_size_t); boolean_t pmap_ts_referenced(vm_page_t m); +void pmap_unwire(pmap_t pmap, vm_offset_t start, vm_offset_t end); void pmap_zero_page(vm_page_t); void pmap_zero_page_area(vm_page_t, int off, int size); void pmap_zero_page_idle(vm_page_t); diff --git a/sys/vm/vm_extern.h b/sys/vm/vm_extern.h index 39a1a21..f639a22 100644 --- a/sys/vm/vm_extern.h +++ b/sys/vm/vm_extern.h @@ -81,8 +81,6 @@ int vm_fault_hold(vm_map_t map, vm_offset_t vaddr, vm_prot_t fault_type, int fault_flags, vm_page_t *m_hold); int vm_fault_quick_hold_pages(vm_map_t map, vm_offset_t addr, vm_size_t len, vm_prot_t prot, vm_page_t *ma, int max_count); -void vm_fault_unwire(vm_map_t, vm_offset_t, vm_offset_t, boolean_t); -int vm_fault_wire(vm_map_t, vm_offset_t, vm_offset_t, boolean_t); int vm_forkproc(struct thread *, struct proc *, struct thread *, struct vmspace *, int); void vm_waitproc(struct proc *); int vm_mmap(vm_map_t, vm_offset_t *, vm_size_t, vm_prot_t, vm_prot_t, int, objtype_t, void *, vm_ooffset_t); diff --git a/sys/vm/vm_fault.c b/sys/vm/vm_fault.c index 09f7423..a3a4c36 100644 --- a/sys/vm/vm_fault.c +++ b/sys/vm/vm_fault.c @@ -1203,68 +1203,6 @@ error: } /* - * vm_fault_wire: - * - * Wire down a range of virtual addresses in a map. - */ -int -vm_fault_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, - boolean_t fictitious) -{ - vm_offset_t va; - int rv; - - /* - * We simulate a fault to get the page and enter it in the physical - * map. For user wiring, we only ask for read access on currently - * read-only sections. - */ - for (va = start; va < end; va += PAGE_SIZE) { - rv = vm_fault(map, va, VM_PROT_NONE, VM_FAULT_CHANGE_WIRING); - if (rv) { - if (va != start) - vm_fault_unwire(map, start, va, fictitious); - return (rv); - } - } - return (KERN_SUCCESS); -} - -/* - * vm_fault_unwire: - * - * Unwire a range of virtual addresses in a map. - */ -void -vm_fault_unwire(vm_map_t map, vm_offset_t start, vm_offset_t end, - boolean_t fictitious) -{ - vm_paddr_t pa; - vm_offset_t va; - vm_page_t m; - pmap_t pmap; - - pmap = vm_map_pmap(map); - - /* - * Since the pages are wired down, we must be able to get their - * mappings from the physical map system. - */ - for (va = start; va < end; va += PAGE_SIZE) { - pa = pmap_extract(pmap, va); - if (pa != 0) { - pmap_change_wiring(pmap, va, FALSE); - if (!fictitious) { - m = PHYS_TO_VM_PAGE(pa); - vm_page_lock(m); - vm_page_unwire(m, TRUE); - vm_page_unlock(m); - } - } - } -} - -/* * Routine: * vm_fault_copy_entry * Function: diff --git a/sys/vm/vm_map.c b/sys/vm/vm_map.c index 891b68c..15611bf 100644 --- a/sys/vm/vm_map.c +++ b/sys/vm/vm_map.c @@ -132,6 +132,7 @@ static void _vm_map_init(vm_map_t map, pmap_t pmap, vm_offset_t min, vm_offset_t max); static void vm_map_entry_deallocate(vm_map_entry_t entry, boolean_t system_map); static void vm_map_entry_dispose(vm_map_t map, vm_map_entry_t entry); +static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry); #ifdef INVARIANTS static void vm_map_zdtor(void *mem, int size, void *arg); static void vmspace_zdtor(void *mem, int size, void *arg); @@ -139,6 +140,8 @@ static void vmspace_zdtor(void *mem, int size, void *arg); static int vm_map_stack_locked(vm_map_t map, vm_offset_t addrbos, vm_size_t max_ssize, vm_size_t growsize, vm_prot_t prot, vm_prot_t max, int cow); +static void vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, + vm_offset_t failed_addr); #define ENTRY_CHARGED(e) ((e)->cred != NULL || \ ((e)->object.vm_object != NULL && (e)->object.vm_object->cred != NULL && \ @@ -2407,16 +2410,10 @@ done: (entry->eflags & MAP_ENTRY_USER_WIRED))) { if (user_unwire) entry->eflags &= ~MAP_ENTRY_USER_WIRED; - entry->wired_count--; - if (entry->wired_count == 0) { - /* - * Retain the map lock. - */ - vm_fault_unwire(map, entry->start, entry->end, - entry->object.vm_object != NULL && - (entry->object.vm_object->flags & - OBJ_FICTITIOUS) != 0); - } + if (entry->wired_count == 1) + vm_map_entry_unwire(map, entry); + else + entry->wired_count--; } KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, ("vm_map_unwire: in-transition flag missing %p", entry)); @@ -2437,6 +2434,42 @@ done: } /* + * vm_map_wire_entry_failure: + * + * Handle a wiring failure on the given entry. + * + * The map should be locked. + */ +static void +vm_map_wire_entry_failure(vm_map_t map, vm_map_entry_t entry, + vm_offset_t failed_addr) +{ + + VM_MAP_ASSERT_LOCKED(map); + KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0 && + entry->wired_count == 1, + ("vm_map_wire_entry_failure: entry %p isn't being wired", entry)); + KASSERT(failed_addr < entry->end, + ("vm_map_wire_entry_failure: entry %p was fully wired", entry)); + + /* + * If any pages at the start of this entry were successfully wired, + * then unwire them. + */ + if (failed_addr > entry->start) { + pmap_unwire(map->pmap, entry->start, failed_addr); + vm_object_unwire(entry->object.vm_object, entry->offset, + failed_addr - entry->start, PQ_ACTIVE); + } + + /* + * Assign an out-of-range value to represent the failure to wire this + * entry. + */ + entry->wired_count = -1; +} + +/* * vm_map_wire: * * Implements both kernel and user wiring. @@ -2446,10 +2479,10 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, int flags) { vm_map_entry_t entry, first_entry, tmp_entry; - vm_offset_t saved_end, saved_start; + vm_offset_t faddr, saved_end, saved_start; unsigned int last_timestamp; int rv; - boolean_t fictitious, need_wakeup, result, user_wire; + boolean_t need_wakeup, result, user_wire; vm_prot_t prot; if (start == end) @@ -2542,17 +2575,24 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, entry->wired_count++; saved_start = entry->start; saved_end = entry->end; - fictitious = entry->object.vm_object != NULL && - (entry->object.vm_object->flags & - OBJ_FICTITIOUS) != 0; + /* * Release the map lock, relying on the in-transition * mark. Mark the map busy for fork. */ vm_map_busy(map); vm_map_unlock(map); - rv = vm_fault_wire(map, saved_start, saved_end, - fictitious); + + faddr = saved_start; + do { + /* + * Simulate a fault to get the page and enter + * it into the physical map. + */ + if ((rv = vm_fault(map, faddr, VM_PROT_NONE, + VM_FAULT_CHANGE_WIRING)) != KERN_SUCCESS) + break; + } while ((faddr += PAGE_SIZE) < saved_end); vm_map_lock(map); vm_map_unbusy(map); if (last_timestamp + 1 != map->timestamp) { @@ -2571,23 +2611,22 @@ vm_map_wire(vm_map_t map, vm_offset_t start, vm_offset_t end, first_entry = NULL; entry = tmp_entry; while (entry->end < saved_end) { - if (rv != KERN_SUCCESS) { - KASSERT(entry->wired_count == 1, - ("vm_map_wire: bad count")); - entry->wired_count = -1; - } + /* + * In case of failure, handle entries + * that were not fully wired here; + * fully wired entries are handled + * later. + */ + if (rv != KERN_SUCCESS && + faddr < entry->end) + vm_map_wire_entry_failure(map, + entry, faddr); entry = entry->next; } } last_timestamp = map->timestamp; if (rv != KERN_SUCCESS) { - KASSERT(entry->wired_count == 1, - ("vm_map_wire: bad count")); - /* - * Assign an out-of-range value to represent - * the failure to wire this entry. - */ - entry->wired_count = -1; + vm_map_wire_entry_failure(map, entry, faddr); end = entry->end; goto done; } @@ -2649,19 +2688,16 @@ done: * unnecessary. */ entry->wired_count = 0; - } else { - if (!user_wire || - (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) + } else if (!user_wire || + (entry->eflags & MAP_ENTRY_USER_WIRED) == 0) { + /* + * Undo the wiring. Wiring succeeded on this entry + * but failed on a later entry. + */ + if (entry->wired_count == 1) + vm_map_entry_unwire(map, entry); + else entry->wired_count--; - if (entry->wired_count == 0) { - /* - * Retain the map lock. - */ - vm_fault_unwire(map, entry->start, entry->end, - entry->object.vm_object != NULL && - (entry->object.vm_object->flags & - OBJ_FICTITIOUS) != 0); - } } next_entry_done: KASSERT((entry->eflags & MAP_ENTRY_IN_TRANSITION) != 0, @@ -2797,9 +2833,13 @@ vm_map_sync( static void vm_map_entry_unwire(vm_map_t map, vm_map_entry_t entry) { - vm_fault_unwire(map, entry->start, entry->end, - entry->object.vm_object != NULL && - (entry->object.vm_object->flags & OBJ_FICTITIOUS) != 0); + + VM_MAP_ASSERT_LOCKED(map); + KASSERT(entry->wired_count > 0, + ("vm_map_entry_unwire: entry %p isn't wired", entry)); + pmap_unwire(map->pmap, entry->start, entry->end); + vm_object_unwire(entry->object.vm_object, entry->offset, entry->end - + entry->start, PQ_ACTIVE); entry->wired_count = 0; } diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c index 6cfb0d4..94c3d30 100644 --- a/sys/vm/vm_object.c +++ b/sys/vm/vm_object.c @@ -2203,6 +2203,78 @@ vm_object_set_writeable_dirty(vm_object_t object) vm_object_set_flag(object, OBJ_MIGHTBEDIRTY); } +/* + * vm_object_unwire: + * + * For each page offset within the specified range of the given object, + * find the highest-level page in the shadow chain and unwire it. A page + * must exist at every page offset, and the highest-level page must be + * wired. + */ +void +vm_object_unwire(vm_object_t object, vm_ooffset_t offset, vm_size_t length, + uint8_t queue) +{ + vm_object_t tobject; + vm_page_t m, tm; + vm_pindex_t end_pindex, pindex, tpindex; + int depth, locked_depth; + + KASSERT((offset & PAGE_MASK) == 0, + ("vm_object_unwire: offset is not page aligned")); + KASSERT((length & PAGE_MASK) == 0, + ("vm_object_unwire: length is not a multiple of PAGE_SIZE")); + /* The wired count of a fictitious page never changes. */ + if ((object->flags & OBJ_FICTITIOUS) != 0) + return; + pindex = OFF_TO_IDX(offset); + end_pindex = pindex + atop(length); + locked_depth = 1; + VM_OBJECT_RLOCK(object); + m = vm_page_find_least(object, pindex); + while (pindex < end_pindex) { + if (m == NULL || pindex < m->pindex) { + /* + * The first object in the shadow chain doesn't + * contain a page at the current index. Therefore, + * the page must exist in a backing object. + */ + tobject = object; + tpindex = pindex; + depth = 0; + do { + tpindex += + OFF_TO_IDX(tobject->backing_object_offset); + tobject = tobject->backing_object; + KASSERT(tobject != NULL, + ("vm_object_unwire: missing page")); + if ((tobject->flags & OBJ_FICTITIOUS) != 0) + goto next_page; + depth++; + if (depth == locked_depth) { + locked_depth++; + VM_OBJECT_RLOCK(tobject); + } + } while ((tm = vm_page_lookup(tobject, tpindex)) == + NULL); + } else { + tm = m; + m = TAILQ_NEXT(m, listq); + } + vm_page_lock(tm); + vm_page_unwire(tm, queue); + vm_page_unlock(tm); +next_page: + pindex++; + } + /* Release the accumulated object locks. */ + for (depth = 0; depth < locked_depth; depth++) { + tobject = object->backing_object; + VM_OBJECT_RUNLOCK(object); + object = tobject; + } +} + #include "opt_ddb.h" #ifdef DDB #include <sys/kernel.h> diff --git a/sys/vm/vm_object.h b/sys/vm/vm_object.h index 4b21e55..48ba743 100644 --- a/sys/vm/vm_object.h +++ b/sys/vm/vm_object.h @@ -295,6 +295,8 @@ void vm_object_shadow (vm_object_t *, vm_ooffset_t *, vm_size_t); void vm_object_split(vm_map_entry_t); boolean_t vm_object_sync(vm_object_t, vm_ooffset_t, vm_size_t, boolean_t, boolean_t); +void vm_object_unwire(vm_object_t object, vm_ooffset_t offset, + vm_size_t length, uint8_t queue); #endif /* _KERNEL */ #endif /* _VM_OBJECT_ */ |