summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--sys/amd64/amd64/pmap.c5
-rw-r--r--sys/amd64/amd64/uma_machdep.c16
-rw-r--r--sys/arm64/arm64/uma_machdep.c16
-rw-r--r--sys/fs/tmpfs/tmpfs_subr.c9
-rw-r--r--sys/kern/uipc_shm.c9
-rw-r--r--sys/kern/vfs_bio.c8
-rw-r--r--sys/mips/mips/uma_machdep.c4
-rw-r--r--sys/powerpc/aim/mmu_oea64.c18
-rw-r--r--sys/powerpc/aim/slb.c18
-rw-r--r--sys/powerpc/powerpc/uma_machdep.c15
-rw-r--r--sys/sparc64/sparc64/vm_machdep.c18
-rw-r--r--sys/vm/phys_pager.c9
-rw-r--r--sys/vm/swap_pager.c4
-rw-r--r--sys/vm/uma.h5
-rw-r--r--sys/vm/uma_core.c18
-rw-r--r--sys/vm/vm_kern.c15
-rw-r--r--sys/vm/vm_object.c5
-rw-r--r--sys/vm/vm_page.c106
-rw-r--r--sys/vm/vm_page.h8
-rw-r--r--sys/vm/vm_radix.c6
-rw-r--r--sys/vm/vm_radix.h1
-rw-r--r--sys/x86/iommu/intel_utils.c13
22 files changed, 177 insertions, 149 deletions
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index c7317b1..b1c7d84 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -2566,9 +2566,8 @@ pmap_pinit_type(pmap_t pmap, enum pmap_type pm_type, int flags)
/*
* allocate the page directory page
*/
- while ((pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL |
- VM_ALLOC_NOOBJ | VM_ALLOC_WIRED | VM_ALLOC_ZERO)) == NULL)
- VM_WAIT;
+ pml4pg = vm_page_alloc(NULL, 0, VM_ALLOC_NORMAL | VM_ALLOC_NOOBJ |
+ VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_WAITOK);
pml4phys = VM_PAGE_TO_PHYS(pml4pg);
pmap->pm_pml4 = (pml4_entry_t *)PHYS_TO_DMAP(pml4phys);
diff --git a/sys/amd64/amd64/uma_machdep.c b/sys/amd64/amd64/uma_machdep.c
index db566ae..220d095 100644
--- a/sys/amd64/amd64/uma_machdep.c
+++ b/sys/amd64/amd64/uma_machdep.c
@@ -46,20 +46,12 @@ uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
vm_page_t m;
vm_paddr_t pa;
void *va;
- int pflags;
*flags = UMA_SLAB_PRIV;
- pflags = malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
- for (;;) {
- m = vm_page_alloc(NULL, 0, pflags);
- if (m == NULL) {
- if (wait & M_NOWAIT)
- return (NULL);
- else
- VM_WAIT;
- } else
- break;
- }
+ m = vm_page_alloc(NULL, 0,
+ malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED);
+ if (m == NULL)
+ return (NULL);
pa = m->phys_addr;
if ((wait & M_NODUMP) == 0)
dump_add_page(pa);
diff --git a/sys/arm64/arm64/uma_machdep.c b/sys/arm64/arm64/uma_machdep.c
index 9b9df5c..34dc675 100644
--- a/sys/arm64/arm64/uma_machdep.c
+++ b/sys/arm64/arm64/uma_machdep.c
@@ -46,20 +46,12 @@ uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
vm_page_t m;
vm_paddr_t pa;
void *va;
- int pflags;
*flags = UMA_SLAB_PRIV;
- pflags = malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
- for (;;) {
- m = vm_page_alloc(NULL, 0, pflags);
- if (m == NULL) {
- if (wait & M_NOWAIT)
- return (NULL);
- else
- VM_WAIT;
- } else
- break;
- }
+ m = vm_page_alloc(NULL, 0,
+ malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED);
+ if (m == NULL)
+ return (NULL);
pa = m->phys_addr;
if ((wait & M_NODUMP) == 0)
dump_add_page(pa);
diff --git a/sys/fs/tmpfs/tmpfs_subr.c b/sys/fs/tmpfs/tmpfs_subr.c
index d247fbf..1227d48 100644
--- a/sys/fs/tmpfs/tmpfs_subr.c
+++ b/sys/fs/tmpfs/tmpfs_subr.c
@@ -1394,13 +1394,10 @@ retry:
goto retry;
MPASS(m->valid == VM_PAGE_BITS_ALL);
} else if (vm_pager_has_page(uobj, idx, NULL, NULL)) {
- m = vm_page_alloc(uobj, idx, VM_ALLOC_NORMAL);
- if (m == NULL) {
- VM_OBJECT_WUNLOCK(uobj);
- VM_WAIT;
- VM_OBJECT_WLOCK(uobj);
+ m = vm_page_alloc(uobj, idx, VM_ALLOC_NORMAL |
+ VM_ALLOC_WAITFAIL);
+ if (m == NULL)
goto retry;
- }
rv = vm_pager_get_pages(uobj, &m, 1, NULL,
NULL);
vm_page_lock(m);
diff --git a/sys/kern/uipc_shm.c b/sys/kern/uipc_shm.c
index 5228253..d4155c6 100644
--- a/sys/kern/uipc_shm.c
+++ b/sys/kern/uipc_shm.c
@@ -449,13 +449,10 @@ retry:
if (vm_page_sleep_if_busy(m, "shmtrc"))
goto retry;
} else if (vm_pager_has_page(object, idx, NULL, NULL)) {
- m = vm_page_alloc(object, idx, VM_ALLOC_NORMAL);
- if (m == NULL) {
- VM_OBJECT_WUNLOCK(object);
- VM_WAIT;
- VM_OBJECT_WLOCK(object);
+ m = vm_page_alloc(object, idx,
+ VM_ALLOC_NORMAL | VM_ALLOC_WAITFAIL);
+ if (m == NULL)
goto retry;
- }
rv = vm_pager_get_pages(object, &m, 1, NULL,
NULL);
vm_page_lock(m);
diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 9954356..7909e6b 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -4440,18 +4440,14 @@ vm_hold_load_pages(struct buf *bp, vm_offset_t from, vm_offset_t to)
index = (from - trunc_page((vm_offset_t)bp->b_data)) >> PAGE_SHIFT;
for (pg = from; pg < to; pg += PAGE_SIZE, index++) {
-tryagain:
/*
* note: must allocate system pages since blocking here
* could interfere with paging I/O, no matter which
* process we are.
*/
p = vm_page_alloc(NULL, 0, VM_ALLOC_SYSTEM | VM_ALLOC_NOOBJ |
- VM_ALLOC_WIRED | VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT));
- if (p == NULL) {
- VM_WAIT;
- goto tryagain;
- }
+ VM_ALLOC_WIRED | VM_ALLOC_COUNT((to - pg) >> PAGE_SHIFT) |
+ VM_ALLOC_WAITOK);
pmap_qenter(pg, &p, 1);
bp->b_pages[index] = p;
}
diff --git a/sys/mips/mips/uma_machdep.c b/sys/mips/mips/uma_machdep.c
index b4006e1..bc32075 100644
--- a/sys/mips/mips/uma_machdep.c
+++ b/sys/mips/mips/uma_machdep.c
@@ -50,6 +50,10 @@ uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
*flags = UMA_SLAB_PRIV;
pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
+#ifndef __mips_n64
+ pflags &= ~(VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
+ pflags |= VM_ALLOC_NOWAIT;
+#endif
for (;;) {
m = vm_page_alloc_freelist(VM_FREELIST_DIRECT, pflags);
diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
index 71ce72a..aa96807 100644
--- a/sys/powerpc/aim/mmu_oea64.c
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -1513,7 +1513,7 @@ moea64_uma_page_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags,
struct pvo_entry *pvo;
vm_offset_t va;
vm_page_t m;
- int pflags, needed_lock;
+ int needed_lock;
/*
* This entire routine is a horrible hack to avoid bothering kmem
@@ -1524,17 +1524,11 @@ moea64_uma_page_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags,
*flags = UMA_SLAB_PRIV;
needed_lock = !PMAP_LOCKED(kernel_pmap);
- pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
-
- for (;;) {
- m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
- if (m == NULL) {
- if (wait & M_NOWAIT)
- return (NULL);
- VM_WAIT;
- } else
- break;
- }
+
+ m = vm_page_alloc(NULL, 0,
+ malloc2vm_flags(wait) | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
+ if (m == NULL)
+ return (NULL);
va = VM_PAGE_TO_PHYS(m);
diff --git a/sys/powerpc/aim/slb.c b/sys/powerpc/aim/slb.c
index 72a595b..caf6f4f 100644
--- a/sys/powerpc/aim/slb.c
+++ b/sys/powerpc/aim/slb.c
@@ -483,24 +483,16 @@ slb_uma_real_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
static vm_offset_t realmax = 0;
void *va;
vm_page_t m;
- int pflags;
if (realmax == 0)
realmax = platform_real_maxaddr();
*flags = UMA_SLAB_PRIV;
- pflags = malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED;
-
- for (;;) {
- m = vm_page_alloc_contig(NULL, 0, pflags, 1, 0, realmax,
- PAGE_SIZE, PAGE_SIZE, VM_MEMATTR_DEFAULT);
- if (m == NULL) {
- if (wait & M_NOWAIT)
- return (NULL);
- VM_WAIT;
- } else
- break;
- }
+ m = vm_page_alloc_contig(NULL, 0,
+ malloc2vm_flags(wait) | VM_ALLOC_NOOBJ | VM_ALLOC_WIRED,
+ 1, 0, realmax, PAGE_SIZE, PAGE_SIZE, VM_MEMATTR_DEFAULT);
+ if (m == NULL)
+ return (NULL);
va = (void *) VM_PAGE_TO_PHYS(m);
diff --git a/sys/powerpc/powerpc/uma_machdep.c b/sys/powerpc/powerpc/uma_machdep.c
index d5a458f..e9be5ca 100644
--- a/sys/powerpc/powerpc/uma_machdep.c
+++ b/sys/powerpc/powerpc/uma_machdep.c
@@ -55,20 +55,13 @@ uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
void *va;
vm_paddr_t pa;
vm_page_t m;
- int pflags;
*flags = UMA_SLAB_PRIV;
- pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
- for (;;) {
- m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
- if (m == NULL) {
- if (wait & M_NOWAIT)
- return (NULL);
- VM_WAIT;
- } else
- break;
- }
+ m = vm_page_alloc(NULL, 0,
+ malloc2vm_flags(wait) | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
+ if (m == NULL)
+ return (NULL);
pa = VM_PAGE_TO_PHYS(m);
diff --git a/sys/sparc64/sparc64/vm_machdep.c b/sys/sparc64/sparc64/vm_machdep.c
index a8b244c..10d1d36 100644
--- a/sys/sparc64/sparc64/vm_machdep.c
+++ b/sys/sparc64/sparc64/vm_machdep.c
@@ -394,24 +394,16 @@ uma_small_alloc(uma_zone_t zone, vm_size_t bytes, u_int8_t *flags, int wait)
{
vm_paddr_t pa;
vm_page_t m;
- int pflags;
void *va;
PMAP_STATS_INC(uma_nsmall_alloc);
*flags = UMA_SLAB_PRIV;
- pflags = malloc2vm_flags(wait) | VM_ALLOC_WIRED;
-
- for (;;) {
- m = vm_page_alloc(NULL, 0, pflags | VM_ALLOC_NOOBJ);
- if (m == NULL) {
- if (wait & M_NOWAIT)
- return (NULL);
- else
- VM_WAIT;
- } else
- break;
- }
+
+ m = vm_page_alloc(NULL, 0,
+ malloc2vm_flags(wait) | VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
+ if (m == NULL)
+ return (NULL);
pa = VM_PAGE_TO_PHYS(m);
if (dcache_color_ignore == 0 && m->md.color != DCACHE_COLOR(pa)) {
diff --git a/sys/vm/phys_pager.c b/sys/vm/phys_pager.c
index 7adc7c6..0977cfd 100644
--- a/sys/vm/phys_pager.c
+++ b/sys/vm/phys_pager.c
@@ -209,13 +209,10 @@ retry:
if (m == NULL) {
ahead = MIN(end - i, PHYSALLOC);
m = vm_page_alloc(object, i, VM_ALLOC_NORMAL |
- VM_ALLOC_ZERO | VM_ALLOC_COUNT(ahead));
- if (m == NULL) {
- VM_OBJECT_WUNLOCK(object);
- VM_WAIT;
- VM_OBJECT_WLOCK(object);
+ VM_ALLOC_ZERO | VM_ALLOC_WAITFAIL |
+ VM_ALLOC_COUNT(ahead));
+ if (m == NULL)
goto retry;
- }
if ((m->flags & PG_ZERO) == 0)
pmap_zero_page(m);
m->valid = VM_PAGE_BITS_ALL;
diff --git a/sys/vm/swap_pager.c b/sys/vm/swap_pager.c
index f663ce1..0dfc98d 100644
--- a/sys/vm/swap_pager.c
+++ b/sys/vm/swap_pager.c
@@ -1765,7 +1765,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
vm_pageout_oom(VM_OOM_SWAPZ);
pause("swzonxb", 10);
} else
- VM_WAIT;
+ uma_zwait(swblk_zone);
VM_OBJECT_WLOCK(object);
sb = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
rdpi);
@@ -1795,7 +1795,7 @@ swp_pager_meta_build(vm_object_t object, vm_pindex_t pindex, daddr_t swapblk)
vm_pageout_oom(VM_OOM_SWAPZ);
pause("swzonxp", 10);
} else
- VM_WAIT;
+ uma_zwait(swpctrie_zone);
VM_OBJECT_WLOCK(object);
sb1 = SWAP_PCTRIE_LOOKUP(&object->un_pager.swp.swp_blks,
rdpi);
diff --git a/sys/vm/uma.h b/sys/vm/uma.h
index f4c2de8..569b951 100644
--- a/sys/vm/uma.h
+++ b/sys/vm/uma.h
@@ -365,6 +365,11 @@ uma_zfree(uma_zone_t zone, void *item)
}
/*
+ * Wait until the specified zone can allocate an item.
+ */
+void uma_zwait(uma_zone_t zone);
+
+/*
* XXX The rest of the prototypes in this header are h0h0 magic for the VM.
* If you think you need to use it for a normal zone you're probably incorrect.
*/
diff --git a/sys/vm/uma_core.c b/sys/vm/uma_core.c
index bdb6fae..295e4e2 100644
--- a/sys/vm/uma_core.c
+++ b/sys/vm/uma_core.c
@@ -1127,7 +1127,9 @@ noobj_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
npages = howmany(bytes, PAGE_SIZE);
while (npages > 0) {
p = vm_page_alloc(NULL, 0, VM_ALLOC_INTERRUPT |
- VM_ALLOC_WIRED | VM_ALLOC_NOOBJ);
+ VM_ALLOC_WIRED | VM_ALLOC_NOOBJ |
+ ((wait & M_WAITOK) != 0 ? VM_ALLOC_WAITOK :
+ VM_ALLOC_NOWAIT));
if (p != NULL) {
/*
* Since the page does not belong to an object, its
@@ -1137,11 +1139,6 @@ noobj_alloc(uma_zone_t zone, vm_size_t bytes, uint8_t *flags, int wait)
npages--;
continue;
}
- if (wait & M_WAITOK) {
- VM_WAIT;
- continue;
- }
-
/*
* Page allocation failed, free intermediate pages and
* exit.
@@ -2082,6 +2079,15 @@ uma_zdestroy(uma_zone_t zone)
sx_sunlock(&uma_drain_lock);
}
+void
+uma_zwait(uma_zone_t zone)
+{
+ void *item;
+
+ item = uma_zalloc_arg(zone, NULL, M_WAITOK);
+ uma_zfree(zone, item);
+}
+
/* See uma.h */
void *
uma_zalloc_arg(uma_zone_t zone, void *udata, int flags)
diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c
index 8e9ab76..5a0eb4d 100644
--- a/sys/vm/vm_kern.c
+++ b/sys/vm/vm_kern.c
@@ -172,6 +172,8 @@ kmem_alloc_attr(vmem_t *vmem, vm_size_t size, int flags, vm_paddr_t low,
return (0);
offset = addr - VM_MIN_KERNEL_ADDRESS;
pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
+ pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
+ pflags |= VM_ALLOC_NOWAIT;
VM_OBJECT_WLOCK(object);
for (i = 0; i < size; i += PAGE_SIZE) {
tries = 0;
@@ -227,6 +229,8 @@ kmem_alloc_contig(struct vmem *vmem, vm_size_t size, int flags, vm_paddr_t low,
return (0);
offset = addr - VM_MIN_KERNEL_ADDRESS;
pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
+ pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
+ pflags |= VM_ALLOC_NOWAIT;
npages = atop(size);
VM_OBJECT_WLOCK(object);
tries = 0;
@@ -338,10 +342,13 @@ kmem_back(vm_object_t object, vm_offset_t addr, vm_size_t size, int flags)
offset = addr - VM_MIN_KERNEL_ADDRESS;
pflags = malloc2vm_flags(flags) | VM_ALLOC_NOBUSY | VM_ALLOC_WIRED;
+ pflags &= ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
+ if (flags & M_WAITOK)
+ pflags |= VM_ALLOC_WAITFAIL;
i = 0;
-retry:
VM_OBJECT_WLOCK(object);
+retry:
mpred = vm_radix_lookup_le(&object->rtree, atop(offset + i));
for (; i < size; i += PAGE_SIZE, mpred = m) {
m = vm_page_alloc_after(object, atop(offset + i), pflags,
@@ -353,11 +360,9 @@ retry:
* aren't on any queues.
*/
if (m == NULL) {
- VM_OBJECT_WUNLOCK(object);
- if ((flags & M_NOWAIT) == 0) {
- VM_WAIT;
+ if ((flags & M_NOWAIT) == 0)
goto retry;
- }
+ VM_OBJECT_WUNLOCK(object);
kmem_unback(object, addr, i);
return (KERN_NO_SPACE);
}
diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 8ceee4e..1e11078 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -1412,7 +1412,7 @@ retry:
if (vm_page_rename(m, new_object, idx)) {
VM_OBJECT_WUNLOCK(new_object);
VM_OBJECT_WUNLOCK(orig_object);
- VM_WAIT;
+ vm_radix_wait();
VM_OBJECT_WLOCK(orig_object);
VM_OBJECT_WLOCK(new_object);
goto retry;
@@ -1474,8 +1474,9 @@ vm_object_collapse_scan_wait(vm_object_t object, vm_page_t p, vm_page_t next,
vm_page_lock(p);
VM_OBJECT_WUNLOCK(object);
VM_OBJECT_WUNLOCK(backing_object);
+ /* The page is only NULL when rename fails. */
if (p == NULL)
- VM_WAIT;
+ vm_radix_wait();
else
vm_page_busy_sleep(p, "vmocol", false);
VM_OBJECT_WLOCK(object);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 888af45..49398f2 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -167,6 +167,7 @@ static void vm_page_insert_radixdone(vm_page_t m, vm_object_t object,
vm_page_t mpred);
static int vm_page_reclaim_run(int req_class, u_long npages, vm_page_t m_run,
vm_paddr_t high);
+static int vm_page_alloc_fail(vm_object_t object, int req);
SYSINIT(vm_page, SI_SUB_VM, SI_ORDER_SECOND, vm_page_init_fakepg, NULL);
@@ -1606,6 +1607,8 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex, int req,
((req & (VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)) !=
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)),
("inconsistent object(%p)/req(%x)", object, req));
+ KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0,
+ ("Can't sleep and retry object insertion."));
KASSERT(mpred == NULL || mpred->pindex < pindex,
("mpred %p doesn't precede pindex 0x%jx", mpred,
(uintmax_t)pindex));
@@ -1627,6 +1630,7 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex, int req,
* Allocate a page if the number of free pages exceeds the minimum
* for the request class.
*/
+again:
mtx_lock(&vm_page_queue_free_mtx);
if (vm_cnt.v_free_count > vm_cnt.v_free_reserved ||
(req_class == VM_ALLOC_SYSTEM &&
@@ -1659,10 +1663,8 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex, int req,
/*
* Not allocatable, give up.
*/
- mtx_unlock(&vm_page_queue_free_mtx);
- atomic_add_int(&vm_pageout_deficit,
- max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
- pagedaemon_wakeup();
+ if (vm_page_alloc_fail(object, req))
+ goto again;
return (NULL);
}
@@ -1716,6 +1718,11 @@ vm_page_alloc_after(vm_object_t object, vm_pindex_t pindex, int req,
m->busy_lock = VPB_UNBUSIED;
/* Don't change PG_ZERO. */
vm_page_free_toq(m);
+ if (req & VM_ALLOC_WAITFAIL) {
+ VM_OBJECT_WUNLOCK(object);
+ vm_radix_wait();
+ VM_OBJECT_WLOCK(object);
+ }
return (NULL);
}
@@ -1793,6 +1800,8 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
(VM_ALLOC_NOBUSY | VM_ALLOC_SBUSY)),
("vm_page_alloc_contig: inconsistent object(%p)/req(%x)", object,
req));
+ KASSERT(object == NULL || (req & VM_ALLOC_WAITOK) == 0,
+ ("Can't sleep and retry object insertion."));
if (object != NULL) {
VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT((object->flags & OBJ_FICTITIOUS) == 0,
@@ -1818,6 +1827,7 @@ vm_page_alloc_contig(vm_object_t object, vm_pindex_t pindex, int req,
* Can we allocate the pages without the number of free pages falling
* below the lower bound for the allocation class?
*/
+again:
mtx_lock(&vm_page_queue_free_mtx);
if (vm_cnt.v_free_count >= npages + vm_cnt.v_free_reserved ||
(req_class == VM_ALLOC_SYSTEM &&
@@ -1839,9 +1849,8 @@ retry:
m_ret = vm_phys_alloc_contig(npages, low, high,
alignment, boundary);
} else {
- mtx_unlock(&vm_page_queue_free_mtx);
- atomic_add_int(&vm_pageout_deficit, npages);
- pagedaemon_wakeup();
+ if (vm_page_alloc_fail(object, req))
+ goto again;
return (NULL);
}
if (m_ret != NULL) {
@@ -1910,6 +1919,11 @@ retry:
/* Don't change PG_ZERO. */
vm_page_free_toq(m);
}
+ if (req & VM_ALLOC_WAITFAIL) {
+ VM_OBJECT_WUNLOCK(object);
+ vm_radix_wait();
+ VM_OBJECT_WLOCK(object);
+ }
return (NULL);
}
mpred = m;
@@ -1982,18 +1996,17 @@ vm_page_alloc_freelist(int flind, int req)
/*
* Do not allocate reserved pages unless the req has asked for it.
*/
+again:
mtx_lock(&vm_page_queue_free_mtx);
if (vm_cnt.v_free_count > vm_cnt.v_free_reserved ||
(req_class == VM_ALLOC_SYSTEM &&
vm_cnt.v_free_count > vm_cnt.v_interrupt_free_min) ||
(req_class == VM_ALLOC_INTERRUPT &&
- vm_cnt.v_free_count > 0))
+ vm_cnt.v_free_count > 0)) {
m = vm_phys_alloc_freelist_pages(flind, VM_FREEPOOL_DIRECT, 0);
- else {
- mtx_unlock(&vm_page_queue_free_mtx);
- atomic_add_int(&vm_pageout_deficit,
- max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
- pagedaemon_wakeup();
+ } else {
+ if (vm_page_alloc_fail(NULL, req))
+ goto again;
return (NULL);
}
if (m == NULL) {
@@ -2557,11 +2570,11 @@ vm_page_reclaim_contig(int req, u_long npages, vm_paddr_t low, vm_paddr_t high,
* Sleep until free pages are available for allocation.
* - Called in various places before memory allocations.
*/
-void
-vm_wait(void)
+static void
+_vm_wait(void)
{
- mtx_lock(&vm_page_queue_free_mtx);
+ mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
if (curproc == pageproc) {
vm_pageout_pages_needed = 1;
msleep(&vm_pageout_pages_needed, &vm_page_queue_free_mtx,
@@ -2579,6 +2592,46 @@ vm_wait(void)
}
}
+void
+vm_wait(void)
+{
+
+ mtx_lock(&vm_page_queue_free_mtx);
+ _vm_wait();
+}
+
+/*
+ * vm_page_alloc_fail:
+ *
+ * Called when a page allocation function fails. Informs the
+ * pagedaemon and performs the requested wait. Requires the
+ * page_queue_free and object lock on entry. Returns with the
+ * object lock held and free lock released. Returns an error when
+ * retry is necessary.
+ *
+ */
+static int
+vm_page_alloc_fail(vm_object_t object, int req)
+{
+
+ mtx_assert(&vm_page_queue_free_mtx, MA_OWNED);
+
+ atomic_add_int(&vm_pageout_deficit,
+ max((u_int)req >> VM_ALLOC_COUNT_SHIFT, 1));
+ pagedaemon_wakeup();
+ if (req & (VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL)) {
+ if (object != NULL)
+ VM_OBJECT_WUNLOCK(object);
+ _vm_wait();
+ if (object != NULL)
+ VM_OBJECT_WLOCK(object);
+ if (req & VM_ALLOC_WAITOK)
+ return (EAGAIN);
+ } else
+ mtx_unlock(&vm_page_queue_free_mtx);
+ return (0);
+}
+
/*
* vm_waitpfault: (also see VM_WAITPFAULT macro)
*
@@ -3190,11 +3243,16 @@ vm_page_grab(vm_object_t object, vm_pindex_t pindex, int allocflags)
{
vm_page_t m;
int sleep;
+ int pflags;
VM_OBJECT_ASSERT_WLOCKED(object);
KASSERT((allocflags & VM_ALLOC_SBUSY) == 0 ||
(allocflags & VM_ALLOC_IGN_SBUSY) != 0,
("vm_page_grab: VM_ALLOC_SBUSY/VM_ALLOC_IGN_SBUSY mismatch"));
+ pflags = allocflags &
+ ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK | VM_ALLOC_WAITFAIL);
+ if ((allocflags & VM_ALLOC_NOWAIT) == 0)
+ pflags |= VM_ALLOC_WAITFAIL;
retrylookup:
if ((m = vm_page_lookup(object, pindex)) != NULL) {
sleep = (allocflags & VM_ALLOC_IGN_SBUSY) != 0 ?
@@ -3228,13 +3286,10 @@ retrylookup:
return (m);
}
}
- m = vm_page_alloc(object, pindex, allocflags);
+ m = vm_page_alloc(object, pindex, pflags);
if (m == NULL) {
if ((allocflags & VM_ALLOC_NOWAIT) != 0)
return (NULL);
- VM_OBJECT_WUNLOCK(object);
- VM_WAIT;
- VM_OBJECT_WLOCK(object);
goto retrylookup;
}
if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
@@ -3273,6 +3328,7 @@ vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
vm_page_t *ma, int count)
{
vm_page_t m, mpred;
+ int pflags;
int i;
bool sleep;
@@ -3287,6 +3343,10 @@ vm_page_grab_pages(vm_object_t object, vm_pindex_t pindex, int allocflags,
("vm_page_grab_pages: VM_ALLOC_SBUSY/IGN_SBUSY mismatch"));
if (count == 0)
return (0);
+ pflags = allocflags & ~(VM_ALLOC_NOWAIT | VM_ALLOC_WAITOK |
+ VM_ALLOC_WAITFAIL | VM_ALLOC_IGN_SBUSY);
+ if ((allocflags & VM_ALLOC_NOWAIT) == 0)
+ pflags |= VM_ALLOC_WAITFAIL;
i = 0;
retrylookup:
m = vm_radix_lookup_le(&object->rtree, pindex + i);
@@ -3327,14 +3387,10 @@ retrylookup:
vm_page_sbusy(m);
} else {
m = vm_page_alloc_after(object, pindex + i,
- (allocflags & ~VM_ALLOC_IGN_SBUSY) |
- VM_ALLOC_COUNT(count - i), mpred);
+ pflags | VM_ALLOC_COUNT(count - i), mpred);
if (m == NULL) {
if ((allocflags & VM_ALLOC_NOWAIT) != 0)
break;
- VM_OBJECT_WUNLOCK(object);
- VM_WAIT;
- VM_OBJECT_WLOCK(object);
goto retrylookup;
}
}
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 3bb33fc..f9db153 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -405,6 +405,8 @@ vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa);
#define VM_ALLOC_INTERRUPT 1
#define VM_ALLOC_SYSTEM 2
#define VM_ALLOC_CLASS_MASK 3
+#define VM_ALLOC_WAITOK 0x0008 /* (acf) Sleep and retry */
+#define VM_ALLOC_WAITFAIL 0x0010 /* (acf) Sleep and return error */
#define VM_ALLOC_WIRED 0x0020 /* (acfgp) Allocate a wired page */
#define VM_ALLOC_ZERO 0x0040 /* (acfgp) Allocate a prezeroed page */
#define VM_ALLOC_NOOBJ 0x0100 /* (acg) No associated object */
@@ -414,7 +416,7 @@ vm_page_t PHYS_TO_VM_PAGE(vm_paddr_t pa);
#define VM_ALLOC_IGN_SBUSY 0x1000 /* (gp) Ignore shared busy flag */
#define VM_ALLOC_NODUMP 0x2000 /* (ag) don't include in dump */
#define VM_ALLOC_SBUSY 0x4000 /* (acgp) Shared busy the page */
-#define VM_ALLOC_NOWAIT 0x8000 /* (gp) Do not sleep */
+#define VM_ALLOC_NOWAIT 0x8000 /* (acfgp) Do not sleep */
#define VM_ALLOC_COUNT_SHIFT 16
#define VM_ALLOC_COUNT(count) ((count) << VM_ALLOC_COUNT_SHIFT)
@@ -433,6 +435,10 @@ malloc2vm_flags(int malloc_flags)
pflags |= VM_ALLOC_ZERO;
if ((malloc_flags & M_NODUMP) != 0)
pflags |= VM_ALLOC_NODUMP;
+ if ((malloc_flags & M_NOWAIT))
+ pflags |= VM_ALLOC_NOWAIT;
+ if ((malloc_flags & M_WAITOK))
+ pflags |= VM_ALLOC_WAITOK;
return (pflags);
}
#endif
diff --git a/sys/vm/vm_radix.c b/sys/vm/vm_radix.c
index 1a19dd9..546d316 100644
--- a/sys/vm/vm_radix.c
+++ b/sys/vm/vm_radix.c
@@ -775,6 +775,12 @@ vm_radix_replace(struct vm_radix *rtree, vm_page_t newpage)
panic("%s: original replacing page not found", __func__);
}
+void
+vm_radix_wait(void)
+{
+ uma_zwait(vm_radix_node_zone);
+}
+
#ifdef DDB
/*
* Show details about the given radix node.
diff --git a/sys/vm/vm_radix.h b/sys/vm/vm_radix.h
index d96e50b..df81180 100644
--- a/sys/vm/vm_radix.h
+++ b/sys/vm/vm_radix.h
@@ -36,6 +36,7 @@
#ifdef _KERNEL
int vm_radix_insert(struct vm_radix *rtree, vm_page_t page);
+void vm_radix_wait(void);
boolean_t vm_radix_is_singleton(struct vm_radix *rtree);
vm_page_t vm_radix_lookup(struct vm_radix *rtree, vm_pindex_t index);
vm_page_t vm_radix_lookup_ge(struct vm_radix *rtree, vm_pindex_t index);
diff --git a/sys/x86/iommu/intel_utils.c b/sys/x86/iommu/intel_utils.c
index 0dcb618..96ee694 100644
--- a/sys/x86/iommu/intel_utils.c
+++ b/sys/x86/iommu/intel_utils.c
@@ -257,9 +257,12 @@ vm_page_t
dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags)
{
vm_page_t m;
- int zeroed;
+ int zeroed, aflags;
zeroed = (flags & DMAR_PGF_ZERO) != 0 ? VM_ALLOC_ZERO : 0;
+ aflags = zeroed | VM_ALLOC_NOBUSY | VM_ALLOC_SYSTEM | VM_ALLOC_NODUMP |
+ ((flags & DMAR_PGF_WAITOK) != 0 ? VM_ALLOC_WAITFAIL :
+ VM_ALLOC_NOWAIT);
for (;;) {
if ((flags & DMAR_PGF_OBJL) == 0)
VM_OBJECT_WLOCK(obj);
@@ -269,8 +272,7 @@ dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags)
VM_OBJECT_WUNLOCK(obj);
break;
}
- m = vm_page_alloc_contig(obj, idx, VM_ALLOC_NOBUSY |
- VM_ALLOC_SYSTEM | VM_ALLOC_NODUMP | zeroed, 1, 0,
+ m = vm_page_alloc_contig(obj, idx, aflags, 1, 0,
dmar_high, PAGE_SIZE, 0, VM_MEMATTR_DEFAULT);
if ((flags & DMAR_PGF_OBJL) == 0)
VM_OBJECT_WUNLOCK(obj);
@@ -282,11 +284,6 @@ dmar_pgalloc(vm_object_t obj, vm_pindex_t idx, int flags)
}
if ((flags & DMAR_PGF_WAITOK) == 0)
break;
- if ((flags & DMAR_PGF_OBJL) != 0)
- VM_OBJECT_WUNLOCK(obj);
- VM_WAIT;
- if ((flags & DMAR_PGF_OBJL) != 0)
- VM_OBJECT_WLOCK(obj);
}
return (m);
}
OpenPOWER on IntegriCloud