Push down the responsibility for zeroing a physical page from the

caller to vm_page_grab().  Although this gives VM_ALLOC_ZERO a
different meaning for vm_page_grab() than for vm_page_alloc(), I feel
such change is necessary to accomplish other goals.  Specifically, I
want to make the PG_ZERO flag immutable between the time it is
allocated by vm_page_alloc() and freed by vm_page_free() or
vm_page_free_zero() to avoid locking overheads.  Once we gave up on
the ability to automatically recognize a zeroed page upon entry to
vm_page_free(), the ability to mutate the PG_ZERO flag became useless.
Instead, I would like to say that "Once a page becomes valid, its
PG_ZERO flag must be ignored."

7 files changed, 2 insertions, 13 deletions

diff --git a/sys/alpha/alpha/pmap.c b/sys/alpha/alpha/pmap.c
index 11571cb..30afacb 100644
--- a/sys/alpha/alpha/pmap.c
+++ b/sys/alpha/alpha/pmap.c
@@ -1194,8 +1194,6 @@ _pmap_allocpte(pmap, ptepindex)
 		VM_OBJECT_LOCK(pmap->pm_pteobj);
 	m = vm_page_grab(pmap->pm_pteobj, ptepindex,
 	    VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
-	if ((m->flags & PG_ZERO) == 0)
-		pmap_zero_page(m);
 
 	KASSERT(m->queue == PQ_NONE,
 		("_pmap_allocpte: %p->queue != PQ_NONE", m));
@@ -1242,7 +1240,6 @@ _pmap_allocpte(pmap, ptepindex)
 
 	vm_page_lock_queues();
 	m->valid = VM_PAGE_BITS_ALL;
-	vm_page_flag_clear(m, PG_ZERO);
 	vm_page_wakeup(m);
 	vm_page_unlock_queues();
 	if (!is_object_locked)
diff --git a/sys/dev/agp/agp.c b/sys/dev/agp/agp.c
index fc4fd1f..f507aa6 100644
--- a/sys/dev/agp/agp.c
+++ b/sys/dev/agp/agp.c
@@ -524,8 +524,6 @@ agp_generic_bind_memory(device_t dev, struct agp_memory *mem,
 		m = vm_page_grab(mem->am_obj, OFF_TO_IDX(i),
 		    VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
 		VM_OBJECT_UNLOCK(mem->am_obj);
-		if ((m->flags & PG_ZERO) == 0)
-			pmap_zero_page(m);
 		AGP_DPF("found page pa=%#x\n", VM_PAGE_TO_PHYS(m));
 
 		/*
diff --git a/sys/dev/agp/agp_i810.c b/sys/dev/agp/agp_i810.c
index 9210247..30aca50 100644
--- a/sys/dev/agp/agp_i810.c
+++ b/sys/dev/agp/agp_i810.c
@@ -612,8 +612,6 @@ agp_i810_alloc_memory(device_t dev, int type, vm_size_t size)
 		m = vm_page_grab(mem->am_obj, 0,
 		    VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
 		VM_OBJECT_UNLOCK(mem->am_obj);
-		if ((m->flags & PG_ZERO) == 0)
-			pmap_zero_page(m);
 		vm_page_lock_queues();
 		mem->am_physical = VM_PAGE_TO_PHYS(m);
 		vm_page_wakeup(m);
diff --git a/sys/pci/agp.c b/sys/pci/agp.c
index fc4fd1f..f507aa6 100644
--- a/sys/pci/agp.c
+++ b/sys/pci/agp.c
@@ -524,8 +524,6 @@ agp_generic_bind_memory(device_t dev, struct agp_memory *mem,
 		m = vm_page_grab(mem->am_obj, OFF_TO_IDX(i),
 		    VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
 		VM_OBJECT_UNLOCK(mem->am_obj);
-		if ((m->flags & PG_ZERO) == 0)
-			pmap_zero_page(m);
 		AGP_DPF("found page pa=%#x\n", VM_PAGE_TO_PHYS(m));
 
 		/*
diff --git a/sys/pci/agp_i810.c b/sys/pci/agp_i810.c
index 9210247..30aca50 100644
--- a/sys/pci/agp_i810.c
+++ b/sys/pci/agp_i810.c
@@ -612,8 +612,6 @@ agp_i810_alloc_memory(device_t dev, int type, vm_size_t size)
 		m = vm_page_grab(mem->am_obj, 0,
 		    VM_ALLOC_WIRED | VM_ALLOC_ZERO | VM_ALLOC_RETRY);
 		VM_OBJECT_UNLOCK(mem->am_obj);
-		if ((m->flags & PG_ZERO) == 0)
-			pmap_zero_page(m);
 		vm_page_lock_queues();
 		mem->am_physical = VM_PAGE_TO_PHYS(m);
 		vm_page_wakeup(m);
diff --git a/sys/vm/vm_kern.c b/sys/vm/vm_kern.c
index 1a0af6a..3e21a99 100644
--- a/sys/vm/vm_kern.c
+++ b/sys/vm/vm_kern.c
@@ -198,8 +198,6 @@ kmem_alloc(map, size)
 
 		mem = vm_page_grab(kernel_object, OFF_TO_IDX(offset + i),
 				VM_ALLOC_ZERO | VM_ALLOC_RETRY);
-		if ((mem->flags & PG_ZERO) == 0)
-			pmap_zero_page(mem);
 		mem->valid = VM_PAGE_BITS_ALL;
 		vm_page_lock_queues();
 		vm_page_unmanage(mem);
diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 789ad98..45e3fc9 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -1460,6 +1460,8 @@ retrylookup:
 			return NULL;
 		goto retrylookup;
 	}
+	if (allocflags & VM_ALLOC_ZERO && (m->flags & PG_ZERO) == 0)
+		pmap_zero_page(m);
 
 	return m;
 }