Split the vm_page flags PG_WRITEABLE and PG_REFERENCED into atomic

flags field. Updates to the atomic flags are performed using the atomic
ops on the containing word, do not require any vm lock to be held, and
are non-blocking. The vm_page_aflag_set(9) and vm_page_aflag_clear(9)
functions are provided to modify afalgs.

Document the changes to flags field to only require the page lock.

Introduce vm_page_reference(9) function to provide a stable KPI and
KBI for filesystems like tmpfs and zfs which need to mark a page as
referenced.

Reviewed by:    alc, attilio
Tested by:      marius, flo (sparc64); andreast (powerpc, powerpc64)
Approved by:	re (bz)

1 files changed, 13 insertions, 15 deletions

diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index e9c9927..5dd450e 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -497,7 +497,7 @@ vm_pageout_flush(vm_page_t *mc, int count, int flags, int mreq, int *prunlen)
 		vm_page_t mt = mc[i];
 
 		KASSERT(pageout_status[i] == VM_PAGER_PEND ||
-		    (mt->flags & PG_WRITEABLE) == 0,
+		    (mt->aflags & PGA_WRITEABLE) == 0,
 		    ("vm_pageout_flush: page %p is not write protected", mt));
 		switch (pageout_status[i]) {
 		case VM_PAGER_OK:
@@ -597,12 +597,10 @@ vm_pageout_object_deactivate_pages(pmap_t pmap, vm_object_t first_object,
 				continue;
 			}
 			actcount = pmap_ts_referenced(p);
-			if ((p->flags & PG_REFERENCED) != 0) {
+			if ((p->aflags & PGA_REFERENCED) != 0) {
 				if (actcount == 0)
 					actcount = 1;
-				vm_page_lock_queues();
-				vm_page_flag_clear(p, PG_REFERENCED);
-				vm_page_unlock_queues();
+				vm_page_aflag_clear(p, PGA_REFERENCED);
 			}
 			if (p->queue != PQ_ACTIVE && actcount != 0) {
 				vm_page_activate(p);
@@ -846,7 +844,7 @@ rescan0:
 		 * references.
 		 */
 		if (object->ref_count == 0) {
-			vm_page_flag_clear(m, PG_REFERENCED);
+			vm_page_aflag_clear(m, PGA_REFERENCED);
 			KASSERT(!pmap_page_is_mapped(m),
 			    ("vm_pageout_scan: page %p is mapped", m));
 
@@ -859,7 +857,7 @@ rescan0:
 		 * level VM system not knowing anything about existing 
 		 * references.
 		 */
-		} else if (((m->flags & PG_REFERENCED) == 0) &&
+		} else if (((m->aflags & PGA_REFERENCED) == 0) &&
 			(actcount = pmap_ts_referenced(m))) {
 			vm_page_activate(m);
 			vm_page_unlock(m);
@@ -874,8 +872,8 @@ rescan0:
 		 * "activation count" higher than normal so that we will less 
 		 * likely place pages back onto the inactive queue again.
 		 */
-		if ((m->flags & PG_REFERENCED) != 0) {
-			vm_page_flag_clear(m, PG_REFERENCED);
+		if ((m->aflags & PGA_REFERENCED) != 0) {
+			vm_page_aflag_clear(m, PGA_REFERENCED);
 			actcount = pmap_ts_referenced(m);
 			vm_page_activate(m);
 			vm_page_unlock(m);
@@ -891,7 +889,7 @@ rescan0:
 		 * be updated.
 		 */
 		if (m->dirty != VM_PAGE_BITS_ALL &&
-		    (m->flags & PG_WRITEABLE) != 0) {
+		    (m->aflags & PGA_WRITEABLE) != 0) {
 			/*
 			 * Avoid a race condition: Unless write access is
 			 * removed from the page, another processor could
@@ -938,7 +936,7 @@ rescan0:
 			 * before being freed.  This significantly extends
 			 * the thrash point for a heavily loaded machine.
 			 */
-			vm_page_flag_set(m, PG_WINATCFLS);
+			m->flags |= PG_WINATCFLS;
 			vm_page_requeue(m);
 		} else if (maxlaunder > 0) {
 			/*
@@ -1178,7 +1176,7 @@ unlock_and_continue:
 		 */
 		actcount = 0;
 		if (object->ref_count != 0) {
-			if (m->flags & PG_REFERENCED) {
+			if (m->aflags & PGA_REFERENCED) {
 				actcount += 1;
 			}
 			actcount += pmap_ts_referenced(m);
@@ -1192,7 +1190,7 @@ unlock_and_continue:
 		/*
 		 * Since we have "tested" this bit, we need to clear it now.
 		 */
-		vm_page_flag_clear(m, PG_REFERENCED);
+		vm_page_aflag_clear(m, PGA_REFERENCED);
 
 		/*
 		 * Only if an object is currently being used, do we use the
@@ -1435,8 +1433,8 @@ vm_pageout_page_stats()
 		}
 
 		actcount = 0;
-		if (m->flags & PG_REFERENCED) {
-			vm_page_flag_clear(m, PG_REFERENCED);
+		if (m->aflags & PGA_REFERENCED) {
+			vm_page_aflag_clear(m, PGA_REFERENCED);
 			actcount += 1;
 		}