Reimplement vm_object_page_clean(), using the fact that vm object memq

is ordered by page index. This greatly simplifies the implementation,
since we no longer need to mark the pages with VPO_CLEANCHK to denote
the progress. It is enough to remember the current position by index
before dropping the object lock.

Remove VPO_CLEANCHK and VM_PAGER_IGNORE_CLEANCHK as unused.
Garbage-collect vm.msync_flush_flags sysctl.

Suggested and reviewed by:	alc
Tested by:	pho

3 files changed, 73 insertions, 191 deletions

diff --git a/sys/vm/vm_object.c b/sys/vm/vm_object.c
index 11284ab..7f29adb 100644
--- a/sys/vm/vm_object.c
+++ b/sys/vm/vm_object.c
@@ -101,13 +101,6 @@ __FBSDID("$FreeBSD$");
 #define MSYNC_FLUSH_HARDSEQ	0x01
 #define MSYNC_FLUSH_SOFTSEQ	0x02
 
-/*
- * msync / VM object flushing optimizations
- */
-static int msync_flush_flags = MSYNC_FLUSH_HARDSEQ | MSYNC_FLUSH_SOFTSEQ;
-SYSCTL_INT(_vm, OID_AUTO, msync_flush_flags, CTLFLAG_RW, &msync_flush_flags, 0,
-    "Enable sequential iteration optimization");
-
 static int old_msync;
 SYSCTL_INT(_vm, OID_AUTO, old_msync, CTLFLAG_RW, &old_msync, 0,
     "Use old (insecure) msync behavior");
@@ -762,276 +755,167 @@ vm_object_terminate(vm_object_t object)
  *	The object must be locked.
  */
 void
-vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end, int flags)
+vm_object_page_clean(vm_object_t object, vm_pindex_t start, vm_pindex_t end,
+    int flags)
 {
-	vm_page_t p, np;
-	vm_pindex_t tstart, tend;
-	vm_pindex_t pi;
-	int clearobjflags;
-	int pagerflags;
-	int curgeneration;
+	vm_page_t np, p;
+	vm_pindex_t pi, tend;
+	int clearobjflags, curgeneration, n, pagerflags;
 
 	mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED);
 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
-	if ((object->flags & OBJ_MIGHTBEDIRTY) == 0)
+	if ((object->flags & OBJ_MIGHTBEDIRTY) == 0 ||
+	    object->resident_page_count == 0)
 		return;
 	KASSERT(object->type == OBJT_VNODE, ("Not a vnode object"));
 
-	pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) ? VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
-	pagerflags |= (flags & OBJPC_INVAL) ? VM_PAGER_PUT_INVAL : 0;
+	pagerflags = (flags & (OBJPC_SYNC | OBJPC_INVAL)) != 0 ?
+	    VM_PAGER_PUT_SYNC : VM_PAGER_CLUSTER_OK;
+	pagerflags |= (flags & OBJPC_INVAL) != 0 ? VM_PAGER_PUT_INVAL : 0;
 
-	vm_object_set_flag(object, OBJ_CLEANING);
+	tend = (end == 0) ? object->size : end;
 
-	tstart = start;
-	if (end == 0) {
-		tend = object->size;
-	} else {
-		tend = end;
-	}
-
-	/*
-	 * If the caller is smart and only msync()s a range he knows is
-	 * dirty, we may be able to avoid an object scan.  This results in
-	 * a phenominal improvement in performance.  We cannot do this
-	 * as a matter of course because the object may be huge - e.g.
-	 * the size might be in the gigabytes or terrabytes.
-	 */
-	if (msync_flush_flags & MSYNC_FLUSH_HARDSEQ) {
-		vm_pindex_t tscan;
-		int scanlimit;
-		int scanreset;
-
-		scanreset = object->resident_page_count / EASY_SCAN_FACTOR;
-		if (scanreset < 16)
-			scanreset = 16;
-		pagerflags |= VM_PAGER_IGNORE_CLEANCHK;
-
-		scanlimit = scanreset;
-		tscan = tstart;
-		while (tscan < tend) {
-			curgeneration = object->generation;
-			p = vm_page_lookup(object, tscan);
-			if (p == NULL || p->valid == 0) {
-				if (--scanlimit == 0)
-					break;
-				++tscan;
-				continue;
-			}
-			vm_page_test_dirty(p);
-			if (p->dirty == 0) {
-				if (--scanlimit == 0)
-					break;
-				++tscan;
-				continue;
-			}
-			/*
-			 * If we have been asked to skip nosync pages and 
-			 * this is a nosync page, we can't continue.
-			 */
-			if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) {
-				if (--scanlimit == 0)
-					break;
-				++tscan;
-				continue;
-			}
-			scanlimit = scanreset;
-
-			/*
-			 * This returns 0 if it was unable to busy the first
-			 * page (i.e. had to sleep).
-			 */
-			tscan += vm_object_page_collect_flush(object, p, curgeneration, pagerflags);
-
-		}
-
-		/*
-		 * If everything was dirty and we flushed it successfully,
-		 * and the requested range is not the entire object, we
-		 * don't have to mess with CLEANCHK or MIGHTBEDIRTY and can
-		 * return immediately.
-		 */
-		if (tscan >= tend && (tstart || tend < object->size)) {
-			vm_object_clear_flag(object, OBJ_CLEANING);
-			return;
-		}
-		pagerflags &= ~VM_PAGER_IGNORE_CLEANCHK;
-	}
+	vm_object_set_flag(object, OBJ_CLEANING);
 
 	/*
-	 * Generally set CLEANCHK interlock and make the page read-only so
-	 * we can then clear the object flags.
+	 * Make the page read-only so we can then clear the object flags.
 	 *
 	 * However, if this is a nosync mmap then the object is likely to 
 	 * stay dirty so do not mess with the page and do not clear the
 	 * object flags.
 	 */
 	clearobjflags = 1;
-	TAILQ_FOREACH(p, &object->memq, listq) {
-		p->oflags |= VPO_CLEANCHK;
-		if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC))
+	for (p = vm_page_find_least(object, start);
+	    p != NULL && p->pindex < tend; p = TAILQ_NEXT(p, listq)) {
+		if ((flags & OBJPC_NOSYNC) != 0 &&
+		    (p->oflags & VPO_NOSYNC) != 0)
 			clearobjflags = 0;
 		else
 			pmap_remove_write(p);
 	}
 
-	if (clearobjflags && (tstart == 0) && (tend == object->size))
+	if (clearobjflags && (start == 0) && (tend == object->size))
 		vm_object_clear_flag(object, OBJ_MIGHTBEDIRTY);
 
 rescan:
+	p = vm_page_find_least(object, start);
 	curgeneration = object->generation;
 
-	for (p = TAILQ_FIRST(&object->memq); p; p = np) {
-		int n;
-
-		np = TAILQ_NEXT(p, listq);
-
-again:
+	for (; p != NULL; p = np) {
 		pi = p->pindex;
-		if ((p->oflags & VPO_CLEANCHK) == 0 ||
-			(pi < tstart) || (pi >= tend) ||
-		    p->valid == 0) {
-			p->oflags &= ~VPO_CLEANCHK;
+		if (pi >= tend)
+			break;
+		np = TAILQ_NEXT(p, listq);
+		if (p->valid == 0)
 			continue;
+		while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) {
+			if (object->generation != curgeneration)
+				goto rescan;
 		}
-
 		vm_page_test_dirty(p);
-		if (p->dirty == 0) {
-			p->oflags &= ~VPO_CLEANCHK;
+		if (p->dirty == 0)
 			continue;
-		}
+
 		/*
 		 * If we have been asked to skip nosync pages and this is a
 		 * nosync page, skip it.  Note that the object flags were
 		 * not cleared in this case so we do not have to set them.
 		 */
-		if ((flags & OBJPC_NOSYNC) && (p->oflags & VPO_NOSYNC)) {
-			p->oflags &= ~VPO_CLEANCHK;
+		if ((flags & OBJPC_NOSYNC) != 0 &&
+		    (p->oflags & VPO_NOSYNC) != 0)
 			continue;
-		}
 
 		n = vm_object_page_collect_flush(object, p,
-			curgeneration, pagerflags);
-		if (n == 0)
-			goto rescan;
-
+		    curgeneration, pagerflags);
+		KASSERT(n > 0, ("vm_object_page_collect_flush failed"));
 		if (object->generation != curgeneration)
 			goto rescan;
-
-		/*
-		 * Try to optimize the next page.  If we can't we pick up
-		 * our (random) scan where we left off.
-		 */
-		if (msync_flush_flags & MSYNC_FLUSH_SOFTSEQ)
-			if ((p = vm_page_lookup(object, pi + n)) != NULL)
-				goto again;
+		np = vm_page_find_least(object, pi + n);
 	}
 #if 0
-	VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC)?MNT_WAIT:0, curproc);
+	VOP_FSYNC(vp, (pagerflags & VM_PAGER_PUT_SYNC) ? MNT_WAIT : 0);
 #endif
 
 	vm_object_clear_flag(object, OBJ_CLEANING);
-	return;
 }
 
 static int
-vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration, int pagerflags)
+vm_object_page_collect_flush(vm_object_t object, vm_page_t p, int curgeneration,
+    int pagerflags)
 {
 	int runlen;
 	int maxf;
 	int chkb;
 	int maxb;
-	int i;
+	int i, index;
 	vm_pindex_t pi;
 	vm_page_t maf[vm_pageout_page_count];
 	vm_page_t mab[vm_pageout_page_count];
 	vm_page_t ma[vm_pageout_page_count];
+	vm_page_t tp, p1;
 
 	mtx_assert(&vm_page_queue_mtx, MA_NOTOWNED);
 	vm_page_lock_assert(p, MA_NOTOWNED);
 	VM_OBJECT_LOCK_ASSERT(object, MA_OWNED);
 	pi = p->pindex;
-	while (vm_page_sleep_if_busy(p, TRUE, "vpcwai")) {
-		if (object->generation != curgeneration) {
-			return(0);
-		}
-	}
 	maxf = 0;
-	for(i = 1; i < vm_pageout_page_count; i++) {
-		vm_page_t tp;
-
-		if ((tp = vm_page_lookup(object, pi + i)) != NULL) {
-			if ((tp->oflags & VPO_BUSY) ||
-				((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
-				 (tp->oflags & VPO_CLEANCHK) == 0) ||
-				(tp->busy != 0))
-				break;
-			vm_page_test_dirty(tp);
-			if (tp->dirty == 0) {
-				tp->oflags &= ~VPO_CLEANCHK;
-				break;
-			}
-			maf[ i - 1 ] = tp;
-			maxf++;
-			continue;
-		}
-		break;
+	for (i = 1, p1 = p; i < vm_pageout_page_count; i++) {
+		tp = vm_page_next(p1);
+		if (tp == NULL || tp->busy != 0 || (tp->oflags & VPO_BUSY) != 0)
+			break;
+		vm_page_test_dirty(tp);
+		if (tp->dirty == 0)
+			break;
+		maf[i - 1] = p1 = tp;
+		maxf++;
 	}
 
 	maxb = 0;
 	chkb = vm_pageout_page_count -  maxf;
-	if (chkb) {
-		for(i = 1; i < chkb;i++) {
-			vm_page_t tp;
-
-			if ((tp = vm_page_lookup(object, pi - i)) != NULL) {
-				if ((tp->oflags & VPO_BUSY) ||
-					((pagerflags & VM_PAGER_IGNORE_CLEANCHK) == 0 &&
-					 (tp->oflags & VPO_CLEANCHK) == 0) ||
-					(tp->busy != 0))
-					break;
-				vm_page_test_dirty(tp);
-				if (tp->dirty == 0) {
-					tp->oflags &= ~VPO_CLEANCHK;
-					break;
-				}
-				mab[ i - 1 ] = tp;
-				maxb++;
-				continue;
-			}
+	for (i = 1, p1 = p; i < chkb; i++) {
+		tp = vm_page_prev(p1);
+		if (tp == NULL || tp->busy != 0 || (tp->oflags & VPO_BUSY) != 0)
 			break;
-		}
+		vm_page_test_dirty(tp);
+		if (tp->dirty == 0)
+			break;
+		mab[i - 1] = p1 = tp;
+		maxb++;
 	}
 
-	for(i = 0; i < maxb; i++) {
-		int index = (maxb - i) - 1;
+	for (i = 0; i < maxb; i++) {
+		index = (maxb - i) - 1;
 		ma[index] = mab[i];
-		ma[index]->oflags &= ~VPO_CLEANCHK;
 	}
-	p->oflags &= ~VPO_CLEANCHK;
 	ma[maxb] = p;
-	for(i = 0; i < maxf; i++) {
-		int index = (maxb + i) + 1;
+	for (i = 0; i < maxf; i++) {
+		index = (maxb + i) + 1;
 		ma[index] = maf[i];
-		ma[index]->oflags &= ~VPO_CLEANCHK;
 	}
 	runlen = maxb + maxf + 1;
 
 	vm_pageout_flush(ma, runlen, pagerflags);
 	for (i = 0; i < runlen; i++) {
-		if (ma[i]->dirty) {
-			pmap_remove_write(ma[i]);
-			ma[i]->oflags |= VPO_CLEANCHK;
-
+		if (ma[i]->dirty != 0) {
+			KASSERT((ma[i]->flags & PG_WRITEABLE) == 0,
+	("vm_object_page_collect_flush: page %p is not write protected",
+			    ma[i]));
+		}
+	}
+	for (i = 0; i < maxf; i++) {
+		if (ma[i + maxb + 1]->dirty != 0) {
 			/*
 			 * maxf will end up being the actual number of pages
 			 * we wrote out contiguously, non-inclusive of the
 			 * first page.  We do not count look-behind pages.
 			 */
-			if (i >= maxb + 1 && (maxf > i - maxb - 1))
-				maxf = i - maxb - 1;
+			if (maxf > i) {
+				maxf = i;
+				break;
+			}
 		}
 	}
-	return(maxf + 1);
+	return (maxf + 1);
 }
 
 /*
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index c769a7b..1b2f39a 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -143,7 +143,6 @@ struct vm_page {
  */
 #define	VPO_BUSY	0x0001	/* page is in transit */
 #define	VPO_WANTED	0x0002	/* someone is waiting for page */
-#define	VPO_CLEANCHK	0x0100	/* page will be checked for cleaning */
 #define	VPO_SWAPINPROG	0x0200	/* swap I/O in progress on page */
 #define	VPO_NOSYNC	0x0400	/* do not collect for syncer */
 
diff --git a/sys/vm/vm_pager.h b/sys/vm/vm_pager.h
index 75d8f0a..519af42 100644
--- a/sys/vm/vm_pager.h
+++ b/sys/vm/vm_pager.h
@@ -90,7 +90,6 @@ extern struct pagerops sgpagerops;
 
 #define	VM_PAGER_PUT_SYNC		0x0001
 #define	VM_PAGER_PUT_INVAL		0x0002
-#define VM_PAGER_IGNORE_CLEANCHK	0x0004
 #define VM_PAGER_CLUSTER_OK		0x0008
 
 #ifdef _KERNEL