- Under heavy IO load the buf daemon can run for many hundereds of

   milliseconds due to what is essentially n^2 algorithmic complexity.  This
   change makes the algorithm N*2 instead.  This heavy processing manifested
   itself as skipping in audio and video playback due to the long scheduling
   latencies and contention on giant by pcm.
 - flushbufqueues() is now responsible for flushing multiple buffers
   rather than one at a time.  This allows us to save our progress in the
   list by using a sentinal.  We must do the numdirtywakeup() and
   waitrunningbufspace() here now rather than in buf_daemon().
 - Also add a uio_yield() after we have processed the list once for bufs
   without deps and again for bufs with deps.  This is to release Giant
   and allow any other giant locked code to proceed.

Tested by:	Many users on current@
Revealed by:	schedgraph traces sent by Emil Mikulic & Anthony Ginepro

1 files changed, 29 insertions, 7 deletions

diff --git a/sys/kern/vfs_bio.c b/sys/kern/vfs_bio.c
index 8c6a6b9..f6f91d2 100644
--- a/sys/kern/vfs_bio.c
+++ b/sys/kern/vfs_bio.c
@@ -1985,8 +1985,7 @@ buf_daemon()
 				flushbufqueues(1);
 				break;
 			}
-			waitrunningbufspace();
-			numdirtywakeup((lodirtybuffers + hidirtybuffers) / 2);
+			uio_yield();
 		}
 
 		/*
@@ -2034,13 +2033,28 @@ static int
 flushbufqueues(int flushdeps)
 {
 	struct thread *td = curthread;
+	struct buf sentinal;
 	struct vnode *vp;
 	struct mount *mp;
 	struct buf *bp;
 	int hasdeps;
-
+	int flushed;
+	int target;
+
+	target = numdirtybuffers - lodirtybuffers;
+	if (flushdeps && target > 2)
+		target /= 2;
+	flushed = 0;
+	bp = NULL;
 	mtx_lock(&bqlock);
-	TAILQ_FOREACH(bp, &bufqueues[QUEUE_DIRTY], b_freelist) {
+	TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], &sentinal, b_freelist);
+	while (flushed != target) {
+		bp = TAILQ_FIRST(&bufqueues[QUEUE_DIRTY]);
+		if (bp == &sentinal)
+			break;
+		TAILQ_REMOVE(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
+		TAILQ_INSERT_TAIL(&bufqueues[QUEUE_DIRTY], bp, b_freelist);
+
 		if (BUF_LOCK(bp, LK_EXCLUSIVE | LK_NOWAIT, NULL) != 0)
 			continue;
 		BO_LOCK(bp->b_bufobj);
@@ -2055,7 +2069,10 @@ flushbufqueues(int flushdeps)
 			bremfreel(bp);
 			mtx_unlock(&bqlock);
 			brelse(bp);
-			return (1);
+			flushed++;
+			numdirtywakeup((lodirtybuffers + hidirtybuffers) / 2);
+			mtx_lock(&bqlock);
+			continue;
 		}
 
 		if (LIST_FIRST(&bp->b_dep) != NULL && buf_countdeps(bp, 0)) {
@@ -2089,13 +2106,18 @@ flushbufqueues(int flushdeps)
 			vn_finished_write(mp);
 			VOP_UNLOCK(vp, 0, td);
 			flushwithdeps += hasdeps;
-			return (1);
+			flushed++;
+			waitrunningbufspace();
+			numdirtywakeup((lodirtybuffers + hidirtybuffers) / 2);
+			mtx_lock(&bqlock);
+			continue;
 		}
 		vn_finished_write(mp);
 		BUF_UNLOCK(bp);
 	}
+	TAILQ_REMOVE(&bufqueues[QUEUE_DIRTY], &sentinal, b_freelist);
 	mtx_unlock(&bqlock);
-	return (0);
+	return (flushed);
 }
 
 /*