Bug fixes and performance improvements from John Dyson and myself:

1) check va before clearing the page clean flag. Not doing so was
	causing the vnode pager error 5 messages when paging from
	NFS. (pmap.c)
2) put back interrupt protection in idle_loop. Bruce didn't think
	it was necessary, John insists that it is (and I agree). (swtch.s)
3) various improvements to the clustering code (vm_machdep.c). It's
	now enabled/used by default.
4) bad disk blocks are now handled properly when doing clustered IOs.
	(wd.c, vm_machdep.c)
5) bogus bad block handling fixed in wd.c.
6) algorithm improvements to the pageout/pagescan daemons. It's amazing
	how well 4MB machines work now.

4 files changed, 131 insertions, 39 deletions

diff --git a/sys/amd64/amd64/cpu_switch.S b/sys/amd64/amd64/cpu_switch.S
index 17f246c..4dbc672 100644
--- a/sys/amd64/amd64/cpu_switch.S
+++ b/sys/amd64/amd64/cpu_switch.S
@@ -33,7 +33,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: swtch.s,v 1.4 1994/01/31 10:26:59 davidg Exp $
+ *	$Id: swtch.s,v 1.5 1994/04/02 07:00:30 davidg Exp $
  */
 
 #include "npx.h"	/* for NNPX */
@@ -156,8 +156,10 @@ _idle:
 
 	ALIGN_TEXT
 idle_loop:
+	cli
 	cmpl	$0,_whichqs
-	jne	sw1
+	jne	sw1a
+	sti
 	hlt					/* wait for interrupt */
 	jmp	idle_loop
 
@@ -214,6 +216,7 @@ ENTRY(swtch)
 	/* save is done, now choose a new process or idle */
 sw1:
 	cli
+sw1a:
 	movl	_whichqs,%edi
 2:
 	/* XXX - bsf is sloow */
diff --git a/sys/amd64/amd64/pmap.c b/sys/amd64/amd64/pmap.c
index 9502df0..cc2e9a4 100644
--- a/sys/amd64/amd64/pmap.c
+++ b/sys/amd64/amd64/pmap.c
@@ -39,7 +39,7 @@
  * SUCH DAMAGE.
  *
  *	from:	@(#)pmap.c	7.7 (Berkeley)	5/12/91
- *	$Id: pmap.c,v 1.22 1994/03/30 02:17:45 davidg Exp $
+ *	$Id: pmap.c,v 1.23 1994/04/14 07:49:38 davidg Exp $
  */
 
 /*
@@ -153,6 +153,7 @@ static inline pv_entry_t	get_pv_entry();
 void				pmap_alloc_pv_entry();
 void				pmap_clear_modify();
 void				i386_protection_init();
+extern vm_offset_t pager_sva, pager_eva;
 
 #if BSDVM_COMPAT
 #include "msgbuf.h"
@@ -851,8 +852,10 @@ pmap_remove(pmap, sva, eva)
 		if (pmap_is_managed(pa)) {
 			if ((((int) oldpte & PG_M) && (sva < USRSTACK || sva > UPT_MAX_ADDRESS))
 				|| (sva >= USRSTACK && sva < USRSTACK+(UPAGES*NBPG))) {
-				m = PHYS_TO_VM_PAGE(pa);
-				m->flags &= ~PG_CLEAN;
+				if (sva < pager_sva || sva >= pager_eva) {
+					m = PHYS_TO_VM_PAGE(pa);
+					m->flags &= ~PG_CLEAN;
+				}
 			}
 
 			pv = pa_to_pvh(pa);
@@ -946,8 +949,10 @@ pmap_remove(pmap, sva, eva)
 
 		if ((((int) oldpte & PG_M) && (va < USRSTACK || va > UPT_MAX_ADDRESS))
 			|| (va >= USRSTACK && va < USRSTACK+(UPAGES*NBPG))) {
-			m = PHYS_TO_VM_PAGE(pa);
-			m->flags &= ~PG_CLEAN;
+			if (va < pager_sva || va >= pager_eva) {
+				m = PHYS_TO_VM_PAGE(pa);
+				m->flags &= ~PG_CLEAN;
+			}
 		}
 
 		pv = pa_to_pvh(pa);
@@ -1018,7 +1023,9 @@ pmap_remove_all(pa)
 		if ( (m->flags & PG_CLEAN) &&
 			((((int) *pte) & PG_M) && (pv->pv_va < USRSTACK || pv->pv_va > UPT_MAX_ADDRESS))
 			|| (pv->pv_va >= USRSTACK && pv->pv_va < USRSTACK+(UPAGES*NBPG))) {
-			m->flags &= ~PG_CLEAN;
+			if (pv->pv_va < pager_sva || pv->pv_va >= pager_eva) {
+				m->flags &= ~PG_CLEAN;
+			}
 		}
 
 		*pte = 0;
@@ -1724,8 +1731,16 @@ pmap_testbit(pa, bit)
 			 * then mark UPAGES as always modified, and
 			 * ptes as never modified.
 			 */
+			if (bit & PG_U ) {
+				if ((pv->pv_va >= pager_sva) && (pv->pv_va < pager_eva)) {
+					continue;
+				}
+			} 
 			if (bit & PG_M ) {
 				if (pv->pv_va >= USRSTACK) {
+					if (pv->pv_va >= pager_sva && pv->pv_va < pager_eva) {
+						continue;
+					}
 					if (pv->pv_va < USRSTACK+(UPAGES*NBPG)) {
 						splx(s);
 						return TRUE;
@@ -1780,8 +1795,6 @@ pmap_changebit(pa, bit, setem)
 			 * don't write protect pager mappings
 			 */
 			if (!setem && (bit == PG_RW)) {
-				extern vm_offset_t pager_sva, pager_eva;
-
 				if (va >= pager_sva && va < pager_eva)
 					continue;
 			}
diff --git a/sys/amd64/amd64/swtch.s b/sys/amd64/amd64/swtch.s
index 17f246c..4dbc672 100644
--- a/sys/amd64/amd64/swtch.s
+++ b/sys/amd64/amd64/swtch.s
@@ -33,7 +33,7 @@
  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
  * SUCH DAMAGE.
  *
- *	$Id: swtch.s,v 1.4 1994/01/31 10:26:59 davidg Exp $
+ *	$Id: swtch.s,v 1.5 1994/04/02 07:00:30 davidg Exp $
  */
 
 #include "npx.h"	/* for NNPX */
@@ -156,8 +156,10 @@ _idle:
 
 	ALIGN_TEXT
 idle_loop:
+	cli
 	cmpl	$0,_whichqs
-	jne	sw1
+	jne	sw1a
+	sti
 	hlt					/* wait for interrupt */
 	jmp	idle_loop
 
@@ -214,6 +216,7 @@ ENTRY(swtch)
 	/* save is done, now choose a new process or idle */
 sw1:
 	cli
+sw1a:
 	movl	_whichqs,%edi
 2:
 	/* XXX - bsf is sloow */
diff --git a/sys/amd64/amd64/vm_machdep.c b/sys/amd64/amd64/vm_machdep.c
index c9b1964..89eb127 100644
--- a/sys/amd64/amd64/vm_machdep.c
+++ b/sys/amd64/amd64/vm_machdep.c
@@ -38,7 +38,7 @@
  *
  *	from: @(#)vm_machdep.c	7.3 (Berkeley) 5/13/91
  *	Utah $Hdr: vm_machdep.c 1.16.1.1 89/06/23$
- *	$Id: vm_machdep.c,v 1.18 1994/04/05 03:23:09 davidg Exp $
+ *	$Id: vm_machdep.c,v 1.19 1994/04/14 07:49:40 davidg Exp $
  */
 
 #include "npx.h"
@@ -56,6 +56,11 @@
 
 #define b_cylin b_resid
 
+#define MAXCLSTATS 256
+int clstats[MAXCLSTATS];
+int rqstats[MAXCLSTATS];
+
+
 #ifndef NOBOUNCE
 
 caddr_t		bouncememory;
@@ -121,10 +126,20 @@ vm_bounce_kva_free(addr, size, now)
 	int s = splbio();
 	kvaf[kvasfreecnt].addr = addr;
 	kvaf[kvasfreecnt++].size = size;
-	if( now)
+	if( now) {
+		/*
+		 * this will do wakeups
+		 */
 		vm_bounce_kva(0,0);
-	else
-		wakeup((caddr_t) io_map);
+	} else {
+		if (bmwait) {
+		/*
+		 * if anyone is waiting on the bounce-map, then wakeup
+		 */
+			wakeup((caddr_t) io_map);
+			bmwait = 0;
+		}
+	}
 	splx(s);
 }
 
@@ -225,6 +240,49 @@ more:
 }
 
 /*
+ * same as vm_bounce_kva -- but really allocate
+ */
+vm_offset_t
+vm_bounce_kva_alloc(count) 
+int count;
+{
+	int i;
+	vm_offset_t kva;
+	vm_offset_t pa;
+	if( bouncepages == 0) {
+		kva = (vm_offset_t) malloc(count*NBPG, M_TEMP, M_WAITOK);
+		return kva;
+	}
+	kva = vm_bounce_kva(count, 1);
+	for(i=0;i<count;i++) {
+		pa = vm_bounce_page_find(1);
+		pmap_kenter(kva + i * NBPG, pa);
+	}
+	return kva;
+}
+
+/*
+ * same as vm_bounce_kva_free -- but really free
+ */
+void
+vm_bounce_kva_alloc_free(kva, count)
+	vm_offset_t kva;
+	int count;
+{
+	int i;
+	vm_offset_t pa;
+	if( bouncepages == 0) {
+		free((caddr_t) kva, M_TEMP);
+		return;
+	}
+	for(i = 0; i < count; i++) {
+		pa = pmap_kextract(kva + i * NBPG);
+		vm_bounce_page_free(pa, 1);
+	}
+	vm_bounce_kva_free(kva, count);
+}
+
+/*
  * do the things necessary to the struct buf to implement
  * bounce buffers...  inserted before the disk sort
  */
@@ -393,14 +451,6 @@ vm_bounce_free(bp)
  */
 	bouncekva = i386_trunc_page((vm_offset_t) bp->b_un.b_addr);
 	vm_bounce_kva_free( bouncekva, countvmpg*NBPG, 0);
-	if (bmwait) {
-		/*
-		 * if anyone is waiting on the bounce-map, then wakeup
-		 */
-		wakeup((caddr_t) io_map);
-		bmwait = 0;
-	}
-
 	bp->b_un.b_addr = bp->b_savekva;
 	bp->b_savekva = 0;
 	bp->b_flags &= ~B_BOUNCE;
@@ -476,6 +526,8 @@ cldisksort(struct buf *dp, struct buf *bp, vm_offset_t maxio)
 	vm_offset_t orig1begin, orig2begin;
 	vm_offset_t kvanew, kvaorig;
 
+	if( bp->b_bcount < MAXCLSTATS*PAGE_SIZE)
+		++rqstats[bp->b_bcount/PAGE_SIZE];
 	/*
 	 * If nothing on the activity queue, then
 	 * we become the only thing.
@@ -494,22 +546,22 @@ cldisksort(struct buf *dp, struct buf *bp, vm_offset_t maxio)
 	 * and add ourselves to it.
 	 */
 
-	if (bp->b_cylin < ap->b_cylin) {
+	if (bp->b_pblkno < ap->b_pblkno) {
 		while (ap->av_forw) {
 			/*
 			 * Check for an ``inversion'' in the
-			 * normally ascending cylinder numbers,
+			 * normally ascending block numbers,
 			 * indicating the start of the second request list.
 			 */
-			if (ap->av_forw->b_cylin < ap->b_cylin) {
+			if (ap->av_forw->b_pblkno < ap->b_pblkno) {
 				/*
 				 * Search the second request list
 				 * for the first request at a larger
-				 * cylinder number.  We go before that;
+				 * block number.  We go before that;
 				 * if there is no such request, we go at end.
 				 */
 				do {
-					if (bp->b_cylin < ap->av_forw->b_cylin)
+					if (bp->b_pblkno < ap->av_forw->b_pblkno)
 						goto insert;
 					ap = ap->av_forw;
 				} while (ap->av_forw);
@@ -532,21 +584,33 @@ cldisksort(struct buf *dp, struct buf *bp, vm_offset_t maxio)
 		 * We want to go after the current request
 		 * if there is an inversion after it (i.e. it is
 		 * the end of the first request list), or if
-		 * the next request is a larger cylinder than our request.
+		 * the next request is a larger block than our request.
 		 */
-		if (ap->av_forw->b_cylin < ap->b_cylin ||
-		    bp->b_cylin < ap->av_forw->b_cylin )
+		if (ap->av_forw->b_pblkno < ap->b_pblkno ||
+		    bp->b_pblkno < ap->av_forw->b_pblkno )
 			goto insert;
 		ap = ap->av_forw;
 	}
 
 insert:
+
+#if 0
+	/*
+	 * read clustering with new read-ahead disk drives hurts mostly, so
+	 * we don't bother...
+	 */
+	if( bp->b_flags & B_READ)
+		goto nocluster;
+#endif
 	/*
 	 * we currently only cluster I/O transfers that are at page-aligned
 	 * kvas and transfers that are multiples of page lengths.
 	 */
-	if(((bp->b_bcount & PAGE_MASK) == 0) &&
+	if ((bp->b_flags & B_BAD) == 0 &&
+		((bp->b_bcount & PAGE_MASK) == 0) &&
 		(((vm_offset_t) bp->b_un.b_addr & PAGE_MASK) == 0)) {
+		if( maxio > MAXCLSTATS*PAGE_SIZE)
+			maxio = MAXCLSTATS*PAGE_SIZE;
 		/*
 		 * merge with previous?
 		 * conditions:
@@ -558,9 +622,10 @@ insert:
 		 *	   is a multiple of a page in length.
 		 *	5) And the total I/O size would be below the maximum.
 		 */
-		if( (ap->b_blkno + (ap->b_bcount / DEV_BSIZE) == bp->b_blkno) &&
+		if( (ap->b_pblkno + (ap->b_bcount / DEV_BSIZE) == bp->b_pblkno) &&
 			(dp->b_actf != ap) &&
 			((ap->b_flags & ~B_CLUSTER) == bp->b_flags) &&
+			((ap->b_flags & B_BAD) == 0) &&
 			((ap->b_bcount & PAGE_MASK) == 0) &&
 			(((vm_offset_t) ap->b_un.b_addr & PAGE_MASK) == 0) &&
 			(ap->b_bcount + bp->b_bcount < maxio)) {
@@ -597,6 +662,7 @@ insert:
 				 * build the new bp to be handed off to the device
 				 */
 
+				--clstats[ap->b_bcount/PAGE_SIZE];
 				*newbp = *ap;
 				newbp->b_flags |= B_CLUSTER;
 				newbp->b_un.b_addr = (caddr_t) kvanew;
@@ -604,6 +670,7 @@ insert:
 				newbp->b_bufsize = newbp->b_bcount;
 				newbp->b_clusterf = ap;
 				newbp->b_clusterl = bp;
+				++clstats[newbp->b_bcount/PAGE_SIZE];
 
 				/*
 				 * enter the new bp onto the device queue
@@ -635,6 +702,7 @@ insert:
 				 * free the old kva
 				 */
 				vm_bounce_kva_free( orig1begin, ap->b_bufsize, 0);
+				--clstats[ap->b_bcount/PAGE_SIZE];
 
 				ap->b_un.b_addr = (caddr_t) kvanew;
 
@@ -645,6 +713,7 @@ insert:
 
 				ap->b_bcount += bp->b_bcount;
 				ap->b_bufsize = ap->b_bcount;
+				++clstats[ap->b_bcount/PAGE_SIZE];
 			}
 			return;
 		/*
@@ -657,8 +726,9 @@ insert:
 		 *	5) And the total I/O size would be below the maximum.
 		 */
 		} else if( ap->av_forw &&
-			(bp->b_blkno + (bp->b_bcount / DEV_BSIZE) == ap->av_forw->b_blkno) &&
+			(bp->b_pblkno + (bp->b_bcount / DEV_BSIZE) == ap->av_forw->b_pblkno) &&
 			(bp->b_flags == (ap->av_forw->b_flags & ~B_CLUSTER)) &&
+			((ap->av_forw->b_flags & B_BAD) == 0) &&
 			((ap->av_forw->b_bcount & PAGE_MASK) == 0) &&
 			(((vm_offset_t) ap->av_forw->b_un.b_addr & PAGE_MASK) == 0) &&
 			(ap->av_forw->b_bcount + bp->b_bcount < maxio)) {
@@ -678,7 +748,6 @@ insert:
 				goto nocluster;
 			}
 			
-
 			/*
 			 * if next isn't a cluster we need to create one
 			 */
@@ -694,18 +763,18 @@ insert:
 				}
 
 				cldiskvamerge( kvanew, orig1begin, orig1pages, orig2begin, orig2pages);
-
-				pmap_update();
-
 				ap = ap->av_forw;
+				--clstats[ap->b_bcount/PAGE_SIZE];
 				*newbp = *ap;
 				newbp->b_flags |= B_CLUSTER;
 				newbp->b_un.b_addr = (caddr_t) kvanew;
 				newbp->b_blkno = bp->b_blkno;
+				newbp->b_pblkno = bp->b_pblkno;
 				newbp->b_bcount += bp->b_bcount;
 				newbp->b_bufsize = newbp->b_bcount;
 				newbp->b_clusterf = bp;
 				newbp->b_clusterl = ap;
+				++clstats[newbp->b_bcount/PAGE_SIZE];
 
 				if( ap->av_forw)
 					ap->av_forw->av_back = newbp;
@@ -734,10 +803,13 @@ insert:
 				ap->b_clusterf->av_back = bp;
 				ap->b_clusterf = bp;
 				bp->av_back = NULL;
+				--clstats[ap->b_bcount/PAGE_SIZE];
 			
 				ap->b_blkno = bp->b_blkno;
+				ap->b_pblkno = bp->b_pblkno;
 				ap->b_bcount += bp->b_bcount;
 				ap->b_bufsize = ap->b_bcount;
+				++clstats[ap->b_bcount/PAGE_SIZE];
 
 			}
 			return;
@@ -747,6 +819,7 @@ insert:
 	 * don't merge
 	 */
 nocluster:
+	++clstats[bp->b_bcount/PAGE_SIZE];
 	bp->av_forw = ap->av_forw;
 	if( bp->av_forw)
 		bp->av_forw->av_back = bp;