The core of this patch is to vm/vm_page.h. The effects are two-fold: (1) to

eliminate an extra (useless) level of indirection in half of the page
queue accesses and (2) to use a single name for each queue throughout,
instead of, e.g., "vm_page_queue_active" in some places and
"vm_page_queues[PQ_ACTIVE]" in others.

Reviewed by:	dillon

3 files changed, 58 insertions, 78 deletions

diff --git a/sys/vm/vm_page.c b/sys/vm/vm_page.c
index 99a8fef..6dfac16 100644
--- a/sys/vm/vm_page.c
+++ b/sys/vm/vm_page.c
@@ -98,36 +98,23 @@ static int vm_page_bucket_count;	/* How big is array? */
 static int vm_page_hash_mask;		/* Mask for hash function */
 static volatile int vm_page_bucket_generation;
 
-struct pglist vm_page_queue_free[PQ_L2_SIZE] = {{0}};
-struct pglist vm_page_queue_active = {0};
-struct pglist vm_page_queue_inactive = {0};
-struct pglist vm_page_queue_cache[PQ_L2_SIZE] = {{0}};
-
-struct vpgqueues vm_page_queues[PQ_COUNT] = {{0}};
+struct vpgqueues vm_page_queues[PQ_COUNT];
 
 static void
 vm_page_queue_init(void) {
 	int i;
 
 	for(i=0;i<PQ_L2_SIZE;i++) {
-		vm_page_queues[PQ_FREE+i].pl = &vm_page_queue_free[i];
 		vm_page_queues[PQ_FREE+i].cnt = &cnt.v_free_count;
 	}
-	vm_page_queues[PQ_INACTIVE].pl = &vm_page_queue_inactive;
 	vm_page_queues[PQ_INACTIVE].cnt = &cnt.v_inactive_count;
 
-	vm_page_queues[PQ_ACTIVE].pl = &vm_page_queue_active;
 	vm_page_queues[PQ_ACTIVE].cnt = &cnt.v_active_count;
 	for(i=0;i<PQ_L2_SIZE;i++) {
-		vm_page_queues[PQ_CACHE+i].pl = &vm_page_queue_cache[i];
 		vm_page_queues[PQ_CACHE+i].cnt = &cnt.v_cache_count;
 	}
-	for(i=PQ_FREE;i<PQ_COUNT;i++) {
-		if (vm_page_queues[i].pl) {
-			TAILQ_INIT(vm_page_queues[i].pl);
-		} else {
-			panic("vm_page_queue_init: queue %d is null", i);
-		}
+	for(i=0;i<PQ_COUNT;i++) {
+		TAILQ_INIT(&vm_page_queues[i].pl);
 	}
 }
 
@@ -316,7 +303,7 @@ vm_page_startup(starta, enda, vaddr)
 			m->flags = 0;
 			m->pc = (pa >> PAGE_SHIFT) & PQ_L2_MASK;
 			m->queue = m->pc + PQ_FREE;
-			TAILQ_INSERT_HEAD(vm_page_queues[m->queue].pl, m, pageq);
+			TAILQ_INSERT_HEAD(&vm_page_queues[m->queue].pl, m, pageq);
 			vm_page_queues[m->queue].lcnt++;
 			pa += PAGE_SIZE;
 		}
@@ -586,7 +573,7 @@ vm_page_unqueue_nowakeup(m)
 	if (queue != PQ_NONE) {
 		pq = &vm_page_queues[queue];
 		m->queue = PQ_NONE;
-		TAILQ_REMOVE(pq->pl, m, pageq);
+		TAILQ_REMOVE(&pq->pl, m, pageq);
 		(*pq->cnt)--;
 		pq->lcnt--;
 	}
@@ -610,7 +597,7 @@ vm_page_unqueue(m)
 	if (queue != PQ_NONE) {
 		m->queue = PQ_NONE;
 		pq = &vm_page_queues[queue];
-		TAILQ_REMOVE(pq->pl, m, pageq);
+		TAILQ_REMOVE(&pq->pl, m, pageq);
 		(*pq->cnt)--;
 		pq->lcnt--;
 		if ((queue - m->pc) == PQ_CACHE) {
@@ -656,10 +643,10 @@ _vm_page_list_find(basequeue, index)
 	 */
 
 	for(i = PQ_L2_SIZE / 2; i > 0; --i) {
-		if ((m = TAILQ_FIRST(pq[(index + i) & PQ_L2_MASK].pl)) != NULL)
+		if ((m = TAILQ_FIRST(&pq[(index + i) & PQ_L2_MASK].pl)) != NULL)
 			break;
 
-		if ((m = TAILQ_FIRST(pq[(index - i) & PQ_L2_MASK].pl)) != NULL)
+		if ((m = TAILQ_FIRST(&pq[(index - i) & PQ_L2_MASK].pl)) != NULL)
 			break;
 	}
 	return(m);
@@ -833,7 +820,7 @@ loop:
 	{
 		struct vpgqueues *pq = &vm_page_queues[m->queue];
 
-		TAILQ_REMOVE(pq->pl, m, pageq);
+		TAILQ_REMOVE(&pq->pl, m, pageq);
 		(*pq->cnt)--;
 		pq->lcnt--;
 	}
@@ -1009,7 +996,7 @@ vm_page_activate(m)
 		if (m->wire_count == 0) {
 			m->queue = PQ_ACTIVE;
 			vm_page_queues[PQ_ACTIVE].lcnt++;
-			TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
+			TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 			if (m->act_count < ACT_INIT)
 				m->act_count = ACT_INIT;
 			cnt.v_active_count++;
@@ -1158,10 +1145,10 @@ vm_page_free_toq(vm_page_t m)
 	 */
 
 	if (m->flags & PG_ZERO) {
-		TAILQ_INSERT_TAIL(pq->pl, m, pageq);
+		TAILQ_INSERT_TAIL(&pq->pl, m, pageq);
 		++vm_page_zero_count;
 	} else {
-		TAILQ_INSERT_HEAD(pq->pl, m, pageq);
+		TAILQ_INSERT_HEAD(&pq->pl, m, pageq);
 	}
 
 	vm_page_free_wakeup();
@@ -1234,12 +1221,12 @@ vm_page_unwire(m, activate)
 		if (m->wire_count == 0) {
 			cnt.v_wire_count--;
 			if (activate) {
-				TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
+				TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 				m->queue = PQ_ACTIVE;
 				vm_page_queues[PQ_ACTIVE].lcnt++;
 				cnt.v_active_count++;
 			} else {
-				TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
+				TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 				m->queue = PQ_INACTIVE;
 				vm_page_queues[PQ_INACTIVE].lcnt++;
 				cnt.v_inactive_count++;
@@ -1281,9 +1268,9 @@ _vm_page_deactivate(vm_page_t m, int athead)
 			cnt.v_reactivated++;
 		vm_page_unqueue(m);
 		if (athead)
-			TAILQ_INSERT_HEAD(&vm_page_queue_inactive, m, pageq);
+			TAILQ_INSERT_HEAD(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 		else
-			TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
+			TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 		m->queue = PQ_INACTIVE;
 		vm_page_queues[PQ_INACTIVE].lcnt++;
 		cnt.v_inactive_count++;
@@ -1335,7 +1322,7 @@ vm_page_cache(m)
 	vm_page_unqueue_nowakeup(m);
 	m->queue = PQ_CACHE + m->pc;
 	vm_page_queues[m->queue].lcnt++;
-	TAILQ_INSERT_TAIL(vm_page_queues[m->queue].pl, m, pageq);
+	TAILQ_INSERT_TAIL(&vm_page_queues[m->queue].pl, m, pageq);
 	cnt.v_cache_count++;
 	vm_page_free_wakeup();
 	splx(s);
@@ -1746,7 +1733,7 @@ again:
 			vm_page_t m, next;
 
 again1:
-			for (m = TAILQ_FIRST(&vm_page_queue_inactive);
+			for (m = TAILQ_FIRST(&vm_page_queues[PQ_INACTIVE].pl);
 				m != NULL;
 				m = next) {
 
@@ -1773,7 +1760,7 @@ again1:
 					vm_page_cache(m);
 			}
 
-			for (m = TAILQ_FIRST(&vm_page_queue_active);
+			for (m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
 				m != NULL;
 				m = next) {
 
@@ -1829,7 +1816,7 @@ again1:
 				vm_page_free(m);
 			}
 
-			TAILQ_REMOVE(vm_page_queues[m->queue].pl, m, pageq);
+			TAILQ_REMOVE(&vm_page_queues[m->queue].pl, m, pageq);
 			vm_page_queues[m->queue].lcnt--;
 			cnt.v_free_count--;
 			m->valid = VM_PAGE_BITS_ALL;
diff --git a/sys/vm/vm_page.h b/sys/vm/vm_page.h
index 2d7e740..0cfc618 100644
--- a/sys/vm/vm_page.h
+++ b/sys/vm/vm_page.h
@@ -210,11 +210,13 @@ struct vm_page {
 #define PQ_COUNT	(2 + 2*PQ_L2_SIZE)
 #endif
 
-extern struct vpgqueues {
-	struct pglist *pl;
+struct vpgqueues {
+	struct pglist pl;
 	int	*cnt;
 	int	lcnt;
-} vm_page_queues[PQ_COUNT];
+};
+
+extern struct vpgqueues vm_page_queues[PQ_COUNT];
 
 #endif
 
@@ -272,15 +274,6 @@ extern struct vpgqueues {
  *
  */
 
-#if !defined(KLD_MODULE)
-
-extern struct pglist vm_page_queue_free[PQ_L2_SIZE];/* memory free queue */
-extern struct pglist vm_page_queue_active;	/* active memory queue */
-extern struct pglist vm_page_queue_inactive;	/* inactive memory queue */
-extern struct pglist vm_page_queue_cache[PQ_L2_SIZE];/* cache memory queue */
-
-#endif
-
 extern int vm_page_zero_count;
 
 extern vm_page_t vm_page_array;		/* First resident page in table */
@@ -595,17 +588,17 @@ vm_page_list_find(int basequeue, int index, boolean_t prefer_zero)
 
 #if PQ_L2_SIZE > 1
 	if (prefer_zero) {
-		m = TAILQ_LAST(vm_page_queues[basequeue+index].pl, pglist);
+		m = TAILQ_LAST(&vm_page_queues[basequeue+index].pl, pglist);
 	} else {
-		m = TAILQ_FIRST(vm_page_queues[basequeue+index].pl);
+		m = TAILQ_FIRST(&vm_page_queues[basequeue+index].pl);
 	}
 	if (m == NULL)
 		m = _vm_page_list_find(basequeue, index);
 #else
 	if (prefer_zero) {
-		m = TAILQ_LAST(vm_page_queues[basequeue].pl, pglist);
+		m = TAILQ_LAST(&vm_page_queues[basequeue].pl, pglist);
 	} else {
-		m = TAILQ_FIRST(vm_page_queues[basequeue].pl);
+		m = TAILQ_FIRST(&vm_page_queues[basequeue].pl);
 	}
 #endif
 	return(m);
diff --git a/sys/vm/vm_pageout.c b/sys/vm/vm_pageout.c
index fd5f956..8f930af 100644
--- a/sys/vm/vm_pageout.c
+++ b/sys/vm/vm_pageout.c
@@ -500,8 +500,8 @@ vm_pageout_object_deactivate_pages(map, object, desired, map_remove_only)
 						vm_page_deactivate(p);
 					} else {
 						s = splvm();
-						TAILQ_REMOVE(&vm_page_queue_active, p, pageq);
-						TAILQ_INSERT_TAIL(&vm_page_queue_active, p, pageq);
+						TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
+						TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
 						splx(s);
 					}
 				} else {
@@ -510,8 +510,8 @@ vm_pageout_object_deactivate_pages(map, object, desired, map_remove_only)
 					if (p->act_count < (ACT_MAX - ACT_ADVANCE))
 						p->act_count += ACT_ADVANCE;
 					s = splvm();
-					TAILQ_REMOVE(&vm_page_queue_active, p, pageq);
-					TAILQ_INSERT_TAIL(&vm_page_queue_active, p, pageq);
+					TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
+					TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, p, pageq);
 					splx(s);
 				}
 			} else if (p->queue == PQ_INACTIVE) {
@@ -674,7 +674,7 @@ vm_pageout_scan()
 rescan0:
 	addl_page_shortage = addl_page_shortage_init;
 	maxscan = cnt.v_inactive_count;
-	for (m = TAILQ_FIRST(&vm_page_queue_inactive);
+	for (m = TAILQ_FIRST(&vm_page_queues[PQ_INACTIVE].pl);
 	     m != NULL && maxscan-- > 0 && page_shortage > 0;
 	     m = next) {
 
@@ -688,8 +688,8 @@ rescan0:
 
 		if (m->hold_count) {
 			s = splvm();
-			TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
-			TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
+			TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
+			TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 			splx(s);
 			addl_page_shortage++;
 			continue;
@@ -794,8 +794,8 @@ rescan0:
 			 */
 			if (!swap_pageouts_ok || (object->flags & OBJ_DEAD)) {
 				s = splvm();
-				TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
-				TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
+				TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
+				TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 				splx(s);
 				continue;
 			}
@@ -816,8 +816,8 @@ rescan0:
 			    object->type != OBJT_SWAP &&
 			    cnt.v_free_count < cnt.v_free_reserved) {
 				s = splvm();
-				TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
-				TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m,
+				TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
+				TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m,
 				    pageq);
 				splx(s);
 				continue;
@@ -860,8 +860,8 @@ rescan0:
 						(m->busy == 0) &&
 						(m->flags & PG_BUSY) == 0) {
 						s = splvm();
-						TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
-						TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
+						TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
+						TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 						splx(s);
 					}
 					if (object->flags & OBJ_MIGHTBEDIRTY)
@@ -895,8 +895,8 @@ rescan0:
 				 */
 				if (m->hold_count) {
 					s = splvm();
-					TAILQ_REMOVE(&vm_page_queue_inactive, m, pageq);
-					TAILQ_INSERT_TAIL(&vm_page_queue_inactive, m, pageq);
+					TAILQ_REMOVE(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
+					TAILQ_INSERT_TAIL(&vm_page_queues[PQ_INACTIVE].pl, m, pageq);
 					splx(s);
 					if (object->flags & OBJ_MIGHTBEDIRTY)
 						vnodes_skipped++;
@@ -946,7 +946,7 @@ rescan0:
 	 */
 
 	pcount = cnt.v_active_count;
-	m = TAILQ_FIRST(&vm_page_queue_active);
+	m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
 
 	while ((m != NULL) && (pcount-- > 0) && (page_shortage > 0)) {
 
@@ -966,8 +966,8 @@ rescan0:
 		    (m->flags & PG_BUSY) ||
 		    (m->hold_count != 0)) {
 			s = splvm();
-			TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
-			TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
+			TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
+			TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 			splx(s);
 			m = next;
 			continue;
@@ -1006,8 +1006,8 @@ rescan0:
 		 */
 		if (actcount && (m->object->ref_count != 0)) {
 			s = splvm();
-			TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
-			TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
+			TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
+			TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 			splx(s);
 		} else {
 			m->act_count -= min(m->act_count, ACT_DECLINE);
@@ -1025,8 +1025,8 @@ rescan0:
 				}
 			} else {
 				s = splvm();
-				TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
-				TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
+				TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
+				TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 				splx(s);
 			}
 		}
@@ -1167,7 +1167,7 @@ vm_pageout_page_stats()
 			pcount = tpcount;
 	}
 
-	m = TAILQ_FIRST(&vm_page_queue_active);
+	m = TAILQ_FIRST(&vm_page_queues[PQ_ACTIVE].pl);
 	while ((m != NULL) && (pcount-- > 0)) {
 		int actcount;
 
@@ -1183,8 +1183,8 @@ vm_pageout_page_stats()
 		    (m->flags & PG_BUSY) ||
 		    (m->hold_count != 0)) {
 			s = splvm();
-			TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
-			TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
+			TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
+			TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 			splx(s);
 			m = next;
 			continue;
@@ -1202,8 +1202,8 @@ vm_pageout_page_stats()
 			if (m->act_count > ACT_MAX)
 				m->act_count = ACT_MAX;
 			s = splvm();
-			TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
-			TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
+			TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
+			TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 			splx(s);
 		} else {
 			if (m->act_count == 0) {
@@ -1219,8 +1219,8 @@ vm_pageout_page_stats()
 			} else {
 				m->act_count -= min(m->act_count, ACT_DECLINE);
 				s = splvm();
-				TAILQ_REMOVE(&vm_page_queue_active, m, pageq);
-				TAILQ_INSERT_TAIL(&vm_page_queue_active, m, pageq);
+				TAILQ_REMOVE(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
+				TAILQ_INSERT_TAIL(&vm_page_queues[PQ_ACTIVE].pl, m, pageq);
 				splx(s);
 			}
 		}