o s/vhpt_size/pmap_vhpt_log2size/g

o  s/vhpt_base/pmap_vhpt_base/g
o  s/vhpt_bucket/pmap_vhpt_bucket/g
o  Declare the above in <machine/pmap.h>
o  Move the vm.stats.vhpt.* sysctls to machdep.vhpt.*
o  Create a tunable machdep.vhpt.log2size, with corresponding sysctl.
   The tunable allows the user to specify the VHPT size from the loader.
o  Don't keep track of the number of PTEs in the VHPT. Calculate the
   population when necessary by iterating the buckets and summing up
   the length of the buckets.
o  Don't perform the tpa instruction with a bucket lock held. The
   instruction can (theoretically) fault and locking is not needed.

3 files changed, 88 insertions, 48 deletions

diff --git a/sys/ia64/ia64/mp_machdep.c b/sys/ia64/ia64/mp_machdep.c
index a045ff5..dedc395 100644
--- a/sys/ia64/ia64/mp_machdep.c
+++ b/sys/ia64/ia64/mp_machdep.c
@@ -64,8 +64,6 @@ MALLOC_DECLARE(M_PMAP);
 
 void ia64_ap_startup(void);
 
-extern uint64_t vhpt_base[];
-extern size_t vhpt_size;
 extern uint64_t ia64_lapic_address;
 
 #define	LID_SAPIC_ID(x)		((int)((x) >> 24) & 0xff)
@@ -93,7 +91,7 @@ ia64_ap_startup(void)
 	ia64_set_k4((intptr_t)pcpup);
 
 	__asm __volatile("mov cr.pta=%0;; srlz.i;;" ::
-	    "r" (ap_vhpt + (1<<8) + (vhpt_size<<2) + 1));
+	    "r" (ap_vhpt + (1<<8) + (pmap_vhpt_log2size<<2) + 1));
 
 	ap_awake = 1;
 	ap_delay = 0;
@@ -248,7 +246,7 @@ cpu_mp_start()
 			ap_pcpu = pc;
 			ap_stack = malloc(KSTACK_PAGES * PAGE_SIZE, M_PMAP,
 			    M_WAITOK);
-			ap_vhpt = vhpt_base[pc->pc_cpuid];
+			ap_vhpt = pmap_vhpt_base[pc->pc_cpuid];
 			ap_delay = 2000;
 			ap_awake = 0;
 
diff --git a/sys/ia64/ia64/pmap.c b/sys/ia64/ia64/pmap.c
index b7293c1..eff0598 100644
--- a/sys/ia64/ia64/pmap.c
+++ b/sys/ia64/ia64/pmap.c
@@ -148,22 +148,21 @@ MALLOC_DEFINE(M_PMAP, "PMAP", "PMAP Structures");
 #define	pmap_set_wired(lpte)		(lpte)->pte |= PTE_WIRED
 
 /*
- * Statically allocated kernel pmap
+ * The VHPT bucket head structure.
  */
-struct pmap kernel_pmap_store;
-
-vm_offset_t virtual_avail;	/* VA of first avail page (after kernel bss) */
-vm_offset_t virtual_end;	/* VA of last avail page (end of kernel AS) */
-
 struct ia64_bucket {
 	uint64_t	chain;
 	struct mtx	mutex;
 	u_int		length;
 };
 
-struct ia64_bucket *vhpt_bucket;
-uint64_t vhpt_base[MAXCPU];
-size_t vhpt_size;
+/*
+ * Statically allocated kernel pmap
+ */
+struct pmap kernel_pmap_store;
+
+vm_offset_t virtual_avail;	/* VA of first avail page (after kernel bss) */
+vm_offset_t virtual_end;	/* VA of last avail page (end of kernel AS) */
 
 /*
  * Kernel virtual memory management.
@@ -209,16 +208,31 @@ int pmap_pagedaemon_waken;
 static uma_zone_t ptezone;
 
 /*
- * VHPT instrumentation.
+ * Virtual Hash Page Table (VHPT) data.
  */
+/* SYSCTL_DECL(_machdep); */
+SYSCTL_NODE(_machdep, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
+
+struct ia64_bucket *pmap_vhpt_bucket;
+
+int pmap_vhpt_nbuckets;
+SYSCTL_INT(_machdep_vhpt, OID_AUTO, nbuckets, CTLFLAG_RD,
+    &pmap_vhpt_nbuckets, 0, "");
+
+uint64_t pmap_vhpt_base[MAXCPU];
+
+int pmap_vhpt_log2size = 0;
+TUNABLE_INT("machdep.vhpt.log2size", &pmap_vhpt_log2size);
+SYSCTL_INT(_machdep_vhpt, OID_AUTO, log2size, CTLFLAG_RD,
+    &pmap_vhpt_log2size, 0, "");
+
 static int pmap_vhpt_inserts;
-static int pmap_vhpt_resident;
-SYSCTL_DECL(_vm_stats);
-SYSCTL_NODE(_vm_stats, OID_AUTO, vhpt, CTLFLAG_RD, 0, "");
-SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, inserts, CTLFLAG_RD,
-	   &pmap_vhpt_inserts, 0, "");
-SYSCTL_INT(_vm_stats_vhpt, OID_AUTO, resident, CTLFLAG_RD,
-	   &pmap_vhpt_resident, 0, "");
+SYSCTL_INT(_machdep_vhpt, OID_AUTO, inserts, CTLFLAG_RD,
+    &pmap_vhpt_inserts, 0, "");
+
+static int pmap_vhpt_population(SYSCTL_HANDLER_ARGS);
+SYSCTL_PROC(_machdep_vhpt, OID_AUTO, population, CTLTYPE_INT | CTLFLAG_RD,
+    NULL, 0, pmap_vhpt_population, "I", "");
 
 static PMAP_INLINE void	free_pv_entry(pv_entry_t pv);
 static pv_entry_t get_pv_entry(void);
@@ -353,17 +367,27 @@ pmap_bootstrap()
 	 * physical memory and try to locate a region which is large
 	 * enough to contain the VHPT (which must be a power of two in
 	 * size and aligned to a natural boundary).
+	 * We silently bump up the VHPT size to the minimum size if the
+	 * user has set the tunable too small. Likewise, the VHPT size
+	 * is silently capped to the maximum allowed.
 	 */
-	vhpt_size = 15;
-	size = 1UL << vhpt_size;
-	while (size < Maxmem * 32) {
-		vhpt_size++;
-		size <<= 1;
-	}
+	TUNABLE_INT_FETCH("machdep.vhpt.log2size", &pmap_vhpt_log2size);
+	if (pmap_vhpt_log2size == 0) {
+		pmap_vhpt_log2size = 15;
+		size = 1UL << pmap_vhpt_log2size;
+		while (size < Maxmem * 32) {
+			pmap_vhpt_log2size++;
+			size <<= 1;
+		}
+	} else if (pmap_vhpt_log2size < 15)
+		pmap_vhpt_log2size = 15;
+	if (pmap_vhpt_log2size > 61)
+		pmap_vhpt_log2size = 61;
 
-	vhpt_base[0] = 0;
+	pmap_vhpt_base[0] = 0;
 	base = limit = 0;
-	while (vhpt_base[0] == 0) {
+	size = 1UL << pmap_vhpt_log2size;
+	while (pmap_vhpt_base[0] == 0) {
 		if (bootverbose)
 			printf("Trying VHPT size 0x%lx\n", size);
 		for (i = 0; i < count; i += 2) {
@@ -377,12 +401,12 @@ pmap_bootstrap()
 		}
 		if (!phys_avail[i]) {
 			/* Can't fit, try next smaller size. */
-			vhpt_size--;
+			pmap_vhpt_log2size--;
 			size >>= 1;
 		} else
-			vhpt_base[0] = IA64_PHYS_TO_RR7(base);
+			pmap_vhpt_base[0] = IA64_PHYS_TO_RR7(base);
 	}
-	if (vhpt_size < 15)
+	if (pmap_vhpt_log2size < 15)
 		panic("Can't find space for VHPT");
 
 	if (bootverbose)
@@ -402,27 +426,29 @@ pmap_bootstrap()
 	} else
 		phys_avail[i] = limit;
 
-	count = size / sizeof(struct ia64_lpte);
+	pmap_vhpt_nbuckets = size / sizeof(struct ia64_lpte);
 
-	vhpt_bucket = (void *)pmap_steal_memory(count * sizeof(struct ia64_bucket));
-	pte = (struct ia64_lpte *)vhpt_base[0];
-	for (i = 0; i < count; i++) {
+	pmap_vhpt_bucket = (void *)pmap_steal_memory(pmap_vhpt_nbuckets *
+	    sizeof(struct ia64_bucket));
+	pte = (struct ia64_lpte *)pmap_vhpt_base[0];
+	for (i = 0; i < pmap_vhpt_nbuckets; i++) {
 		pte[i].pte = 0;
 		pte[i].itir = 0;
 		pte[i].tag = 1UL << 63;	/* Invalid tag */
-		pte[i].chain = (uintptr_t)(vhpt_bucket + i);
+		pte[i].chain = (uintptr_t)(pmap_vhpt_bucket + i);
 		/* Stolen memory is zeroed! */
-		mtx_init(&vhpt_bucket[i].mutex, "VHPT bucket lock", NULL,
+		mtx_init(&pmap_vhpt_bucket[i].mutex, "VHPT bucket lock", NULL,
 		    MTX_SPIN);
 	}
 
 	for (i = 1; i < MAXCPU; i++) {
-		vhpt_base[i] = vhpt_base[i - 1] + size;
-		bcopy((void *)vhpt_base[i - 1], (void *)vhpt_base[i], size);
+		pmap_vhpt_base[i] = pmap_vhpt_base[i - 1] + size;
+		bcopy((void *)pmap_vhpt_base[i - 1], (void *)pmap_vhpt_base[i],
+		    size);
 	}
 
 	__asm __volatile("mov cr.pta=%0;; srlz.i;;" ::
-	    "r" (vhpt_base[0] + (1<<8) + (vhpt_size<<2) + 1));
+	    "r" (pmap_vhpt_base[0] + (1<<8) + (pmap_vhpt_log2size<<2) + 1));
 
 	virtual_avail = VM_MIN_KERNEL_ADDRESS;
 	virtual_end = VM_MAX_KERNEL_ADDRESS;
@@ -460,6 +486,19 @@ pmap_bootstrap()
 	map_gateway_page();
 }
 
+static int
+pmap_vhpt_population(SYSCTL_HANDLER_ARGS)
+{
+	int count, error, i;
+
+	count = 0;
+	for (i = 0; i < pmap_vhpt_nbuckets; i++)
+		count += pmap_vhpt_bucket[i].length;
+
+	error = SYSCTL_OUT(req, &count, sizeof(count));
+	return (error);
+}
+
 /*
  *	Initialize a vm_page's machine-dependent fields.
  */
@@ -568,10 +607,10 @@ pmap_invalidate_page(pmap_t pmap, vm_offset_t va)
 	KASSERT((pmap == kernel_pmap || pmap == PCPU_GET(current_pmap)),
 		("invalidating TLB for non-current pmap"));
 
-	vhpt_ofs = ia64_thash(va) - vhpt_base[PCPU_GET(cpuid)];
+	vhpt_ofs = ia64_thash(va) - pmap_vhpt_base[PCPU_GET(cpuid)];
 	critical_enter();
 	for (i = 0; i < MAXCPU; i++) {
-		pte = (struct ia64_lpte *)(vhpt_base[i] + vhpt_ofs);
+		pte = (struct ia64_lpte *)(pmap_vhpt_base[i] + vhpt_ofs);
 		if (pte->tag == ia64_ttag(va))
 			pte->tag = 1UL << 63;
 	}
@@ -799,9 +838,10 @@ pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va)
 {
 	struct ia64_bucket *bckt;
 	struct ia64_lpte *vhpte;
+	uint64_t pte_pa;
 
-	pmap_vhpt_inserts++;
-	pmap_vhpt_resident++;
+	/* Can fault, so get it out of the way. */
+	pte_pa = ia64_tpa((vm_offset_t)pte);
 
 	vhpte = (struct ia64_lpte *)ia64_thash(va);
 	bckt = (struct ia64_bucket *)vhpte->chain;
@@ -809,9 +849,9 @@ pmap_enter_vhpt(struct ia64_lpte *pte, vm_offset_t va)
 	mtx_lock_spin(&bckt->mutex);
 	pte->chain = bckt->chain;
 	ia64_mf();
-	bckt->chain = ia64_tpa((vm_offset_t)pte);
-	ia64_mf();
+	bckt->chain = pte_pa;
 
+	pmap_vhpt_inserts++;
 	bckt->length++;
 	mtx_unlock_spin(&bckt->mutex);
 }
@@ -856,7 +896,6 @@ pmap_remove_vhpt(vm_offset_t va)
 
 	bckt->length--;
 	mtx_unlock_spin(&bckt->mutex);
-	pmap_vhpt_resident--;
 	return (0);
 }
 
diff --git a/sys/ia64/include/pmap.h b/sys/ia64/include/pmap.h
index b1f45e8..173d714 100644
--- a/sys/ia64/include/pmap.h
+++ b/sys/ia64/include/pmap.h
@@ -115,6 +115,9 @@ extern vm_offset_t phys_avail[];
 extern vm_offset_t virtual_avail;
 extern vm_offset_t virtual_end;
 
+extern uint64_t pmap_vhpt_base[];
+extern int pmap_vhpt_log2size;
+
 #define	pmap_page_is_mapped(m)	(!TAILQ_EMPTY(&(m)->md.pv_list))
 
 vm_offset_t pmap_steal_memory(vm_size_t);