Integrate, but do not enable support for dynamically resizing TSBs

4 files changed, 114 insertions, 1 deletions

diff --git a/sys/sun4v/include/pmap.h b/sys/sun4v/include/pmap.h
index ea40fdd..3115fff 100644
--- a/sys/sun4v/include/pmap.h
+++ b/sys/sun4v/include/pmap.h
@@ -46,7 +46,13 @@
 #include <machine/cache.h>
 #include <machine/hypervisorvar.h>
 
+#define TSB_INIT_SHIFT          3
 #define	PMAP_CONTEXT_MAX	8192
+/* 
+ *  We don't want TSBs getting above 1MB - which is enough 
+ *  for a working set of 512MB - revisit in the future 
+ */
+#define TSB_MAX_RESIZE          (20 - TSB_INIT_SHIFT - PAGE_SHIFT)
 
 typedef	struct pmap *pmap_t;
 typedef uint32_t pmap_cpumask_t;
@@ -75,6 +81,7 @@ struct pmap {
 	struct	pmap_statistics pm_stats;
 	uint32_t                pm_tsb_miss_count;
 	uint32_t                pm_tsb_cap_miss_count;
+	vm_paddr_t              pm_old_tsb_pa[TSB_MAX_RESIZE];
 };
 
 #define	PMAP_LOCK(pmap)		mtx_lock(&(pmap)->pm_mtx)
diff --git a/sys/sun4v/include/tsb.h b/sys/sun4v/include/tsb.h
index 6a24718..a293707 100644
--- a/sys/sun4v/include/tsb.h
+++ b/sys/sun4v/include/tsb.h
@@ -51,7 +51,6 @@ extern hv_tsb_info_t kernel_td[MAX_TSB_INFO];
 
 struct hv_tsb_info;
 
-#define TSB_INIT_SHIFT        3
 
 void tsb_init(struct hv_tsb_info *tsb, uint64_t *scratchval, uint64_t page_shift);
 
diff --git a/sys/sun4v/sun4v/pmap.c b/sys/sun4v/sun4v/pmap.c
index 9372fb8..c91be02 100644
--- a/sys/sun4v/sun4v/pmap.c
+++ b/sys/sun4v/sun4v/pmap.c
@@ -195,10 +195,16 @@ static void pmap_insert_entry(pmap_t pmap, vm_offset_t va, vm_page_t m);
 static void pmap_remove_entry(struct pmap *pmap, vm_page_t m, vm_offset_t va);
 static void pmap_remove_tte(pmap_t pmap, tte_t tte_data, vm_offset_t va);
 static void pmap_enter_quick_locked(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot);
+static void pmap_tsb_reset(pmap_t pmap);
+static void pmap_tsb_resize(pmap_t pmap);
 static void pmap_tte_hash_resize(pmap_t pmap);
 
 void pmap_set_ctx_panic(uint64_t error, vm_paddr_t tsb_ra, pmap_t pmap);
 
+struct tsb_resize_info {
+	uint64_t tri_tsbscratch;
+	uint64_t tri_tsb_ra;
+};
 
 /*
  * Quick sort callout for comparing memory regions.
@@ -409,6 +415,8 @@ pmap_activate(struct thread *td)
 
 	pmap->pm_hashscratch = tte_hash_set_scratchpad_user(pmap->pm_hash, pmap->pm_context);
 	pmap->pm_tsbscratch = tsb_set_scratchpad_user(&pmap->pm_tsb);
+	pmap->pm_tsb_miss_count = pmap->pm_tsb_cap_miss_count = 0;
+
 	PCPU_SET(curpmap, pmap);
 	if (pmap->pm_context != 0)
 		if ((err = hv_set_ctxnon0(1, pmap->pm_tsb_ra)) != H_EOK)
@@ -1056,6 +1064,13 @@ pmap_enter(pmap_t pmap, vm_offset_t va, vm_page_t m, vm_prot_t prot,
 		pmap->pm_tsb_cap_miss_count = 0;
 	}
 #endif
+	/*
+	 * 512 is an arbitrary number of tsb misses
+	 */
+	if (0 && pmap->pm_context != 0 && pmap->pm_tsb_miss_count > 512)
+		pmap_tsb_resize(pmap);
+
+
 	vm_page_unlock_queues();
 
 
@@ -1624,6 +1639,7 @@ pmap_pinit0(pmap_t pmap)
 void
 pmap_pinit(pmap_t pmap)
 {
+	int i;
 
 	pmap->pm_context = get_context();
 	pmap->pm_tsb_ra = vtophys(&pmap->pm_tsb);
@@ -1632,7 +1648,11 @@ pmap_pinit(pmap_t pmap)
 	pmap->pm_hash = tte_hash_create(pmap->pm_context, &pmap->pm_hashscratch);
 	tsb_init(&pmap->pm_tsb, &pmap->pm_tsbscratch, TSB_INIT_SHIFT);
 	vm_page_unlock_queues();
+	pmap->pm_tsb_miss_count = pmap->pm_tsb_cap_miss_count = 0;
 	pmap->pm_active = pmap->pm_tlbactive = 0;
+	for (i = 0; i < TSB_MAX_RESIZE; i++)
+		pmap->pm_old_tsb_pa[i] = 0;
+
 	TAILQ_INIT(&pmap->pm_pvlist);
 	PMAP_LOCK_INIT(pmap);
 	bzero(&pmap->pm_stats, sizeof pmap->pm_stats);
@@ -1925,6 +1945,9 @@ pmap_remove_pages(pmap_t pmap)
 		free_pv_entry(pv);
 	}
 	pmap->pm_hash = tte_hash_reset(pmap->pm_hash, &pmap->pm_hashscratch);
+	if (0)
+		pmap_tsb_reset(pmap);
+
 	vm_page_unlock_queues();
 
 	pmap_invalidate_all(pmap);
@@ -1932,6 +1955,27 @@ pmap_remove_pages(pmap_t pmap)
 	PMAP_UNLOCK(pmap);
 }
 
+static void
+pmap_tsb_reset(pmap_t pmap)
+{
+	int i;
+
+	for (i = 1; i < TSB_MAX_RESIZE && pmap->pm_old_tsb_pa[i]; i++) {
+		pmap_free_contig_pages((void *)TLB_PHYS_TO_DIRECT(pmap->pm_old_tsb_pa[i]), 
+				       (1 << (TSB_INIT_SHIFT + i)));
+		pmap->pm_old_tsb_pa[i] = 0;
+	}
+	if (pmap->pm_old_tsb_pa[0] != 0) {
+		vm_paddr_t tsb_pa = pmap->pm_tsb.hvtsb_pa;
+		int size = tsb_size(&pmap->pm_tsb);
+		pmap->pm_tsb.hvtsb_ntte = (1 << (TSB_INIT_SHIFT + PAGE_SHIFT - TTE_SHIFT));
+		pmap->pm_tsb.hvtsb_pa = pmap->pm_old_tsb_pa[0];
+		pmap_free_contig_pages((void *)TLB_PHYS_TO_DIRECT(tsb_pa), size);
+		pmap->pm_tsbscratch = pmap->pm_tsb.hvtsb_pa | (uint64_t)TSB_INIT_SHIFT;
+		pmap->pm_old_tsb_pa[0] = 0;
+	}
+}
+
 void
 pmap_scrub_pages(vm_paddr_t pa, int64_t size)
 {
@@ -1970,6 +2014,57 @@ pmap_remove_tte(pmap_t pmap, tte_t tte_data, vm_offset_t va)
 	}
 }
 
+/* resize the tsb if the number of capacity misses is greater than 1/4 of
+ * the total 
+ */  
+static void
+pmap_tsb_resize(pmap_t pmap)
+{
+	uint32_t miss_count;
+	uint32_t cap_miss_count;
+	struct tsb_resize_info info;
+	hv_tsb_info_t hvtsb;
+	uint64_t tsbscratch;
+
+	KASSERT(pmap == PCPU_GET(curpmap), ("operating on non-current pmap"));
+	miss_count = pmap->pm_tsb_miss_count;
+	cap_miss_count = pmap->pm_tsb_cap_miss_count;
+	int npages_shift = tsb_page_shift(pmap);
+
+	if (npages_shift < (TSB_INIT_SHIFT + TSB_MAX_RESIZE) && 
+	    cap_miss_count > (miss_count >> 1)) {
+		DPRINTF("resizing tsb for proc=%s pid=%d\n", 
+			curthread->td_proc->p_comm, curthread->td_proc->p_pid);
+		pmap->pm_old_tsb_pa[npages_shift - TSB_INIT_SHIFT] = pmap->pm_tsb.hvtsb_pa;
+
+		/* double TSB size */
+		tsb_init(&hvtsb, &tsbscratch, npages_shift + 1);
+#ifdef SMP
+		spinlock_enter();
+		/* reset tsb */
+		bcopy(&hvtsb, &pmap->pm_tsb, sizeof(hv_tsb_info_t));
+		pmap->pm_tsbscratch = tsb_set_scratchpad_user(&pmap->pm_tsb);
+
+		if (hv_set_ctxnon0(1, pmap->pm_tsb_ra) != H_EOK)
+			panic("failed to set TSB 0x%lx - context == %ld\n", 
+			      pmap->pm_tsb_ra, pmap->pm_context);
+		info.tri_tsbscratch = pmap->pm_tsbscratch;
+		info.tri_tsb_ra = pmap->pm_tsb_ra;
+		pmap_ipi(pmap, tl_tsbupdate, pmap->pm_context, vtophys(&info));
+		pmap->pm_tlbactive = pmap->pm_active;
+		spinlock_exit();
+#else 
+		bcopy(&hvtsb, &pmap->pm_tsb, sizeof(hvtsb));
+		if (hv_set_ctxnon0(1, pmap->pm_tsb_ra) != H_EOK)
+			panic("failed to set TSB 0x%lx - context == %ld\n", 
+			      pmap->pm_tsb_ra, pmap->pm_context);
+		pmap->pm_tsbscratch = tsb_set_scratchpad_user(&pmap->pm_tsb);
+#endif		
+	}
+	pmap->pm_tsb_miss_count = 0;
+	pmap->pm_tsb_cap_miss_count = 0;
+}
+
 static void
 pmap_tte_hash_resize(pmap_t pmap)
 {
diff --git a/sys/sun4v/sun4v/tsb.c b/sys/sun4v/sun4v/tsb.c
index 1fc07ad..15824bb 100644
--- a/sys/sun4v/sun4v/tsb.c
+++ b/sys/sun4v/sun4v/tsb.c
@@ -273,3 +273,15 @@ tsb_set_scratchpad_user(hv_tsb_info_t *tsb)
 	membar(Sync);
 	return tsb_scratch;
 }
+
+int
+tsb_size(hv_tsb_info_t *hvtsb)
+{
+	return (hvtsb->hvtsb_ntte >> (PAGE_SHIFT - TTE_SHIFT));
+}
+
+int
+tsb_page_shift(pmap_t pmap)
+{
+	return (pmap->pm_tsbscratch & PAGE_MASK);
+}