Introduce superpages support for ARMv6/v7.

Promoting base pages to superpages can increase TLB coverage and allow for
efficient use of page table entries.  This development provides FreeBSD/ARM
with superpages management mechanism roughly equivalent to what we have for
i386 and amd64 architectures.

1. Add mechanism for automatic promotion of 4KB page mappings to 1MB section
   mappings (and demotion when not needed, respectively).

2. Managed and non-kernel mappings are now superpages-aware.

3. The functionality can be enabled by setting "vm.pmap.sp_enabled" tunable to
   a non-zero value (either in loader.conf or by modifying "sp_enabled"
   variable in pmap-v6.c file).  By default, automatic promotion is currently
   disabled.

Submitted by:	Zbigniew Bodek <zbb@semihalf.com>
Reviewed by:	alc
Sponsored by:	The FreeBSD Foundation, Semihalf

3 files changed, 43 insertions, 9 deletions

diff --git a/sys/arm/include/param.h b/sys/arm/include/param.h
index 9ffb118..15fc279 100644
--- a/sys/arm/include/param.h
+++ b/sys/arm/include/param.h
@@ -108,9 +108,10 @@
 
 #define PDR_SHIFT	20 /* log2(NBPDR) */
 #define NBPDR		(1 << PDR_SHIFT)
+#define PDRMASK		(NBPDR - 1)
 #define NPDEPG          (1 << (32 - PDR_SHIFT))
 
-#define	MAXPAGESIZES	1		/* maximum number of supported page sizes */
+#define	MAXPAGESIZES	2		/* maximum number of supported page sizes */
 
 #ifndef KSTACK_PAGES
 #define KSTACK_PAGES    2
@@ -133,8 +134,8 @@
  */
 #define	trunc_page(x)		((x) & ~PAGE_MASK)
 #define	round_page(x)		(((x) + PAGE_MASK) & ~PAGE_MASK)
-#define	trunc_4mpage(x)		((unsigned)(x) & ~PDRMASK)
-#define	round_4mpage(x)		((((unsigned)(x)) + PDRMASK) & ~PDRMASK)
+#define	trunc_1mpage(x)		((unsigned)(x) & ~PDRMASK)
+#define	round_1mpage(x)		((((unsigned)(x)) + PDRMASK) & ~PDRMASK)
 
 #define	atop(x)			((unsigned)(x) >> PAGE_SHIFT)
 #define	ptoa(x)			((unsigned)(x) << PAGE_SHIFT)
diff --git a/sys/arm/include/pmap.h b/sys/arm/include/pmap.h
index 780ed9b..207abd0 100644
--- a/sys/arm/include/pmap.h
+++ b/sys/arm/include/pmap.h
@@ -97,8 +97,12 @@ enum mem_type {
 #endif
 
 #define	pmap_page_get_memattr(m)	((m)->md.pv_memattr)
-#define	pmap_page_is_mapped(m)	(!TAILQ_EMPTY(&(m)->md.pv_list))
 #define	pmap_page_is_write_mapped(m)	(((m)->aflags & PGA_WRITEABLE) != 0)
+#if (ARM_MMU_V6 + ARM_MMU_V7) > 0
+boolean_t pmap_page_is_mapped(vm_page_t);
+#else
+#define	pmap_page_is_mapped(m)	(!TAILQ_EMPTY(&(m)->md.pv_list))
+#endif
 void pmap_page_set_memattr(vm_page_t m, vm_memattr_t ma);
 
 /*
@@ -258,7 +262,9 @@ void	*pmap_mapdev(vm_offset_t, vm_size_t);
 void	pmap_unmapdev(vm_offset_t, vm_size_t);
 vm_page_t	pmap_use_pt(pmap_t, vm_offset_t);
 void	pmap_debug(int);
+#if (ARM_MMU_V6 + ARM_MMU_V7) == 0
 void	pmap_map_section(vm_offset_t, vm_offset_t, vm_offset_t, int, int);
+#endif
 void	pmap_link_l2pt(vm_offset_t, vm_offset_t, struct pv_addr *);
 vm_size_t	pmap_map_chunk(vm_offset_t, vm_offset_t, vm_offset_t, vm_size_t, int, int);
 void
@@ -411,6 +417,24 @@ extern int pmap_needs_pte_sync;
 #define	L1_C_PROTO		(L1_TYPE_C)
 #define	L2_S_PROTO		(L2_TYPE_S)
 
+/*
+ * Promotion to a 1MB (SECTION) mapping requires that the corresponding
+ * 4KB (SMALL) page mappings have identical settings for the following fields:
+ */
+#define	L2_S_PROMOTE		(L2_S_REF | L2_SHARED | L2_S_PROT_MASK | \
+				 L2_XN | L2_S_PROTO)
+
+/*
+ * In order to compare 1MB (SECTION) entry settings with the 4KB (SMALL)
+ * page mapping it is necessary to read and shift appropriate bits from
+ * L1 entry to positions of the corresponding bits in the L2 entry.
+ */
+#define L1_S_DEMOTE(l1pd)	((((l1pd) & L1_S_PROTO) >> 0) | \
+				(((l1pd) & L1_SHARED) >> 6) | \
+				(((l1pd) & L1_S_REF) >> 6) | \
+				(((l1pd) & L1_S_PROT_MASK) >> 6) | \
+				(((l1pd) & L1_S_XN) >> 4))
+
 #ifndef SMP
 #define ARM_L1S_STRONG_ORD	(0)
 #define ARM_L1S_DEVICE_NOSHARE	(L1_S_TEX(2))
@@ -497,11 +521,15 @@ extern int pmap_needs_pte_sync;
 				 (((pr) & VM_PROT_WRITE) ? L2_S_PROT_W : 0))
 #else
 #define	L1_S_PROT_U		(L1_S_AP(AP_U))
-#define	L1_S_PROT_MASK		(L1_S_APX|L1_S_AP(0x3))
-#define	L1_S_WRITABLE(pd)	(!((pd) & L1_S_APX))
-
-#define	L1_S_PROT(ku, pr)	(L1_S_PROT_MASK & ~((((ku) == PTE_KERNEL) ? L1_S_PROT_U : 0) | \
-				 (((pr) & VM_PROT_WRITE) ? L1_S_APX : 0)))
+#define	L1_S_PROT_W		(L1_S_APX)		/* Write disable */
+#define	L1_S_PROT_MASK		(L1_S_PROT_W|L1_S_PROT_U)
+#define	L1_S_REF		(L1_S_AP(AP_REF))	/* Reference flag */
+#define	L1_S_WRITABLE(pd)	(!((pd) & L1_S_PROT_W))
+#define	L1_S_REFERENCED(pd)	((pd) & L1_S_REF)
+
+#define	L1_S_PROT(ku, pr)	(((((ku) == PTE_KERNEL) ? 0 : L1_S_PROT_U) | \
+				 (((pr) & VM_PROT_WRITE) ? 0 : L1_S_PROT_W) | \
+				 (((pr) & VM_PROT_EXECUTE) ? 0 : L1_S_XN)))
 
 #define	L2_L_PROT_MASK		(L2_APX|L2_AP0(0x3))
 #define	L2_L_PROT(ku, pr)	(L2_L_PROT_MASK & ~((((ku) == PTE_KERNEL) ? L2_S_PROT_U : 0) | \
diff --git a/sys/arm/include/pte.h b/sys/arm/include/pte.h
index 1a91786..d12fc24 100644
--- a/sys/arm/include/pte.h
+++ b/sys/arm/include/pte.h
@@ -159,6 +159,9 @@ typedef	uint32_t	pt_entry_t;		/* page table entry */
  */
 #define	L2_TABLE_SIZE_REAL	0x400	/* 1K */
 
+/* Total number of page table entries in L2 table */
+#define	L2_PTE_NUM_TOTAL	(L2_TABLE_SIZE_REAL / sizeof(pt_entry_t))
+
 /*
  * ARM L1 Descriptors
  */
@@ -173,6 +176,7 @@ typedef	uint32_t	pt_entry_t;		/* page table entry */
 #define	L1_S_B		0x00000004	/* bufferable Section */
 #define	L1_S_C		0x00000008	/* cacheable Section */
 #define	L1_S_IMP	0x00000010	/* implementation defined */
+#define	L1_S_XN		(1 << 4)	/* execute not */
 #define	L1_S_DOM(x)	((x) << 5)	/* domain */
 #define	L1_S_DOM_MASK	L1_S_DOM(0xf)
 #define	L1_S_AP(x)	((x) << 10)	/* access permissions */
@@ -248,6 +252,7 @@ typedef	uint32_t	pt_entry_t;		/* page table entry */
  * Access Permissions for L1 and L2 Descriptors.
  */
 #define	AP_W		0x01		/* writable */
+#define	AP_REF		0x01		/* referenced flag */
 #define	AP_U		0x02		/* user */
 
 /*