arm64: pagezero improvement

This change has been provided to improve pagezero call performance.

Submitted by:          Dominik Ermel <der@semihalf.com>
Obtained from:         Semihalf
Sponsored by:          Cavium
Reviewed by:           kib
Differential Revision: https://reviews.freebsd.org/D5741

4 files changed, 47 insertions, 7 deletions

diff --git a/sys/arm64/arm64/machdep.c b/sys/arm64/arm64/machdep.c
index 0ab4498..2bf108f 100644
--- a/sys/arm64/arm64/machdep.c
+++ b/sys/arm64/arm64/machdep.c
@@ -110,6 +110,13 @@ int64_t icache_line_size;	/* The minimum I cache line size */
 int64_t idcache_line_size;	/* The minimum cache line size */
 int64_t dczva_line_size;	/* The size of cache line the dc zva zeroes */
 
+/* pagezero_* implementations are provided in support.S */
+void pagezero_simple(void *);
+void pagezero_cache(void *);
+
+/* pagezero_simple is default pagezero */
+void (*pagezero)(void *p) = pagezero_simple;
+
 static void
 cpu_startup(void *dummy)
 {
@@ -817,6 +824,9 @@ cache_setup(void)
 		/* Same as with above calculations */
 		dczva_line_shift = DCZID_BS_SIZE(dczid_el0);
 		dczva_line_size = sizeof(int) << dczva_line_shift;
+
+		/* Change pagezero function */
+		pagezero = pagezero_cache;
 	}
 }
 
diff --git a/sys/arm64/arm64/pmap.c b/sys/arm64/arm64/pmap.c
index 0b37610..5eeccdd 100644
--- a/sys/arm64/arm64/pmap.c
+++ b/sys/arm64/arm64/pmap.c
@@ -271,13 +271,6 @@ pagecopy(void *s, void *d)
 	memcpy(d, s, PAGE_SIZE);
 }
 
-static __inline void
-pagezero(void *p)
-{
-
-	bzero(p, PAGE_SIZE);
-}
-
 #define	pmap_l0_index(va)	(((va) >> L0_SHIFT) & L0_ADDR_MASK)
 #define	pmap_l1_index(va)	(((va) >> L1_SHIFT) & Ln_ADDR_MASK)
 #define	pmap_l2_index(va)	(((va) >> L2_SHIFT) & Ln_ADDR_MASK)
diff --git a/sys/arm64/arm64/support.S b/sys/arm64/arm64/support.S
index 2f42a84..4938240 100644
--- a/sys/arm64/arm64/support.S
+++ b/sys/arm64/arm64/support.S
@@ -33,6 +33,7 @@
 __FBSDID("$FreeBSD$");
 
 #include <machine/setjmp.h>
+#include <machine/param.h>
 
 #include "assym.s"
 
@@ -290,3 +291,38 @@ ENTRY(longjmp)
 	mov	x0, x1
 	ret
 END(longjmp)
+
+/*
+ * pagezero, simple implementation
+ */
+ENTRY(pagezero_simple)
+	add	x1, x0, #PAGE_SIZE
+
+1:
+	stp	xzr, xzr, [x0], #0x10
+	stp	xzr, xzr, [x0], #0x10
+	stp	xzr, xzr, [x0], #0x10
+	stp	xzr, xzr, [x0], #0x10
+	cmp	x0, x1
+	b.ne	1b
+	ret
+
+END(pagezero_simple)
+
+/*
+ * pagezero, cache assisted
+ */
+ENTRY(pagezero_cache)
+	add	x1, x0, #PAGE_SIZE
+
+	ldr	x2, =dczva_line_size
+	ldr	x2, [x2]
+
+1:
+	dc	zva, x0
+	add	x0, x0, x2
+	cmp	x0, x1
+	b.ne	1b
+	ret
+
+END(pagezero_cache)
diff --git a/sys/arm64/include/machdep.h b/sys/arm64/include/machdep.h
index 80b2af2..0b1feae 100644
--- a/sys/arm64/include/machdep.h
+++ b/sys/arm64/include/machdep.h
@@ -41,5 +41,6 @@ extern vm_paddr_t physmap[];
 extern u_int physmap_idx;
 
 void initarm(struct arm64_bootparams *);
+extern void (*pagezero)(void *);
 
 #endif /* _MACHINE_MACHDEP_H_ */