MFC r279810, r279811:

  Clean data cache before instruction cache in armv7_icache_sync_range().

  Add minimum cache line sizes to struct cpuinfo, use them in the new cache
  maintenance routines.  Also add a routine to invalidate the branch cache.

5 files changed, 53 insertions, 12 deletions

diff --git a/sys/arm/arm/cpufunc_asm_armv7.S b/sys/arm/arm/cpufunc_asm_armv7.S
index dee9a9a..25f052f 100644
--- a/sys/arm/arm/cpufunc_asm_armv7.S
+++ b/sys/arm/arm/cpufunc_asm_armv7.S
@@ -247,8 +247,8 @@ ENTRY(armv7_idcache_wbinv_range)
 	add	r0, r0, ip
 	subs	r1, r1, ip
 	bhi	.Larmv7_id_wbinv_next
-	isb				/* instruction synchronization barrier */
 	dsb				/* data synchronization barrier */
+	isb				/* instruction synchronization barrier */
 	RET
 END(armv7_idcache_wbinv_range)
 
@@ -258,8 +258,8 @@ ENTRY_NP(armv7_icache_sync_all)
 #else
 	mcr	CP15_ICIALLU
 #endif
-	isb				/* instruction synchronization barrier */
 	dsb				/* data synchronization barrier */
+	isb				/* instruction synchronization barrier */
 	RET
 END(armv7_icache_sync_all)
 
@@ -267,13 +267,13 @@ ENTRY_NP(armv7_icache_sync_range)
 	ldr	ip, .Larmv7_icache_line_size
 	ldr	ip, [ip]
 .Larmv7_sync_next:
-	mcr	CP15_ICIMVAU(r0)
 	mcr	CP15_DCCMVAC(r0)
+	mcr	CP15_ICIMVAU(r0)
 	add	r0, r0, ip
 	subs	r1, r1, ip
 	bhi	.Larmv7_sync_next
-	isb				/* instruction synchronization barrier */
 	dsb				/* data synchronization barrier */
+	isb				/* instruction synchronization barrier */
 	RET
 END(armv7_icache_sync_range)
 
diff --git a/sys/arm/arm/cpuinfo.c b/sys/arm/arm/cpuinfo.c
index b0b8a88..4617003 100644
--- a/sys/arm/arm/cpuinfo.c
+++ b/sys/arm/arm/cpuinfo.c
@@ -34,7 +34,14 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpuinfo.h>
 #include <machine/cpu-v6.h>
 
-struct cpuinfo cpuinfo;
+struct cpuinfo cpuinfo =
+{
+	/* Use safe defaults for start */
+	.dcache_line_size = 32,
+	.dcache_line_mask = 31,
+	.icache_line_size = 32,
+	.icache_line_mask = 31,
+};
 
 /* Read and parse CPU id scheme */
 void
@@ -122,4 +129,10 @@ cpuinfo_init(void)
 	cpuinfo.generic_timer_ext = (cpuinfo.id_pfr1 >> 16) & 0xF;
 	cpuinfo.virtualization_ext = (cpuinfo.id_pfr1 >> 12) & 0xF;
 	cpuinfo.security_ext = (cpuinfo.id_pfr1 >> 4) & 0xF;
+
+	/* L1 Cache sizes */
+	cpuinfo.dcache_line_size = 1 << (CPU_CT_DMINLINE(cpuinfo.ctr ) + 2);
+	cpuinfo.dcache_line_mask = cpuinfo.dcache_line_size - 1;
+	cpuinfo.icache_line_size= 1 << (CPU_CT_IMINLINE(cpuinfo.ctr ) + 2);
+	cpuinfo.icache_line_mask = cpuinfo.icache_line_size - 1;
 }
diff --git a/sys/arm/arm/genassym.c b/sys/arm/arm/genassym.c
index 7f949e6..19c2ed3 100644
--- a/sys/arm/arm/genassym.c
+++ b/sys/arm/arm/genassym.c
@@ -45,6 +45,7 @@ __FBSDID("$FreeBSD$");
 #include <machine/cpu.h>
 #include <machine/proc.h>
 #include <machine/cpufunc.h>
+#include <machine/cpuinfo.h>
 #include <machine/pte.h>
 #include <machine/intr.h>
 #include <machine/sysarch.h>
@@ -146,3 +147,8 @@ ASSYM(MAXCOMLEN, MAXCOMLEN);
 ASSYM(MAXCPU, MAXCPU);
 ASSYM(NIRQ, NIRQ);
 ASSYM(PCPU_SIZE, sizeof(struct pcpu));
+
+ASSYM(DCACHE_LINE_SIZE, offsetof(struct cpuinfo, dcache_line_size));
+ASSYM(DCACHE_LINE_MASK, offsetof(struct cpuinfo, dcache_line_mask));
+ASSYM(ICACHE_LINE_SIZE, offsetof(struct cpuinfo, icache_line_size));
+ASSYM(ICACHE_LINE_MASK, offsetof(struct cpuinfo, icache_line_mask));
diff --git a/sys/arm/include/cpu-v6.h b/sys/arm/include/cpu-v6.h
index 8d1a413..55c4434 100644
--- a/sys/arm/include/cpu-v6.h
+++ b/sys/arm/include/cpu-v6.h
@@ -37,6 +37,9 @@
 
 #define CPU_ASID_KERNEL 0
 
+vm_offset_t dcache_wb_pou_checked(vm_offset_t, vm_size_t);
+vm_offset_t icache_inv_pou_checked(vm_offset_t, vm_size_t);
+
 /*
  * Macros to generate CP15 (system control processor) read/write functions.
  */
@@ -295,7 +298,7 @@ icache_sync(vm_offset_t sva, vm_size_t size)
 	vm_offset_t eva = sva + size;
 
 	dsb();
-	for (va = sva; va < eva; va += arm_dcache_align) {
+	for (va = sva; va < eva; va += cpuinfo.dcache_line_size) {
 #if __ARM_ARCH >= 7 && defined SMP
 		_CP15_DCCMVAU(va);
 #else
@@ -325,6 +328,19 @@ icache_inv_all(void)
 	isb();
 }
 
+/* Invalidate branch predictor buffer */
+static __inline void
+bpb_inv_all(void)
+{
+#if __ARM_ARCH >= 7 && defined SMP
+	_CP15_BPIALLIS();
+#else
+	_CP15_BPIALL();
+#endif
+	dsb();
+	isb();
+}
+
 /* Write back D-cache to PoU */
 static __inline void
 dcache_wb_pou(vm_offset_t sva, vm_size_t size)
@@ -333,7 +349,7 @@ dcache_wb_pou(vm_offset_t sva, vm_size_t size)
 	vm_offset_t eva = sva + size;
 
 	dsb();
-	for (va = sva; va < eva; va += arm_dcache_align) {
+	for (va = sva; va < eva; va += cpuinfo.dcache_line_size) {
 #if __ARM_ARCH >= 7 && defined SMP
 		_CP15_DCCMVAU(va);
 #else
@@ -351,7 +367,7 @@ dcache_inv_poc(vm_offset_t sva, vm_paddr_t pa, vm_size_t size)
 	vm_offset_t eva = sva + size;
 
 	/* invalidate L1 first */
-	for (va = sva; va < eva; va += arm_dcache_align) {
+	for (va = sva; va < eva; va += cpuinfo.dcache_line_size) {
 		_CP15_DCIMVAC(va);
 	}
 	dsb();
@@ -361,7 +377,7 @@ dcache_inv_poc(vm_offset_t sva, vm_paddr_t pa, vm_size_t size)
 	dsb();
 
 	/* then L1 again */
-	for (va = sva; va < eva; va += arm_dcache_align) {
+	for (va = sva; va < eva; va += cpuinfo.dcache_line_size) {
 		_CP15_DCIMVAC(va);
 	}
 	dsb();
@@ -376,7 +392,7 @@ dcache_wb_poc(vm_offset_t sva, vm_paddr_t pa, vm_size_t size)
 
 	dsb();
 
-	for (va = sva; va < eva; va += arm_dcache_align) {
+	for (va = sva; va < eva; va += cpuinfo.dcache_line_size) {
 		_CP15_DCCMVAC(va);
 	}
 	dsb();
@@ -394,7 +410,7 @@ dcache_wbinv_poc(vm_offset_t sva, vm_paddr_t pa, vm_size_t size)
 	dsb();
 
 	/* write back L1 first */
-	for (va = sva; va < eva; va += arm_dcache_align) {
+	for (va = sva; va < eva; va += cpuinfo.dcache_line_size) {
 		_CP15_DCCMVAC(va);
 	}
 	dsb();
@@ -403,7 +419,7 @@ dcache_wbinv_poc(vm_offset_t sva, vm_paddr_t pa, vm_size_t size)
 	cpu_l2cache_wbinv_range(pa, size);
 
 	/* then invalidate L1 */
-	for (va = sva; va < eva; va += arm_dcache_align) {
+	for (va = sva; va < eva; va += cpuinfo.dcache_line_size) {
 		_CP15_DCIMVAC(va);
 	}
 	dsb();
diff --git a/sys/arm/include/cpuinfo.h b/sys/arm/include/cpuinfo.h
index f347294..ce0d8e6 100644
--- a/sys/arm/include/cpuinfo.h
+++ b/sys/arm/include/cpuinfo.h
@@ -82,6 +82,12 @@ struct cpuinfo {
 	int generic_timer_ext;
 	int virtualization_ext;
 	int security_ext;
+
+	/* L1 cache info */
+	int dcache_line_size;
+	int dcache_line_mask;
+	int icache_line_size;
+	int icache_line_mask;
 };
 
 extern struct cpuinfo cpuinfo;