From 35a86976924a9eda7775b5b02ad47268dca1a5b4 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Wed, 2 Apr 2014 17:55:40 +0100 Subject: arm64: Update the TCR_EL1 translation granule definitions for 16K pages The current TCR register setting in arch/arm64/mm/proc.S assumes that TCR_EL1.TG* fields are one bit wide and bit 31 is RES1 (reserved, set to 1). With the addition of 16K pages (currently unsupported in the kernel), the TCR_EL1.TG* fields have been extended to two bits. This patch updates the corresponding Linux definitions and drops the bit 31 setting in proc.S in favour of the new macros. Signed-off-by: Catalin Marinas Reported-by: Joe Sylve --- arch/arm64/include/asm/pgtable-hwdef.h | 6 +++++- arch/arm64/mm/proc.S | 25 ++++++++++++++----------- 2 files changed, 19 insertions(+), 12 deletions(-) (limited to 'arch/arm64') diff --git a/arch/arm64/include/asm/pgtable-hwdef.h b/arch/arm64/include/asm/pgtable-hwdef.h index f7af66b..5fc8a66 100644 --- a/arch/arm64/include/asm/pgtable-hwdef.h +++ b/arch/arm64/include/asm/pgtable-hwdef.h @@ -120,8 +120,12 @@ #define TCR_ORGN_WBnWA ((UL(3) << 10) | (UL(3) << 26)) #define TCR_ORGN_MASK ((UL(3) << 10) | (UL(3) << 26)) #define TCR_SHARED ((UL(3) << 12) | (UL(3) << 28)) +#define TCR_TG0_4K (UL(0) << 14) #define TCR_TG0_64K (UL(1) << 14) -#define TCR_TG1_64K (UL(1) << 30) +#define TCR_TG0_16K (UL(2) << 14) +#define TCR_TG1_16K (UL(1) << 30) +#define TCR_TG1_4K (UL(2) << 30) +#define TCR_TG1_64K (UL(3) << 30) #define TCR_ASID16 (UL(1) << 36) #define TCR_TBI0 (UL(1) << 37) diff --git a/arch/arm64/mm/proc.S b/arch/arm64/mm/proc.S index e085ee6..9042aff 100644 --- a/arch/arm64/mm/proc.S +++ b/arch/arm64/mm/proc.S @@ -28,14 +28,21 @@ #include "proc-macros.S" -#ifndef CONFIG_SMP -/* PTWs cacheable, inner/outer WBWA not shareable */ -#define TCR_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA +#ifdef CONFIG_ARM64_64K_PAGES +#define TCR_TG_FLAGS TCR_TG0_64K | TCR_TG1_64K +#else +#define TCR_TG_FLAGS TCR_TG0_4K | TCR_TG1_4K +#endif + +#ifdef CONFIG_SMP +#define TCR_SMP_FLAGS TCR_SHARED #else -/* PTWs cacheable, inner/outer WBWA shareable */ -#define TCR_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA | TCR_SHARED +#define TCR_SMP_FLAGS 0 #endif +/* PTWs cacheable, inner/outer WBWA */ +#define TCR_CACHE_FLAGS TCR_IRGN_WBWA | TCR_ORGN_WBWA + #define MAIR(attr, mt) ((attr) << ((mt) * 8)) /* @@ -209,18 +216,14 @@ ENTRY(__cpu_setup) * Set/prepare TCR and TTBR. We use 512GB (39-bit) address range for * both user and kernel. */ - ldr x10, =TCR_TxSZ(VA_BITS) | TCR_FLAGS | \ - TCR_ASID16 | TCR_TBI0 | (1 << 31) + ldr x10, =TCR_TxSZ(VA_BITS) | TCR_CACHE_FLAGS | TCR_SMP_FLAGS | \ + TCR_TG_FLAGS | TCR_ASID16 | TCR_TBI0 /* * Read the PARange bits from ID_AA64MMFR0_EL1 and set the IPS bits in * TCR_EL1. */ mrs x9, ID_AA64MMFR0_EL1 bfi x10, x9, #32, #3 -#ifdef CONFIG_ARM64_64K_PAGES - orr x10, x10, TCR_TG0_64K - orr x10, x10, TCR_TG1_64K -#endif msr tcr_el1, x10 ret // return to head.S ENDPROC(__cpu_setup) -- cgit v1.1 From c218bca74eeafa2f8528b6bbb34d112075fcf40a Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Wed, 26 Mar 2014 18:25:55 +0000 Subject: arm64: Relax the kernel cache requirements for boot With system caches for the host OS or architected caches for guest OS we cannot easily guarantee that there are no dirty or stale cache lines for the areas of memory written by the kernel during boot with the MMU off (therefore non-cacheable accesses). This patch adds the necessary cache maintenance during boot and relaxes the booting requirements. Signed-off-by: Catalin Marinas --- arch/arm64/kernel/head.S | 30 ++++++++++++++++++++++++++++-- arch/arm64/mm/cache.S | 9 +++++++++ 2 files changed, 37 insertions(+), 2 deletions(-) (limited to 'arch/arm64') diff --git a/arch/arm64/kernel/head.S b/arch/arm64/kernel/head.S index 61035d6..2610968 100644 --- a/arch/arm64/kernel/head.S +++ b/arch/arm64/kernel/head.S @@ -26,6 +26,7 @@ #include #include #include +#include #include #include #include @@ -229,7 +230,11 @@ ENTRY(set_cpu_boot_mode_flag) cmp w20, #BOOT_CPU_MODE_EL2 b.ne 1f add x1, x1, #4 -1: str w20, [x1] // This CPU has booted in EL1 +1: dc cvac, x1 // Clean potentially dirty cache line + dsb sy + str w20, [x1] // This CPU has booted in EL1 + dc civac, x1 // Clean&invalidate potentially stale cache line + dsb sy ret ENDPROC(set_cpu_boot_mode_flag) @@ -240,8 +245,9 @@ ENDPROC(set_cpu_boot_mode_flag) * This is not in .bss, because we set it sufficiently early that the boot-time * zeroing of .bss would clobber it. */ - .pushsection .data + .pushsection .data..cacheline_aligned ENTRY(__boot_cpu_mode) + .align L1_CACHE_SHIFT .long BOOT_CPU_MODE_EL2 .long 0 .popsection @@ -408,6 +414,15 @@ ENDPROC(__calc_phys_offset) */ __create_page_tables: pgtbl x25, x26, x24 // idmap_pg_dir and swapper_pg_dir addresses + mov x27, lr + + /* + * Invalidate the idmap and swapper page tables to avoid potential + * dirty cache lines being evicted. + */ + mov x0, x25 + add x1, x26, #SWAPPER_DIR_SIZE + bl __inval_cache_range /* * Clear the idmap and swapper page tables. @@ -470,6 +485,17 @@ __create_page_tables: add x0, x26, #2 * PAGE_SIZE // section table address create_pgd_entry x26, x0, x5, x6, x7 #endif + + /* + * Since the page tables have been populated with non-cacheable + * accesses (MMU disabled), invalidate the idmap and swapper page + * tables again to remove any speculatively loaded cache lines. + */ + mov x0, x25 + add x1, x26, #SWAPPER_DIR_SIZE + bl __inval_cache_range + + mov lr, x27 ret ENDPROC(__create_page_tables) .ltorg diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index c46f48b..e803a62 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -168,6 +168,14 @@ ENTRY(__flush_dcache_area) ENDPROC(__flush_dcache_area) /* + * __inval_cache_range(start, end) + * - start - start address of region + * - end - end address of region + */ +ENTRY(__inval_cache_range) + /* FALLTHROUGH */ + +/* * __dma_inv_range(start, end) * - start - virtual start address of region * - end - virtual end address of region @@ -183,6 +191,7 @@ __dma_inv_range: b.lo 1b dsb sy ret +ENDPROC(__inval_cache_range) ENDPROC(__dma_inv_range) /* -- cgit v1.1 From 0a997ecc08e0b551119c56d52a591d9e5b38a7cd Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Fri, 28 Mar 2014 09:49:13 +0000 Subject: Revert "arm64: virt: ensure visibility of __boot_cpu_mode" This reverts commit 82b2f495fba338d1e3098dde1df54944a9c19751. The __boot_cpu_mode variable is flushed in head.S after being written, therefore the additional cache flushing is no longer required. Signed-off-by: Catalin Marinas --- arch/arm64/include/asm/virt.h | 13 ------------- 1 file changed, 13 deletions(-) (limited to 'arch/arm64') diff --git a/arch/arm64/include/asm/virt.h b/arch/arm64/include/asm/virt.h index 130e2be..215ad46 100644 --- a/arch/arm64/include/asm/virt.h +++ b/arch/arm64/include/asm/virt.h @@ -22,7 +22,6 @@ #define BOOT_CPU_MODE_EL2 (0xe12) #ifndef __ASSEMBLY__ -#include /* * __boot_cpu_mode records what mode CPUs were booted in. @@ -38,20 +37,9 @@ extern u32 __boot_cpu_mode[2]; void __hyp_set_vectors(phys_addr_t phys_vector_base); phys_addr_t __hyp_get_vectors(void); -static inline void sync_boot_mode(void) -{ - /* - * As secondaries write to __boot_cpu_mode with caches disabled, we - * must flush the corresponding cache entries to ensure the visibility - * of their writes. - */ - __flush_dcache_area(__boot_cpu_mode, sizeof(__boot_cpu_mode)); -} - /* Reports the availability of HYP mode */ static inline bool is_hyp_mode_available(void) { - sync_boot_mode(); return (__boot_cpu_mode[0] == BOOT_CPU_MODE_EL2 && __boot_cpu_mode[1] == BOOT_CPU_MODE_EL2); } @@ -59,7 +47,6 @@ static inline bool is_hyp_mode_available(void) /* Check if the bootloader has booted CPUs in different modes */ static inline bool is_hyp_mode_mismatched(void) { - sync_boot_mode(); return __boot_cpu_mode[0] != __boot_cpu_mode[1]; } -- cgit v1.1 From ff268ff7f32bf5388b7422f0c0773d88add23423 Mon Sep 17 00:00:00 2001 From: Mark Salter Date: Sat, 5 Apr 2014 15:25:49 +0100 Subject: arm64: fix !CONFIG_COMPAT build failures MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Recent arm64 builds using CONFIG_ARM64_64K_PAGES are failing with: arch/arm64/kernel/perf_regs.c: In function ‘perf_reg_abi’: arch/arm64/kernel/perf_regs.c:41:2: error: implicit declaration of function ‘is_compat_thread’ arch/arm64/kernel/perf_event.c:1398:2: error: unknown type name ‘compat_uptr_t’ This is due to some recent arm64 perf commits with compat support: commit 23c7d70d55c6d9: ARM64: perf: add support for frame pointer unwinding in compat mode commit 2ee0d7fd36a3f8: ARM64: perf: add support for perf registers API Those patches make the arm64 kernel unbuildable if CONFIG_COMPAT is not defined and CONFIG_ARM64_64K_PAGES depends on !CONFIG_COMPAT. This patch allows the arm64 kernel to build with and without CONFIG_COMPAT. Signed-off-by: Mark Salter Signed-off-by: Catalin Marinas --- arch/arm64/kernel/perf_event.c | 4 ++++ arch/arm64/kernel/perf_regs.c | 2 ++ 2 files changed, 6 insertions(+) (limited to 'arch/arm64') diff --git a/arch/arm64/kernel/perf_event.c b/arch/arm64/kernel/perf_event.c index e868c72..baf5afb 100644 --- a/arch/arm64/kernel/perf_event.c +++ b/arch/arm64/kernel/perf_event.c @@ -1386,6 +1386,7 @@ user_backtrace(struct frame_tail __user *tail, return buftail.fp; } +#ifdef CONFIG_COMPAT /* * The registers we're interested in are at the end of the variable * length saved register structure. The fp points at the end of this @@ -1430,6 +1431,7 @@ compat_user_backtrace(struct compat_frame_tail __user *tail, return (struct compat_frame_tail __user *)compat_ptr(buftail.fp) - 1; } +#endif /* CONFIG_COMPAT */ void perf_callchain_user(struct perf_callchain_entry *entry, struct pt_regs *regs) @@ -1451,6 +1453,7 @@ void perf_callchain_user(struct perf_callchain_entry *entry, tail && !((unsigned long)tail & 0xf)) tail = user_backtrace(tail, entry); } else { +#ifdef CONFIG_COMPAT /* AARCH32 compat mode */ struct compat_frame_tail __user *tail; @@ -1459,6 +1462,7 @@ void perf_callchain_user(struct perf_callchain_entry *entry, while ((entry->nr < PERF_MAX_STACK_DEPTH) && tail && !((unsigned long)tail & 0x3)) tail = compat_user_backtrace(tail, entry); +#endif } } diff --git a/arch/arm64/kernel/perf_regs.c b/arch/arm64/kernel/perf_regs.c index f2d6f0a..422ebd6 100644 --- a/arch/arm64/kernel/perf_regs.c +++ b/arch/arm64/kernel/perf_regs.c @@ -2,6 +2,8 @@ #include #include #include + +#include #include #include -- cgit v1.1 From d253b4406df69fa7a74231769d6f6ad80dc33063 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Sat, 5 Apr 2014 01:30:50 +0100 Subject: arm64: Add missing Kconfig for CONFIG_STRICT_DEVMEM The Kconfig for CONFIG_STRICT_DEVMEM is missing despite being used in mmap.c. Add it. Signed-off-by: Laura Abbott Signed-off-by: Catalin Marinas --- arch/arm64/Kconfig.debug | 14 ++++++++++++++ 1 file changed, 14 insertions(+) (limited to 'arch/arm64') diff --git a/arch/arm64/Kconfig.debug b/arch/arm64/Kconfig.debug index 835c559..d10ec33 100644 --- a/arch/arm64/Kconfig.debug +++ b/arch/arm64/Kconfig.debug @@ -6,6 +6,20 @@ config FRAME_POINTER bool default y +config STRICT_DEVMEM + bool "Filter access to /dev/mem" + depends on MMU + help + If this option is disabled, you allow userspace (root) access to all + of memory, including kernel and userspace memory. Accidental + access to this is obviously disastrous, but specific access can + be used by people debugging the kernel. + + If this option is switched on, the /dev/mem file only allows + userspace access to memory mapped peripherals. + + If in doubt, say Y. + config EARLY_PRINTK bool "Early printk support" default y -- cgit v1.1 From ebf81a938dade3b450eb11c57fa744cfac4b523f Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Tue, 1 Apr 2014 18:32:55 +0100 Subject: arm64: Fix DMA range invalidation for cache line unaligned buffers If the buffer needing cache invalidation for inbound DMA does start or end on a cache line aligned address, we need to use the non-destructive clean&invalidate operation. This issue was introduced by commit 7363590d2c46 (arm64: Implement coherent DMA API based on swiotlb). Signed-off-by: Catalin Marinas Reported-by: Jon Medhurst (Tixy) --- arch/arm64/mm/cache.S | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) (limited to 'arch/arm64') diff --git a/arch/arm64/mm/cache.S b/arch/arm64/mm/cache.S index e803a62..fda7568 100644 --- a/arch/arm64/mm/cache.S +++ b/arch/arm64/mm/cache.S @@ -183,12 +183,19 @@ ENTRY(__inval_cache_range) __dma_inv_range: dcache_line_size x2, x3 sub x3, x2, #1 - bic x0, x0, x3 + tst x1, x3 // end cache line aligned? bic x1, x1, x3 -1: dc ivac, x0 // invalidate D / U line - add x0, x0, x2 + b.eq 1f + dc civac, x1 // clean & invalidate D / U line +1: tst x0, x3 // start cache line aligned? + bic x0, x0, x3 + b.eq 2f + dc civac, x0 // clean & invalidate D / U line + b 3f +2: dc ivac, x0 // invalidate D / U line +3: add x0, x0, x2 cmp x0, x1 - b.lo 1b + b.lo 2b dsb sy ret ENDPROC(__inval_cache_range) -- cgit v1.1