From d1ae8c0057921681ca489bba7efbfacbb60d0f28 Mon Sep 17 00:00:00 2001 From: Yi Li Date: Sat, 4 Oct 2014 23:46:43 +0800 Subject: arm64: dmi: Add SMBIOS/DMI support SMBIOS is important for server hardware vendors. It implements a spec for providing descriptive information about the platform. Things like serial numbers, physical layout of the ports, build configuration data, and the like. Signed-off-by: Yi Li Tested-by: Suravee Suthikulpanit Tested-by: Leif Lindholm Signed-off-by: Ard Biesheuvel --- arch/arm64/include/asm/dmi.h | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 arch/arm64/include/asm/dmi.h (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/dmi.h b/arch/arm64/include/asm/dmi.h new file mode 100644 index 0000000..69d37d8 --- /dev/null +++ b/arch/arm64/include/asm/dmi.h @@ -0,0 +1,31 @@ +/* + * arch/arm64/include/asm/dmi.h + * + * Copyright (C) 2013 Linaro Limited. + * Written by: Yi Li (yi.li@linaro.org) + * + * based on arch/ia64/include/asm/dmi.h + * + * This file is subject to the terms and conditions of the GNU General Public + * License. See the file "COPYING" in the main directory of this archive + * for more details. + */ + +#ifndef __ASM_DMI_H +#define __ASM_DMI_H + +#include +#include + +/* + * According to section 2.3.6 of the UEFI spec, the firmware should not + * request a virtual mapping for configuration tables such as SMBIOS. + * This means we have to map them before use. + */ +#define dmi_early_remap(x, l) ioremap_cache(x, l) +#define dmi_early_unmap(x, l) iounmap(x) +#define dmi_remap(x, l) ioremap_cache(x, l) +#define dmi_unmap(x) iounmap(x) +#define dmi_alloc(l) kzalloc(l, GFP_KERNEL) + +#endif -- cgit v1.1 From 5284e1b4bc8ae6fcc1c92c63cf6c876a53292f82 Mon Sep 17 00:00:00 2001 From: Steve Capper Date: Fri, 24 Oct 2014 13:22:20 +0100 Subject: arm64: xchg: Implement cmpxchg_double The arm64 architecture has the ability to exclusively load and store a pair of registers from an address (ldxp/stxp). Also the SLUB can take advantage of a cmpxchg_double implementation to avoid taking some locks. This patch provides an implementation of cmpxchg_double for 64-bit pairs, and activates the logic required for the SLUB to use these functions (HAVE_ALIGNED_STRUCT_PAGE and HAVE_CMPXCHG_DOUBLE). Also definitions of this_cpu_cmpxchg_8 and this_cpu_cmpxchg_double_8 are wired up to cmpxchg_local and cmpxchg_double_local (rather than the stock implementations that perform non-atomic operations with interrupts disabled) as they are used by the SLUB. On a Juno platform running on only the A57s I get quite a noticeable performance improvement with 5 runs of hackbench on v3.17: Baseline | With Patch -----------------+----------- Mean 119.2312 | 106.1782 StdDev 0.4919 | 0.4494 (times taken to complete `./hackbench 100 process 1000', in seconds) Signed-off-by: Steve Capper Signed-off-by: Will Deacon --- arch/arm64/include/asm/cmpxchg.h | 71 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 71 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index ddb9d78..89e397b 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -19,6 +19,7 @@ #define __ASM_CMPXCHG_H #include +#include #include @@ -152,6 +153,51 @@ static inline unsigned long __cmpxchg(volatile void *ptr, unsigned long old, return oldval; } +#define system_has_cmpxchg_double() 1 + +static inline int __cmpxchg_double(volatile void *ptr1, volatile void *ptr2, + unsigned long old1, unsigned long old2, + unsigned long new1, unsigned long new2, int size) +{ + unsigned long loop, lost; + + switch (size) { + case 8: + VM_BUG_ON((unsigned long *)ptr2 - (unsigned long *)ptr1 != 1); + do { + asm volatile("// __cmpxchg_double8\n" + " ldxp %0, %1, %2\n" + " eor %0, %0, %3\n" + " eor %1, %1, %4\n" + " orr %1, %0, %1\n" + " mov %w0, #0\n" + " cbnz %1, 1f\n" + " stxp %w0, %5, %6, %2\n" + "1:\n" + : "=&r"(loop), "=&r"(lost), "+Q" (*(u64 *)ptr1) + : "r" (old1), "r"(old2), "r"(new1), "r"(new2)); + } while (loop); + break; + default: + BUILD_BUG(); + } + + return !lost; +} + +static inline int __cmpxchg_double_mb(volatile void *ptr1, volatile void *ptr2, + unsigned long old1, unsigned long old2, + unsigned long new1, unsigned long new2, int size) +{ + int ret; + + smp_mb(); + ret = __cmpxchg_double(ptr1, ptr2, old1, old2, new1, new2, size); + smp_mb(); + + return ret; +} + static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, unsigned long new, int size) { @@ -182,6 +228,31 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, __ret; \ }) +#define cmpxchg_double(ptr1, ptr2, o1, o2, n1, n2) \ +({\ + int __ret;\ + __ret = __cmpxchg_double_mb((ptr1), (ptr2), (unsigned long)(o1), \ + (unsigned long)(o2), (unsigned long)(n1), \ + (unsigned long)(n2), sizeof(*(ptr1)));\ + __ret; \ +}) + +#define cmpxchg_double_local(ptr1, ptr2, o1, o2, n1, n2) \ +({\ + int __ret;\ + __ret = __cmpxchg_double((ptr1), (ptr2), (unsigned long)(o1), \ + (unsigned long)(o2), (unsigned long)(n1), \ + (unsigned long)(n2), sizeof(*(ptr1)));\ + __ret; \ +}) + +#define this_cpu_cmpxchg_8(ptr, o, n) \ + cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n) + +#define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \ + cmpxchg_double_local(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \ + o1, o2, n1, n2) + #define cmpxchg64(ptr,o,n) cmpxchg((ptr),(o),(n)) #define cmpxchg64_local(ptr,o,n) cmpxchg_local((ptr),(o),(n)) -- cgit v1.1 From 286fb1cc32b11c18da3573a8c8c37a4f9da16e30 Mon Sep 17 00:00:00 2001 From: Geoff Levand Date: Fri, 31 Oct 2014 23:06:47 +0000 Subject: arm64/kvm: Fix assembler compatibility of macros Some of the macros defined in kvm_arm.h are useful in assembly files, but are not compatible with the assembler. Change any C language integer constant definitions using appended U, UL, or ULL to the UL() preprocessor macro. Also, add a preprocessor include of the asm/memory.h file which defines the UL() macro. Fixes build errors like these when using kvm_arm.h in assembly source files: Error: unexpected characters following instruction at operand 3 -- `and x0,x1,#((1U<<25)-1)' Acked-by: Mark Rutland Signed-off-by: Geoff Levand Signed-off-by: Will Deacon --- arch/arm64/include/asm/kvm_arm.h | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/kvm_arm.h b/arch/arm64/include/asm/kvm_arm.h index 7fd3e27..8afb863 100644 --- a/arch/arm64/include/asm/kvm_arm.h +++ b/arch/arm64/include/asm/kvm_arm.h @@ -18,6 +18,7 @@ #ifndef __ARM64_KVM_ARM_H__ #define __ARM64_KVM_ARM_H__ +#include #include /* Hyp Configuration Register (HCR) bits */ @@ -160,9 +161,9 @@ #endif #define VTTBR_BADDR_SHIFT (VTTBR_X - 1) -#define VTTBR_BADDR_MASK (((1LLU << (PHYS_MASK_SHIFT - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT) -#define VTTBR_VMID_SHIFT (48LLU) -#define VTTBR_VMID_MASK (0xffLLU << VTTBR_VMID_SHIFT) +#define VTTBR_BADDR_MASK (((UL(1) << (PHYS_MASK_SHIFT - VTTBR_X)) - 1) << VTTBR_BADDR_SHIFT) +#define VTTBR_VMID_SHIFT (UL(48)) +#define VTTBR_VMID_MASK (UL(0xFF) << VTTBR_VMID_SHIFT) /* Hyp System Trap Register */ #define HSTR_EL2_TTEE (1 << 16) @@ -185,13 +186,13 @@ /* Exception Syndrome Register (ESR) bits */ #define ESR_EL2_EC_SHIFT (26) -#define ESR_EL2_EC (0x3fU << ESR_EL2_EC_SHIFT) -#define ESR_EL2_IL (1U << 25) +#define ESR_EL2_EC (UL(0x3f) << ESR_EL2_EC_SHIFT) +#define ESR_EL2_IL (UL(1) << 25) #define ESR_EL2_ISS (ESR_EL2_IL - 1) #define ESR_EL2_ISV_SHIFT (24) -#define ESR_EL2_ISV (1U << ESR_EL2_ISV_SHIFT) +#define ESR_EL2_ISV (UL(1) << ESR_EL2_ISV_SHIFT) #define ESR_EL2_SAS_SHIFT (22) -#define ESR_EL2_SAS (3U << ESR_EL2_SAS_SHIFT) +#define ESR_EL2_SAS (UL(3) << ESR_EL2_SAS_SHIFT) #define ESR_EL2_SSE (1 << 21) #define ESR_EL2_SRT_SHIFT (16) #define ESR_EL2_SRT_MASK (0x1f << ESR_EL2_SRT_SHIFT) @@ -205,16 +206,16 @@ #define ESR_EL2_FSC_TYPE (0x3c) #define ESR_EL2_CV_SHIFT (24) -#define ESR_EL2_CV (1U << ESR_EL2_CV_SHIFT) +#define ESR_EL2_CV (UL(1) << ESR_EL2_CV_SHIFT) #define ESR_EL2_COND_SHIFT (20) -#define ESR_EL2_COND (0xfU << ESR_EL2_COND_SHIFT) +#define ESR_EL2_COND (UL(0xf) << ESR_EL2_COND_SHIFT) #define FSC_FAULT (0x04) #define FSC_PERM (0x0c) /* Hyp Prefetch Fault Address Register (HPFAR/HDFAR) */ -#define HPFAR_MASK (~0xFUL) +#define HPFAR_MASK (~UL(0xf)) #define ESR_EL2_EC_UNKNOWN (0x00) #define ESR_EL2_EC_WFI (0x01) -- cgit v1.1 From fb7332a9fedfd62b1ba6530c86f39f0fa38afd49 Mon Sep 17 00:00:00 2001 From: Will Deacon Date: Wed, 29 Oct 2014 10:03:09 +0000 Subject: mmu_gather: move minimal range calculations into generic code On architectures with hardware broadcasting of TLB invalidation messages , it makes sense to reduce the range of the mmu_gather structure when unmapping page ranges based on the dirty address information passed to tlb_remove_tlb_entry. arm64 already does this by directly manipulating the start/end fields of the gather structure, but this confuses the generic code which does not expect these fields to change and can end up calculating invalid, negative ranges when forcing a flush in zap_pte_range. This patch moves the minimal range calculation out of the arm64 code and into the generic implementation, simplifying zap_pte_range in the process (which no longer needs to care about start/end, since they will point to the appropriate ranges already). With the range being tracked by core code, the need_flush flag is dropped in favour of checking that the end of the range has actually been set. Cc: Benjamin Herrenschmidt Cc: Peter Zijlstra Cc: Russell King - ARM Linux Cc: Michal Simek Acked-by: Linus Torvalds Signed-off-by: Will Deacon --- arch/arm64/include/asm/tlb.h | 67 ++------------------------------------------ 1 file changed, 3 insertions(+), 64 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/tlb.h b/arch/arm64/include/asm/tlb.h index a82c0c5..c028fe3 100644 --- a/arch/arm64/include/asm/tlb.h +++ b/arch/arm64/include/asm/tlb.h @@ -19,10 +19,6 @@ #ifndef __ASM_TLB_H #define __ASM_TLB_H -#define __tlb_remove_pmd_tlb_entry __tlb_remove_pmd_tlb_entry - -#include - #include #include @@ -37,71 +33,22 @@ static inline void __tlb_remove_table(void *_table) #define tlb_remove_entry(tlb, entry) tlb_remove_page(tlb, entry) #endif /* CONFIG_HAVE_RCU_TABLE_FREE */ -/* - * There's three ways the TLB shootdown code is used: - * 1. Unmapping a range of vmas. See zap_page_range(), unmap_region(). - * tlb->fullmm = 0, and tlb_start_vma/tlb_end_vma will be called. - * 2. Unmapping all vmas. See exit_mmap(). - * tlb->fullmm = 1, and tlb_start_vma/tlb_end_vma will be called. - * Page tables will be freed. - * 3. Unmapping argument pages. See shift_arg_pages(). - * tlb->fullmm = 0, but tlb_start_vma/tlb_end_vma will not be called. - */ +#include + static inline void tlb_flush(struct mmu_gather *tlb) { if (tlb->fullmm) { flush_tlb_mm(tlb->mm); - } else if (tlb->end > 0) { + } else { struct vm_area_struct vma = { .vm_mm = tlb->mm, }; flush_tlb_range(&vma, tlb->start, tlb->end); - tlb->start = TASK_SIZE; - tlb->end = 0; - } -} - -static inline void tlb_add_flush(struct mmu_gather *tlb, unsigned long addr) -{ - if (!tlb->fullmm) { - tlb->start = min(tlb->start, addr); - tlb->end = max(tlb->end, addr + PAGE_SIZE); - } -} - -/* - * Memorize the range for the TLB flush. - */ -static inline void __tlb_remove_tlb_entry(struct mmu_gather *tlb, pte_t *ptep, - unsigned long addr) -{ - tlb_add_flush(tlb, addr); -} - -/* - * In the case of tlb vma handling, we can optimise these away in the - * case where we're doing a full MM flush. When we're doing a munmap, - * the vmas are adjusted to only cover the region to be torn down. - */ -static inline void tlb_start_vma(struct mmu_gather *tlb, - struct vm_area_struct *vma) -{ - if (!tlb->fullmm) { - tlb->start = TASK_SIZE; - tlb->end = 0; } } -static inline void tlb_end_vma(struct mmu_gather *tlb, - struct vm_area_struct *vma) -{ - if (!tlb->fullmm) - tlb_flush(tlb); -} - static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, unsigned long addr) { pgtable_page_dtor(pte); - tlb_add_flush(tlb, addr); tlb_remove_entry(tlb, pte); } @@ -109,7 +56,6 @@ static inline void __pte_free_tlb(struct mmu_gather *tlb, pgtable_t pte, static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, unsigned long addr) { - tlb_add_flush(tlb, addr); tlb_remove_entry(tlb, virt_to_page(pmdp)); } #endif @@ -118,15 +64,8 @@ static inline void __pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmdp, static inline void __pud_free_tlb(struct mmu_gather *tlb, pud_t *pudp, unsigned long addr) { - tlb_add_flush(tlb, addr); tlb_remove_entry(tlb, virt_to_page(pudp)); } #endif -static inline void __tlb_remove_pmd_tlb_entry(struct mmu_gather *tlb, pmd_t *pmdp, - unsigned long address) -{ - tlb_add_flush(tlb, address); -} - #endif -- cgit v1.1 From 7d57511d2dba03a8046c8b428dd9192a4bfc1e73 Mon Sep 17 00:00:00 2001 From: Catalin Marinas Date: Mon, 17 Nov 2014 10:37:40 +0000 Subject: arm64: Add COMPAT_HWCAP_LPAE Commit a469abd0f868 (ARM: elf: add new hwcap for identifying atomic ldrd/strd instructions) introduces HWCAP_ELF for 32-bit ARM applications. As LPAE is always present on arm64, report the corresponding compat HWCAP to user space. Signed-off-by: Catalin Marinas Cc: # 3.11+ Signed-off-by: Will Deacon --- arch/arm64/include/asm/hwcap.h | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/hwcap.h b/arch/arm64/include/asm/hwcap.h index 024c461..0ad7351 100644 --- a/arch/arm64/include/asm/hwcap.h +++ b/arch/arm64/include/asm/hwcap.h @@ -30,6 +30,7 @@ #define COMPAT_HWCAP_IDIVA (1 << 17) #define COMPAT_HWCAP_IDIVT (1 << 18) #define COMPAT_HWCAP_IDIV (COMPAT_HWCAP_IDIVA|COMPAT_HWCAP_IDIVT) +#define COMPAT_HWCAP_LPAE (1 << 20) #define COMPAT_HWCAP_EVTSTRM (1 << 21) #define COMPAT_HWCAP2_AES (1 << 0) -- cgit v1.1 From 15670ef1eac9817cf48da12c885aabcdd88e9add Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Wed, 19 Nov 2014 17:44:12 +0000 Subject: arm64: pgalloc: consistently use PGALLOC_GFP We currently allocate different levels of page tables with a variety of differing flags, and the PGALLOC_GFP flags, intended for use when allocating any level of page table, are only used for ptes in pte_alloc_one. On x86, PGALLOC_GFP is used for all page table allocations. Currently the major differences are: * __GFP_NOTRACK -- Needed to ensure page tables are always accessible in the presence of kmemcheck to prevent recursive faults. Currently kmemcheck cannot be selected for arm64. * __GFP_REPEAT -- Causes the allocator to try to reclaim pages and retry upon a failure to allocate. * __GFP_ZERO -- Sometimes passed explicitly, sometimes zalloc variants are used. While we've no encountered issues so far, it would be preferable to be consistent. This patch ensures all levels of table are allocated in the same manner, with PGALLOC_GFP. Cc: Steve Capper Acked-by: Catalin Marinas Signed-off-by: Mark Rutland Signed-off-by: Will Deacon --- arch/arm64/include/asm/pgalloc.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/pgalloc.h b/arch/arm64/include/asm/pgalloc.h index d5bed02..e20df38 100644 --- a/arch/arm64/include/asm/pgalloc.h +++ b/arch/arm64/include/asm/pgalloc.h @@ -26,11 +26,13 @@ #define check_pgt_cache() do { } while (0) +#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) + #if CONFIG_ARM64_PGTABLE_LEVELS > 2 static inline pmd_t *pmd_alloc_one(struct mm_struct *mm, unsigned long addr) { - return (pmd_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); + return (pmd_t *)__get_free_page(PGALLOC_GFP); } static inline void pmd_free(struct mm_struct *mm, pmd_t *pmd) @@ -50,7 +52,7 @@ static inline void pud_populate(struct mm_struct *mm, pud_t *pud, pmd_t *pmd) static inline pud_t *pud_alloc_one(struct mm_struct *mm, unsigned long addr) { - return (pud_t *)get_zeroed_page(GFP_KERNEL | __GFP_REPEAT); + return (pud_t *)__get_free_page(PGALLOC_GFP); } static inline void pud_free(struct mm_struct *mm, pud_t *pud) @@ -69,8 +71,6 @@ static inline void pgd_populate(struct mm_struct *mm, pgd_t *pgd, pud_t *pud) extern pgd_t *pgd_alloc(struct mm_struct *mm); extern void pgd_free(struct mm_struct *mm, pgd_t *pgd); -#define PGALLOC_GFP (GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO) - static inline pte_t * pte_alloc_one_kernel(struct mm_struct *mm, unsigned long addr) { -- cgit v1.1 From f97fc810798c261b2790c2a1660461a508a479e0 Mon Sep 17 00:00:00 2001 From: Steve Capper Date: Wed, 19 Nov 2014 16:53:43 +0000 Subject: arm64: percpu: Implement this_cpu operations The generic this_cpu operations disable interrupts to ensure that the requested operation is protected from pre-emption. For arm64, this is overkill and can hurt throughput and latency. This patch provides arm64 specific implementations for the this_cpu operations. Rather than disable interrupts, we use the exclusive monitor or atomic operations as appropriate. The following operations are implemented: add, add_return, and, or, read, write, xchg. We also wire up a cmpxchg implementation from cmpxchg.h. Testing was performed using the percpu_test module and hackbench on a Juno board running 3.18-rc4. Signed-off-by: Steve Capper Reviewed-by: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/cmpxchg.h | 6 +- arch/arm64/include/asm/percpu.h | 215 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+), 2 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cmpxchg.h b/arch/arm64/include/asm/cmpxchg.h index 89e397b..cb95930 100644 --- a/arch/arm64/include/asm/cmpxchg.h +++ b/arch/arm64/include/asm/cmpxchg.h @@ -246,8 +246,10 @@ static inline unsigned long __cmpxchg_mb(volatile void *ptr, unsigned long old, __ret; \ }) -#define this_cpu_cmpxchg_8(ptr, o, n) \ - cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n) +#define this_cpu_cmpxchg_1(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n) +#define this_cpu_cmpxchg_2(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n) +#define this_cpu_cmpxchg_4(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n) +#define this_cpu_cmpxchg_8(ptr, o, n) cmpxchg_local(raw_cpu_ptr(&(ptr)), o, n) #define this_cpu_cmpxchg_double_8(ptr1, ptr2, o1, o2, n1, n2) \ cmpxchg_double_local(raw_cpu_ptr(&(ptr1)), raw_cpu_ptr(&(ptr2)), \ diff --git a/arch/arm64/include/asm/percpu.h b/arch/arm64/include/asm/percpu.h index 5279e57..09da25b 100644 --- a/arch/arm64/include/asm/percpu.h +++ b/arch/arm64/include/asm/percpu.h @@ -44,6 +44,221 @@ static inline unsigned long __my_cpu_offset(void) #endif /* CONFIG_SMP */ +#define PERCPU_OP(op, asm_op) \ +static inline unsigned long __percpu_##op(void *ptr, \ + unsigned long val, int size) \ +{ \ + unsigned long loop, ret; \ + \ + switch (size) { \ + case 1: \ + do { \ + asm ("//__per_cpu_" #op "_1\n" \ + "ldxrb %w[ret], %[ptr]\n" \ + #asm_op " %w[ret], %w[ret], %w[val]\n" \ + "stxrb %w[loop], %w[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u8 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + case 2: \ + do { \ + asm ("//__per_cpu_" #op "_2\n" \ + "ldxrh %w[ret], %[ptr]\n" \ + #asm_op " %w[ret], %w[ret], %w[val]\n" \ + "stxrh %w[loop], %w[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u16 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + case 4: \ + do { \ + asm ("//__per_cpu_" #op "_4\n" \ + "ldxr %w[ret], %[ptr]\n" \ + #asm_op " %w[ret], %w[ret], %w[val]\n" \ + "stxr %w[loop], %w[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u32 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + case 8: \ + do { \ + asm ("//__per_cpu_" #op "_8\n" \ + "ldxr %[ret], %[ptr]\n" \ + #asm_op " %[ret], %[ret], %[val]\n" \ + "stxr %w[loop], %[ret], %[ptr]\n" \ + : [loop] "=&r" (loop), [ret] "=&r" (ret), \ + [ptr] "+Q"(*(u64 *)ptr) \ + : [val] "Ir" (val)); \ + } while (loop); \ + break; \ + default: \ + BUILD_BUG(); \ + } \ + \ + return ret; \ +} + +PERCPU_OP(add, add) +PERCPU_OP(and, and) +PERCPU_OP(or, orr) +#undef PERCPU_OP + +static inline unsigned long __percpu_read(void *ptr, int size) +{ + unsigned long ret; + + switch (size) { + case 1: + ret = ACCESS_ONCE(*(u8 *)ptr); + break; + case 2: + ret = ACCESS_ONCE(*(u16 *)ptr); + break; + case 4: + ret = ACCESS_ONCE(*(u32 *)ptr); + break; + case 8: + ret = ACCESS_ONCE(*(u64 *)ptr); + break; + default: + BUILD_BUG(); + } + + return ret; +} + +static inline void __percpu_write(void *ptr, unsigned long val, int size) +{ + switch (size) { + case 1: + ACCESS_ONCE(*(u8 *)ptr) = (u8)val; + break; + case 2: + ACCESS_ONCE(*(u16 *)ptr) = (u16)val; + break; + case 4: + ACCESS_ONCE(*(u32 *)ptr) = (u32)val; + break; + case 8: + ACCESS_ONCE(*(u64 *)ptr) = (u64)val; + break; + default: + BUILD_BUG(); + } +} + +static inline unsigned long __percpu_xchg(void *ptr, unsigned long val, + int size) +{ + unsigned long ret, loop; + + switch (size) { + case 1: + do { + asm ("//__percpu_xchg_1\n" + "ldxrb %w[ret], %[ptr]\n" + "stxrb %w[loop], %w[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u8 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + case 2: + do { + asm ("//__percpu_xchg_2\n" + "ldxrh %w[ret], %[ptr]\n" + "stxrh %w[loop], %w[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u16 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + case 4: + do { + asm ("//__percpu_xchg_4\n" + "ldxr %w[ret], %[ptr]\n" + "stxr %w[loop], %w[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u32 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + case 8: + do { + asm ("//__percpu_xchg_8\n" + "ldxr %[ret], %[ptr]\n" + "stxr %w[loop], %[val], %[ptr]\n" + : [loop] "=&r"(loop), [ret] "=&r"(ret), + [ptr] "+Q"(*(u64 *)ptr) + : [val] "r" (val)); + } while (loop); + break; + default: + BUILD_BUG(); + } + + return ret; +} + +#define _percpu_add(pcp, val) \ + __percpu_add(raw_cpu_ptr(&(pcp)), val, sizeof(pcp)) + +#define _percpu_add_return(pcp, val) (typeof(pcp)) (_percpu_add(pcp, val)) + +#define _percpu_and(pcp, val) \ + __percpu_and(raw_cpu_ptr(&(pcp)), val, sizeof(pcp)) + +#define _percpu_or(pcp, val) \ + __percpu_or(raw_cpu_ptr(&(pcp)), val, sizeof(pcp)) + +#define _percpu_read(pcp) (typeof(pcp)) \ + (__percpu_read(raw_cpu_ptr(&(pcp)), sizeof(pcp))) + +#define _percpu_write(pcp, val) \ + __percpu_write(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp)) + +#define _percpu_xchg(pcp, val) (typeof(pcp)) \ + (__percpu_xchg(raw_cpu_ptr(&(pcp)), (unsigned long)(val), sizeof(pcp))) + +#define this_cpu_add_1(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_2(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_4(pcp, val) _percpu_add(pcp, val) +#define this_cpu_add_8(pcp, val) _percpu_add(pcp, val) + +#define this_cpu_add_return_1(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_2(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_4(pcp, val) _percpu_add_return(pcp, val) +#define this_cpu_add_return_8(pcp, val) _percpu_add_return(pcp, val) + +#define this_cpu_and_1(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_2(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_4(pcp, val) _percpu_and(pcp, val) +#define this_cpu_and_8(pcp, val) _percpu_and(pcp, val) + +#define this_cpu_or_1(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_2(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_4(pcp, val) _percpu_or(pcp, val) +#define this_cpu_or_8(pcp, val) _percpu_or(pcp, val) + +#define this_cpu_read_1(pcp) _percpu_read(pcp) +#define this_cpu_read_2(pcp) _percpu_read(pcp) +#define this_cpu_read_4(pcp) _percpu_read(pcp) +#define this_cpu_read_8(pcp) _percpu_read(pcp) + +#define this_cpu_write_1(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_2(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_4(pcp, val) _percpu_write(pcp, val) +#define this_cpu_write_8(pcp, val) _percpu_write(pcp, val) + +#define this_cpu_xchg_1(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_2(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_4(pcp, val) _percpu_xchg(pcp, val) +#define this_cpu_xchg_8(pcp, val) _percpu_xchg(pcp, val) + #include #endif /* __ASM_PERCPU_H */ -- cgit v1.1 From 9b79f52d1a702dd5b160f9d2ee0199c3122809bb Mon Sep 17 00:00:00 2001 From: Punit Agrawal Date: Tue, 18 Nov 2014 11:41:22 +0000 Subject: arm64: Add support for hooks to handle undefined instructions Add support to register hooks for undefined instructions. The handlers will be called when the undefined instruction and the processor state (as contained in pstate) match criteria used at registration. Signed-off-by: Punit Agrawal Reviewed-by: Catalin Marinas Signed-off-by: Will Deacon --- arch/arm64/include/asm/insn.h | 2 ++ arch/arm64/include/asm/traps.h | 16 ++++++++++++++++ 2 files changed, 18 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 56a9e63..1bb0430 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -354,6 +354,8 @@ bool aarch64_insn_hotpatch_safe(u32 old_insn, u32 new_insn); int aarch64_insn_patch_text_nosync(void *addr, u32 insn); int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt); int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); + +bool aarch32_insn_is_wide(u32 insn); #endif /* __ASSEMBLY__ */ #endif /* __ASM_INSN_H */ diff --git a/arch/arm64/include/asm/traps.h b/arch/arm64/include/asm/traps.h index 10ca8ff..232e4ba 100644 --- a/arch/arm64/include/asm/traps.h +++ b/arch/arm64/include/asm/traps.h @@ -18,6 +18,22 @@ #ifndef __ASM_TRAP_H #define __ASM_TRAP_H +#include + +struct pt_regs; + +struct undef_hook { + struct list_head node; + u32 instr_mask; + u32 instr_val; + u64 pstate_mask; + u64 pstate_val; + int (*fn)(struct pt_regs *regs, u32 instr); +}; + +void register_undef_hook(struct undef_hook *hook); +void unregister_undef_hook(struct undef_hook *hook); + static inline int in_exception_text(unsigned long ptr) { extern char __exception_text_start[]; -- cgit v1.1 From 0be0e44c182c4f13df13903fd1377671d157d7b7 Mon Sep 17 00:00:00 2001 From: Punit Agrawal Date: Tue, 18 Nov 2014 11:41:23 +0000 Subject: arm64: Add AArch32 instruction set condition code checks Port support for AArch32 instruction condition code checking from arm to arm64. Signed-off-by: Punit Agrawal Reviewed-by: Catalin Marinas Signed-off-by: Will Deacon --- arch/arm64/include/asm/opcodes.h | 1 + 1 file changed, 1 insertion(+) create mode 100644 arch/arm64/include/asm/opcodes.h (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/opcodes.h b/arch/arm64/include/asm/opcodes.h new file mode 100644 index 0000000..4e603ea --- /dev/null +++ b/arch/arm64/include/asm/opcodes.h @@ -0,0 +1 @@ +#include <../../arm/include/asm/opcodes.h> -- cgit v1.1 From bd35a4adc4131c530ec7d90242555eac7b3dbe3f Mon Sep 17 00:00:00 2001 From: Punit Agrawal Date: Tue, 18 Nov 2014 11:41:25 +0000 Subject: arm64: Port SWP/SWPB emulation support from arm The SWP instruction was deprecated in the ARMv6 architecture. The ARMv7 multiprocessing extensions mandate that SWP/SWPB instructions are treated as undefined from reset, with the ability to enable them through the System Control Register SW bit. With ARMv8, the option to enable these instructions through System Control Register was dropped as well. To support legacy applications using these instructions, port the emulation of the SWP and SWPB instructions from the arm port to arm64. Reviewed-by: Catalin Marinas Signed-off-by: Punit Agrawal Signed-off-by: Will Deacon --- arch/arm64/include/asm/insn.h | 6 ++++++ 1 file changed, 6 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 1bb0430..3ecc57c 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -356,6 +356,12 @@ int aarch64_insn_patch_text_sync(void *addrs[], u32 insns[], int cnt); int aarch64_insn_patch_text(void *addrs[], u32 insns[], int cnt); bool aarch32_insn_is_wide(u32 insn); + +#define A32_RN_OFFSET 16 +#define A32_RT_OFFSET 12 +#define A32_RT2_OFFSET 0 + +u32 aarch32_insn_extract_reg_num(u32 insn, int offset); #endif /* __ASSEMBLY__ */ #endif /* __ASM_INSN_H */ -- cgit v1.1 From c852f320584600a372646055d8229e063949eee7 Mon Sep 17 00:00:00 2001 From: Punit Agrawal Date: Tue, 18 Nov 2014 11:41:26 +0000 Subject: arm64: Emulate CP15 Barrier instructions The CP15 barrier instructions (CP15ISB, CP15DSB and CP15DMB) are deprecated in the ARMv7 architecture, superseded by ISB, DSB and DMB instructions respectively. Some implementations may provide the ability to disable the CP15 barriers by disabling the CP15BEN bit in SCTLR_EL1. If not enabled, the encodings for these instructions become undefined. To support legacy software using these instructions, this patch register hooks to - * emulate CP15 barriers and warn the user about their use * toggle CP15BEN in SCTLR_EL1 Signed-off-by: Punit Agrawal Reviewed-by: Catalin Marinas Signed-off-by: Will Deacon --- arch/arm64/include/asm/insn.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/insn.h b/arch/arm64/include/asm/insn.h index 3ecc57c..e2ff32a 100644 --- a/arch/arm64/include/asm/insn.h +++ b/arch/arm64/include/asm/insn.h @@ -362,6 +362,8 @@ bool aarch32_insn_is_wide(u32 insn); #define A32_RT2_OFFSET 0 u32 aarch32_insn_extract_reg_num(u32 insn, int offset); +u32 aarch32_insn_mcr_extract_opc2(u32 insn); +u32 aarch32_insn_mcr_extract_crm(u32 insn); #endif /* __ASSEMBLY__ */ #endif /* __ASM_INSN_H */ -- cgit v1.1 From 930da09f5e50dd22fb0a8600388da8677d62d671 Mon Sep 17 00:00:00 2001 From: Andre Przywara Date: Fri, 14 Nov 2014 15:54:07 +0000 Subject: arm64: add cpu_capabilities bitmap For taking note if at least one CPU in the system needs a bug workaround or would benefit from a code optimization, we create a new bitmap to hold (artificial) feature bits. Since elf_hwcap is part of the userland ABI, we keep it alone and introduce a new data structure for that (along with some accessors). Signed-off-by: Andre Przywara Signed-off-by: Will Deacon --- arch/arm64/include/asm/cpufeature.h | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index cd4ac05..20b2b3d6 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -21,9 +21,29 @@ #define MAX_CPU_FEATURES (8 * sizeof(elf_hwcap)) #define cpu_feature(x) ilog2(HWCAP_ ## x) +#define NCAPS 0 + +extern DECLARE_BITMAP(cpu_hwcaps, NCAPS); + static inline bool cpu_have_feature(unsigned int num) { return elf_hwcap & (1UL << num); } +static inline bool cpus_have_cap(unsigned int num) +{ + if (num >= NCAPS) + return false; + return test_bit(num, cpu_hwcaps); +} + +static inline void cpus_set_cap(unsigned int num) +{ + if (num >= NCAPS) + pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n", + num, NCAPS); + else + __set_bit(num, cpu_hwcaps); +} + #endif -- cgit v1.1 From e039ee4ee3fcf174736f2cb0a2eed6cb908348a6 Mon Sep 17 00:00:00 2001 From: Andre Przywara Date: Fri, 14 Nov 2014 15:54:08 +0000 Subject: arm64: add alternative runtime patching With a blatant copy of some x86 bits we introduce the alternative runtime patching "framework" to arm64. This is quite basic for now and we only provide the functions we need at this time. This is connected to the newly introduced feature bits. Signed-off-by: Andre Przywara Signed-off-by: Will Deacon --- arch/arm64/include/asm/alternative-asm.h | 16 ++++++++++++ arch/arm64/include/asm/alternative.h | 43 ++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 arch/arm64/include/asm/alternative-asm.h create mode 100644 arch/arm64/include/asm/alternative.h (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/alternative-asm.h b/arch/arm64/include/asm/alternative-asm.h new file mode 100644 index 0000000..5ee9340 --- /dev/null +++ b/arch/arm64/include/asm/alternative-asm.h @@ -0,0 +1,16 @@ +#ifndef __ASM_ALTERNATIVE_ASM_H +#define __ASM_ALTERNATIVE_ASM_H + +#ifdef __ASSEMBLY__ + +.macro altinstruction_entry orig_offset alt_offset feature orig_len alt_len + .word \orig_offset - . + .word \alt_offset - . + .hword \feature + .byte \orig_len + .byte \alt_len +.endm + +#endif /* __ASSEMBLY__ */ + +#endif /* __ASM_ALTERNATIVE_ASM_H */ diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h new file mode 100644 index 0000000..f6d206e --- /dev/null +++ b/arch/arm64/include/asm/alternative.h @@ -0,0 +1,43 @@ +#ifndef __ASM_ALTERNATIVE_H +#define __ASM_ALTERNATIVE_H + +#include +#include +#include + +struct alt_instr { + s32 orig_offset; /* offset to original instruction */ + s32 alt_offset; /* offset to replacement instruction */ + u16 cpufeature; /* cpufeature bit set for replacement */ + u8 orig_len; /* size of original instruction(s) */ + u8 alt_len; /* size of new instruction(s), <= orig_len */ +}; + +void apply_alternatives(void); +void free_alternatives_memory(void); + +#define ALTINSTR_ENTRY(feature) \ + " .word 661b - .\n" /* label */ \ + " .word 663f - .\n" /* new instruction */ \ + " .hword " __stringify(feature) "\n" /* feature bit */ \ + " .byte 662b-661b\n" /* source len */ \ + " .byte 664f-663f\n" /* replacement len */ + +/* alternative assembly primitive: */ +#define ALTERNATIVE(oldinstr, newinstr, feature) \ + "661:\n\t" \ + oldinstr "\n" \ + "662:\n" \ + ".pushsection .altinstructions,\"a\"\n" \ + ALTINSTR_ENTRY(feature) \ + ".popsection\n" \ + ".pushsection .altinstr_replacement, \"a\"\n" \ + "663:\n\t" \ + newinstr "\n" \ + "664:\n\t" \ + ".popsection\n\t" \ + ".if ((664b-663b) != (662b-661b))\n\t" \ + " .error \"Alternatives instruction length mismatch\"\n\t"\ + ".endif\n" + +#endif /* __ASM_ALTERNATIVE_H */ -- cgit v1.1 From e116a375423393cdb94714e90a96857005d58428 Mon Sep 17 00:00:00 2001 From: Andre Przywara Date: Fri, 14 Nov 2014 15:54:09 +0000 Subject: arm64: detect silicon revisions and set cap bits accordingly After each CPU has been started, we iterate through a list of CPU features or bugs to detect CPUs which need (or could benefit from) kernel code patches. For each feature/bug there is a function which checks if that particular CPU is affected. We will later provide some more generic functions for common things like testing for certain MIDR ranges. We do this for every CPU to cover big.LITTLE systems properly as well. If a certain feature/bug has been detected, the capability bit will be set, so that later the call to apply_alternatives() will trigger the actual code patching. Signed-off-by: Andre Przywara Signed-off-by: Will Deacon --- arch/arm64/include/asm/cpufeature.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 20b2b3d6..744eaf7 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -46,4 +46,6 @@ static inline void cpus_set_cap(unsigned int num) __set_bit(num, cpu_hwcaps); } +void check_local_cpu_errata(void); + #endif -- cgit v1.1 From 301bcfac42897dbd1b0b3c1be49f24654a1bc49e Mon Sep 17 00:00:00 2001 From: Andre Przywara Date: Fri, 14 Nov 2014 15:54:10 +0000 Subject: arm64: add Cortex-A53 cache errata workaround The ARM errata 819472, 826319, 827319 and 824069 define the same workaround for these hardware issues in certain Cortex-A53 parts. Use the new alternatives framework and the CPU MIDR detection to patch "cache clean" into "cache clean and invalidate" instructions if an affected CPU is detected at runtime. Signed-off-by: Andre Przywara [will: add __maybe_unused to squash gcc warning] Signed-off-by: Will Deacon --- arch/arm64/include/asm/alternative-asm.h | 13 +++++++++++++ arch/arm64/include/asm/cpufeature.h | 8 +++++++- arch/arm64/include/asm/cputype.h | 5 +++++ 3 files changed, 25 insertions(+), 1 deletion(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/alternative-asm.h b/arch/arm64/include/asm/alternative-asm.h index 5ee9340..919a678 100644 --- a/arch/arm64/include/asm/alternative-asm.h +++ b/arch/arm64/include/asm/alternative-asm.h @@ -11,6 +11,19 @@ .byte \alt_len .endm +.macro alternative_insn insn1 insn2 cap +661: \insn1 +662: .pushsection .altinstructions, "a" + altinstruction_entry 661b, 663f, \cap, 662b-661b, 664f-663f + .popsection + .pushsection .altinstr_replacement, "ax" +663: \insn2 +664: .popsection + .if ((664b-663b) != (662b-661b)) + .error "Alternatives instruction length mismatch" + .endif +.endm + #endif /* __ASSEMBLY__ */ #endif /* __ASM_ALTERNATIVE_ASM_H */ diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 744eaf7..92b6ee4 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -21,7 +21,11 @@ #define MAX_CPU_FEATURES (8 * sizeof(elf_hwcap)) #define cpu_feature(x) ilog2(HWCAP_ ## x) -#define NCAPS 0 +#define ARM64_WORKAROUND_CLEAN_CACHE 0 + +#define NCAPS 1 + +#ifndef __ASSEMBLY__ extern DECLARE_BITMAP(cpu_hwcaps, NCAPS); @@ -48,4 +52,6 @@ static inline void cpus_set_cap(unsigned int num) void check_local_cpu_errata(void); +#endif /* __ASSEMBLY__ */ + #endif diff --git a/arch/arm64/include/asm/cputype.h b/arch/arm64/include/asm/cputype.h index 379d0b8..8adb986 100644 --- a/arch/arm64/include/asm/cputype.h +++ b/arch/arm64/include/asm/cputype.h @@ -57,6 +57,11 @@ #define MIDR_IMPLEMENTOR(midr) \ (((midr) & MIDR_IMPLEMENTOR_MASK) >> MIDR_IMPLEMENTOR_SHIFT) +#define MIDR_CPU_PART(imp, partnum) \ + (((imp) << MIDR_IMPLEMENTOR_SHIFT) | \ + (0xf << MIDR_ARCHITECTURE_SHIFT) | \ + ((partnum) << MIDR_PARTNUM_SHIFT)) + #define ARM_CPU_IMP_ARM 0x41 #define ARM_CPU_IMP_APM 0x50 -- cgit v1.1 From 5afaa1fc1b320cec48affa7e6949f2493f875c12 Mon Sep 17 00:00:00 2001 From: Andre Przywara Date: Fri, 14 Nov 2014 15:54:11 +0000 Subject: arm64: add Cortex-A57 erratum 832075 workaround The ARM erratum 832075 applies to certain revisions of Cortex-A57, one of the workarounds is to change device loads into using load-aquire semantics. This is achieved using the alternatives framework. Signed-off-by: Andre Przywara Signed-off-by: Will Deacon --- arch/arm64/include/asm/cpufeature.h | 5 +++-- arch/arm64/include/asm/io.h | 23 +++++++++++++++++++---- 2 files changed, 22 insertions(+), 6 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 92b6ee4..0362f80 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -21,9 +21,10 @@ #define MAX_CPU_FEATURES (8 * sizeof(elf_hwcap)) #define cpu_feature(x) ilog2(HWCAP_ ## x) -#define ARM64_WORKAROUND_CLEAN_CACHE 0 +#define ARM64_WORKAROUND_CLEAN_CACHE 0 +#define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE 1 -#define NCAPS 1 +#define NCAPS 2 #ifndef __ASSEMBLY__ diff --git a/arch/arm64/include/asm/io.h b/arch/arm64/include/asm/io.h index 79f1d519..75825b6 100644 --- a/arch/arm64/include/asm/io.h +++ b/arch/arm64/include/asm/io.h @@ -28,6 +28,8 @@ #include #include #include +#include +#include #include @@ -57,28 +59,41 @@ static inline void __raw_writeq(u64 val, volatile void __iomem *addr) static inline u8 __raw_readb(const volatile void __iomem *addr) { u8 val; - asm volatile("ldrb %w0, [%1]" : "=r" (val) : "r" (addr)); + asm volatile(ALTERNATIVE("ldrb %w0, [%1]", + "ldarb %w0, [%1]", + ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) + : "=r" (val) : "r" (addr)); return val; } static inline u16 __raw_readw(const volatile void __iomem *addr) { u16 val; - asm volatile("ldrh %w0, [%1]" : "=r" (val) : "r" (addr)); + + asm volatile(ALTERNATIVE("ldrh %w0, [%1]", + "ldarh %w0, [%1]", + ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) + : "=r" (val) : "r" (addr)); return val; } static inline u32 __raw_readl(const volatile void __iomem *addr) { u32 val; - asm volatile("ldr %w0, [%1]" : "=r" (val) : "r" (addr)); + asm volatile(ALTERNATIVE("ldr %w0, [%1]", + "ldar %w0, [%1]", + ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) + : "=r" (val) : "r" (addr)); return val; } static inline u64 __raw_readq(const volatile void __iomem *addr) { u64 val; - asm volatile("ldr %0, [%1]" : "=r" (val) : "r" (addr)); + asm volatile(ALTERNATIVE("ldr %0, [%1]", + "ldar %0, [%1]", + ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE) + : "=r" (val) : "r" (addr)); return val; } -- cgit v1.1 From 3eebdbe5fc7d64c7a6ef14cc5b8be518ffd563fa Mon Sep 17 00:00:00 2001 From: Mark Rutland Date: Tue, 25 Nov 2014 13:27:43 +0000 Subject: arm64: sanity checks: add ID_AA64DFR{0,1}_EL1 While we currently expect self-hosted debug support to be identical across CPUs, we don't currently sanity check this. This patch adds logging of the ID_AA64DFR{0,1}_EL1 values and associated sanity checking code. It's not clear to me whether we need to check PMUVer, TraceVer, and DebugVer, as we don't currently rely on these fields at all. Signed-off-by: Mark Rutland Cc: Catalin Marinas Acked-by: Will Deacon Signed-off-by: Will Deacon --- arch/arm64/include/asm/cpu.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cpu.h b/arch/arm64/include/asm/cpu.h index 0564430..ace7068 100644 --- a/arch/arm64/include/asm/cpu.h +++ b/arch/arm64/include/asm/cpu.h @@ -30,6 +30,8 @@ struct cpuinfo_arm64 { u32 reg_dczid; u32 reg_midr; + u64 reg_id_aa64dfr0; + u64 reg_id_aa64dfr1; u64 reg_id_aa64isar0; u64 reg_id_aa64isar1; u64 reg_id_aa64mmfr0; -- cgit v1.1 From fcff588633e848aa728a4437ef96d437299ba03d Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Fri, 21 Nov 2014 21:50:38 +0000 Subject: arm64: Treat handle_arch_irq as a function pointer handle_arch_irq isn't actually text, it's just a function pointer. It doesn't need to be stored in the text section and doing so causes problesm if we ever want to make the kernel text read only. Declare handle_arch_irq as a proper function pointer stored in the data section. Reviewed-by: Kees Cook Reviewed-by: Mark Rutland Acked-by: Ard Biesheuvel Tested-by: Mark Rutland Tested-by: Kees Cook Signed-off-by: Laura Abbott Signed-off-by: Will Deacon --- arch/arm64/include/asm/irq.h | 1 - 1 file changed, 1 deletion(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h index e1f7ecd..1eebf5b 100644 --- a/arch/arm64/include/asm/irq.h +++ b/arch/arm64/include/asm/irq.h @@ -3,7 +3,6 @@ #include -extern void (*handle_arch_irq)(struct pt_regs *); extern void migrate_irqs(void); extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); -- cgit v1.1 From af86e5974d3069bd26ebcf7c046c6e59726acaaa Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Fri, 21 Nov 2014 21:50:42 +0000 Subject: arm64: Factor out fixmap initialization from ioremap The fixmap API was originally added for arm64 for early_ioremap purposes. It can be used for other purposes too so move the initialization from ioremap to somewhere more generic. This makes it obvious where the fixmap is being set up and allows for a cleaner implementation of __set_fixmap. Reviewed-by: Kees Cook Acked-by: Mark Rutland Tested-by: Mark Rutland Tested-by: Kees Cook Signed-off-by: Laura Abbott Signed-off-by: Will Deacon --- arch/arm64/include/asm/fixmap.h | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index 5f7bfe6..db26a2f2 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -56,10 +56,11 @@ enum fixed_addresses { #define FIXMAP_PAGE_IO __pgprot(PROT_DEVICE_nGnRE) -extern void __early_set_fixmap(enum fixed_addresses idx, - phys_addr_t phys, pgprot_t flags); +void __init early_fixmap_init(void); -#define __set_fixmap __early_set_fixmap +#define __early_set_fixmap __set_fixmap + +extern void __set_fixmap(enum fixed_addresses idx, phys_addr_t phys, pgprot_t prot); #include -- cgit v1.1 From dab78b6dcb2bfc90038f35ada826844273dde4d6 Mon Sep 17 00:00:00 2001 From: Laura Abbott Date: Wed, 26 Nov 2014 00:14:16 +0000 Subject: arm64: Add FIX_HOLE to permanent fixed addresses Every other architecture with permanent fixed addresses has FIX_HOLE as the first entry. This seems to be designed as a debugging aid but there are a couple of side effects of not having FIX_HOLE: - If the first fixed address is 0, fix_to_virt -> virt_to_fix triggers a BUG_ON for the virtual address being equal to FIXADDR_TOP - fix_to_virt may return a value outside of FIXADDR_START and FIXADDR_TOP which may look like a bug to a developer. Match up with other architectures and make everything clearer by adding FIX_HOLE. Signed-off-by: Laura Abbott Signed-off-by: Will Deacon --- arch/arm64/include/asm/fixmap.h | 1 + 1 file changed, 1 insertion(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/fixmap.h b/arch/arm64/include/asm/fixmap.h index db26a2f2..9ef6eca 100644 --- a/arch/arm64/include/asm/fixmap.h +++ b/arch/arm64/include/asm/fixmap.h @@ -31,6 +31,7 @@ * */ enum fixed_addresses { + FIX_HOLE, FIX_EARLYCON_MEM_BASE, __end_of_permanent_fixed_addresses, -- cgit v1.1 From e185fab7e13087cbd1f7e281d2698f07dbcdd44e Mon Sep 17 00:00:00 2001 From: AKASHI Takahiro Date: Fri, 28 Nov 2014 05:26:37 +0000 Subject: arm64: add seccomp syscall for compat task This patch allows compat task to issue seccomp() system call. Reviewed-by: Kees Cook Signed-off-by: AKASHI Takahiro Signed-off-by: Will Deacon --- arch/arm64/include/asm/unistd32.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/unistd32.h b/arch/arm64/include/asm/unistd32.h index da1f06b..812f192 100644 --- a/arch/arm64/include/asm/unistd32.h +++ b/arch/arm64/include/asm/unistd32.h @@ -787,7 +787,8 @@ __SYSCALL(__NR_sched_setattr, sys_sched_setattr) __SYSCALL(__NR_sched_getattr, sys_sched_getattr) #define __NR_renameat2 382 __SYSCALL(__NR_renameat2, sys_renameat2) - /* 383 for seccomp */ +#define __NR_seccomp 383 +__SYSCALL(__NR_seccomp, sys_seccomp) #define __NR_getrandom 384 __SYSCALL(__NR_getrandom, sys_getrandom) #define __NR_memfd_create 385 -- cgit v1.1 From cc5e9097c9aad6b186a568c534e26746d6bfa483 Mon Sep 17 00:00:00 2001 From: AKASHI Takahiro Date: Fri, 28 Nov 2014 05:26:38 +0000 Subject: arm64: add SIGSYS siginfo for compat task SIGSYS is primarily used in secure computing to notify tracer of syscall events. This patch allows signal handler on compat task to get correct information with SA_SIGINFO specified when this signal is delivered. Reviewed-by: Kees Cook Signed-off-by: AKASHI Takahiro Signed-off-by: Will Deacon --- arch/arm64/include/asm/compat.h | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/compat.h b/arch/arm64/include/asm/compat.h index 56de5aa..3fb053f 100644 --- a/arch/arm64/include/asm/compat.h +++ b/arch/arm64/include/asm/compat.h @@ -205,6 +205,13 @@ typedef struct compat_siginfo { compat_long_t _band; /* POLL_IN, POLL_OUT, POLL_MSG */ int _fd; } _sigpoll; + + /* SIGSYS */ + struct { + compat_uptr_t _call_addr; /* calling user insn */ + int _syscall; /* triggering system call number */ + compat_uint_t _arch; /* AUDIT_ARCH_* of syscall */ + } _sigsys; } _sifields; } compat_siginfo_t; -- cgit v1.1 From a1ae65b219416a72c15577bd4c8c11174fffbb8b Mon Sep 17 00:00:00 2001 From: AKASHI Takahiro Date: Fri, 28 Nov 2014 05:26:39 +0000 Subject: arm64: add seccomp support secure_computing() is called first in syscall_trace_enter() so that a system call will be aborted quickly without doing succeeding syscall tracing if seccomp rules want to deny that system call. On compat task, syscall numbers for system calls allowed in seccomp mode 1 are different from those on normal tasks, and so _NR_seccomp_xxx_32's need to be redefined. Signed-off-by: AKASHI Takahiro Signed-off-by: Will Deacon --- arch/arm64/include/asm/seccomp.h | 25 +++++++++++++++++++++++++ arch/arm64/include/asm/unistd.h | 3 +++ 2 files changed, 28 insertions(+) create mode 100644 arch/arm64/include/asm/seccomp.h (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/seccomp.h b/arch/arm64/include/asm/seccomp.h new file mode 100644 index 0000000..c76fac9 --- /dev/null +++ b/arch/arm64/include/asm/seccomp.h @@ -0,0 +1,25 @@ +/* + * arch/arm64/include/asm/seccomp.h + * + * Copyright (C) 2014 Linaro Limited + * Author: AKASHI Takahiro + * + * This program is free software; you can redistribute it and/or modify + * it under the terms of the GNU General Public License version 2 as + * published by the Free Software Foundation. + */ +#ifndef _ASM_SECCOMP_H +#define _ASM_SECCOMP_H + +#include + +#ifdef CONFIG_COMPAT +#define __NR_seccomp_read_32 __NR_compat_read +#define __NR_seccomp_write_32 __NR_compat_write +#define __NR_seccomp_exit_32 __NR_compat_exit +#define __NR_seccomp_sigreturn_32 __NR_compat_rt_sigreturn +#endif /* CONFIG_COMPAT */ + +#include + +#endif /* _ASM_SECCOMP_H */ diff --git a/arch/arm64/include/asm/unistd.h b/arch/arm64/include/asm/unistd.h index 6d2bf41..49c9aef 100644 --- a/arch/arm64/include/asm/unistd.h +++ b/arch/arm64/include/asm/unistd.h @@ -31,6 +31,9 @@ * Compat syscall numbers used by the AArch64 kernel. */ #define __NR_compat_restart_syscall 0 +#define __NR_compat_exit 1 +#define __NR_compat_read 3 +#define __NR_compat_write 4 #define __NR_compat_sigreturn 119 #define __NR_compat_rt_sigreturn 173 -- cgit v1.1 From a2d25a5391ca219f196f9fee7b535c40d201c6bf Mon Sep 17 00:00:00 2001 From: Vladimir Murzin Date: Mon, 1 Dec 2014 10:53:08 +0000 Subject: arm64: compat: align cacheflush syscall with arch/arm Update handling of cacheflush syscall with changes made in arch/arm counterpart: - return error to userspace when flushing syscall fails - split user cache-flushing into interruptible chunks - don't bother rounding to nearest vma Signed-off-by: Vladimir Murzin [will: changed internal return value from -EINTR to 0 to match arch/arm/] Signed-off-by: Will Deacon --- arch/arm64/include/asm/cacheflush.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cacheflush.h b/arch/arm64/include/asm/cacheflush.h index 689b637..7ae31a2 100644 --- a/arch/arm64/include/asm/cacheflush.h +++ b/arch/arm64/include/asm/cacheflush.h @@ -73,7 +73,7 @@ extern void flush_cache_all(void); extern void flush_cache_range(struct vm_area_struct *vma, unsigned long start, unsigned long end); extern void flush_icache_range(unsigned long start, unsigned long end); extern void __flush_dcache_area(void *addr, size_t len); -extern void __flush_cache_user_range(unsigned long start, unsigned long end); +extern long __flush_cache_user_range(unsigned long start, unsigned long end); static inline void flush_cache_mm(struct mm_struct *mm) { -- cgit v1.1 From e4f88d833bec29b8e6fadc1b2488f0c6370935e1 Mon Sep 17 00:00:00 2001 From: Jungseok Lee Date: Tue, 2 Dec 2014 17:49:24 +0000 Subject: arm64: Implement support for read-mostly sections As putting data which is read mostly together, we can avoid unnecessary cache line bouncing. Other architectures, such as ARM and x86, adopted the same idea. Acked-by: Catalin Marinas Signed-off-by: Jungseok Lee Signed-off-by: Will Deacon --- arch/arm64/include/asm/cache.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cache.h b/arch/arm64/include/asm/cache.h index 88cc05b..bde4499 100644 --- a/arch/arm64/include/asm/cache.h +++ b/arch/arm64/include/asm/cache.h @@ -32,6 +32,8 @@ #ifndef __ASSEMBLY__ +#define __read_mostly __attribute__((__section__(".data..read_mostly"))) + static inline int cache_line_size(void) { u32 cwg = cache_type_cwg(); -- cgit v1.1 From 06f9eb884be81431d54d7d37390043e3b5b7f14a Mon Sep 17 00:00:00 2001 From: Fabio Estevam Date: Thu, 4 Dec 2014 01:17:01 +0000 Subject: arm64: Provide a namespace to NCAPS Building arm64.allmodconfig leads to the following warning: usb/gadget/function/f_ncm.c:203:0: warning: "NCAPS" redefined #define NCAPS (USB_CDC_NCM_NCAP_ETH_FILTER | USB_CDC_NCM_NCAP_CRC_MODE) ^ In file included from /home/build/work/batch/arch/arm64/include/asm/io.h:32:0, from /home/build/work/batch/include/linux/clocksource.h:19, from /home/build/work/batch/include/clocksource/arm_arch_timer.h:19, from /home/build/work/batch/arch/arm64/include/asm/arch_timer.h:27, from /home/build/work/batch/arch/arm64/include/asm/timex.h:19, from /home/build/work/batch/include/linux/timex.h:65, from /home/build/work/batch/include/linux/sched.h:19, from /home/build/work/batch/arch/arm64/include/asm/compat.h:25, from /home/build/work/batch/arch/arm64/include/asm/stat.h:23, from /home/build/work/batch/include/linux/stat.h:5, from /home/build/work/batch/include/linux/module.h:10, from /home/build/work/batch/drivers/usb/gadget/function/f_ncm.c:19: arch/arm64/include/asm/cpufeature.h:27:0: note: this is the location of the previous definition #define NCAPS 2 So add a ARM64 prefix to avoid such problem. Reported-by: Olof's autobuilder Signed-off-by: Fabio Estevam Signed-off-by: Will Deacon --- arch/arm64/include/asm/cpufeature.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/cpufeature.h b/arch/arm64/include/asm/cpufeature.h index 0362f80..07547cc 100644 --- a/arch/arm64/include/asm/cpufeature.h +++ b/arch/arm64/include/asm/cpufeature.h @@ -24,11 +24,11 @@ #define ARM64_WORKAROUND_CLEAN_CACHE 0 #define ARM64_WORKAROUND_DEVICE_LOAD_ACQUIRE 1 -#define NCAPS 2 +#define ARM64_NCAPS 2 #ifndef __ASSEMBLY__ -extern DECLARE_BITMAP(cpu_hwcaps, NCAPS); +extern DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS); static inline bool cpu_have_feature(unsigned int num) { @@ -37,16 +37,16 @@ static inline bool cpu_have_feature(unsigned int num) static inline bool cpus_have_cap(unsigned int num) { - if (num >= NCAPS) + if (num >= ARM64_NCAPS) return false; return test_bit(num, cpu_hwcaps); } static inline void cpus_set_cap(unsigned int num) { - if (num >= NCAPS) + if (num >= ARM64_NCAPS) pr_warn("Attempt to set an illegal CPU capability (%d >= %d)\n", - num, NCAPS); + num, ARM64_NCAPS); else __set_bit(num, cpu_hwcaps); } -- cgit v1.1 From af2c632e234f7158e891c27cc2270f8843f08855 Mon Sep 17 00:00:00 2001 From: Chunyan Zhang Date: Thu, 4 Dec 2014 06:29:35 +0000 Subject: arm64/include/asm: Fixed a warning about 'struct pt_regs' MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit If I include asm/irq.h on the top of my code, and set ARCH=arm64, I'll get a compile warning, details are below: warning: ‘struct pt_regs’ declared inside parameter list [enabled by default] This patch is suggested by Arnd, see: http://lists.infradead.org/pipermail/linux-arm-kernel/2014-December/308270.html Signed-off-by: Chunyan Zhang Signed-off-by: Will Deacon --- arch/arm64/include/asm/irq.h | 2 ++ 1 file changed, 2 insertions(+) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/irq.h b/arch/arm64/include/asm/irq.h index 1eebf5b..94c5367 100644 --- a/arch/arm64/include/asm/irq.h +++ b/arch/arm64/include/asm/irq.h @@ -3,6 +3,8 @@ #include +struct pt_regs; + extern void migrate_irqs(void); extern void set_handle_irq(void (*handle_irq)(struct pt_regs *)); -- cgit v1.1 From 932ded4b0b9bf111fbf9d176ec12152a0d29b0fd Mon Sep 17 00:00:00 2001 From: Andre Przywara Date: Fri, 28 Nov 2014 13:40:45 +0000 Subject: arm64: add module support for alternatives fixups Currently the kernel patches all necessary instructions once at boot time, so modules are not covered by this. Change the apply_alternatives() function to take a beginning and an end pointer and introduce a new variant (apply_alternatives_all()) to cover the existing use case for the static kernel image section. Add a module_finalize() function to arm64 to check for an alternatives section in a module and patch only the instructions from that specific area. Since that module code is not touched before the module initialization has ended, we don't need to halt the machine before doing the patching in the module's code. Signed-off-by: Andre Przywara Signed-off-by: Will Deacon --- arch/arm64/include/asm/alternative.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'arch/arm64/include/asm') diff --git a/arch/arm64/include/asm/alternative.h b/arch/arm64/include/asm/alternative.h index f6d206e..d261f01 100644 --- a/arch/arm64/include/asm/alternative.h +++ b/arch/arm64/include/asm/alternative.h @@ -13,7 +13,8 @@ struct alt_instr { u8 alt_len; /* size of new instruction(s), <= orig_len */ }; -void apply_alternatives(void); +void apply_alternatives_all(void); +void apply_alternatives(void *start, size_t length); void free_alternatives_memory(void); #define ALTINSTR_ENTRY(feature) \ -- cgit v1.1