From 1ec3f937102154b54c3ba97b79d4bd3931bb0eaa Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Mon, 22 Feb 2016 13:41:12 +1100 Subject: powerpc/mm/book3s-64: Clean up some obsolete or misleading comments No code changes. Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash.h | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index 8d1c816..9a0a4ef 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -4,8 +4,7 @@ /* * Common bits between 4K and 64K pages in a linux-style PTE. - * These match the bits in the (hardware-defined) PowerPC PTE as closely - * as possible. Additional bits may be defined in pgtable-hash64-*.h + * Additional bits may be defined in pgtable-hash64-*.h * * Note: We only support user read/write permissions. Supervisor always * have full read/write to pages above PAGE_OFFSET (pages below that @@ -14,13 +13,13 @@ * We could create separate kernel read-only if we used the 3 PP bits * combinations that newer processors provide but we currently don't. */ -#define _PAGE_PTE 0x00001 +#define _PAGE_PTE 0x00001 /* distinguishes PTEs from pointers */ #define _PAGE_PRESENT 0x00002 /* software: pte contains a translation */ #define _PAGE_BIT_SWAP_TYPE 2 -#define _PAGE_USER 0x00004 /* matches one of the PP bits */ -#define _PAGE_EXEC 0x00008 /* No execute on POWER4 and newer (we invert) */ -#define _PAGE_GUARDED 0x00010 -/* We can derive Memory coherence from _PAGE_NO_CACHE */ +#define _PAGE_USER 0x00004 /* page may be accessed by userspace */ +#define _PAGE_EXEC 0x00008 /* execute permission */ +#define _PAGE_GUARDED 0x00010 /* G: guarded (side-effect) page */ +/* M (memory coherence) is always set in the HPTE, so we don't need it here */ #define _PAGE_COHERENT 0x0 #define _PAGE_NO_CACHE 0x00020 /* I: cache inhibit */ #define _PAGE_WRITETHRU 0x00040 /* W: cache write-through */ -- cgit v1.1 From f1a9ae034a263d93ea408825afe699aebf37010a Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Mon, 22 Feb 2016 13:41:13 +1100 Subject: powerpc/mm/book3s-64: Free up 7 high-order bits in the Linux PTE This frees up bits 57-63 in the Linux PTE on 64-bit Book 3S machines. In the 4k page case, this is done just by reducing the size of the RPN field to 39 bits, giving 51-bit real addresses. In the 64k page case, we had 10 unused bits in the middle of the PTE, so this moves the RPN field down 10 bits to make use of those unused bits. This means the RPN field is now 3 bits larger at 37 bits, giving 53-bit real addresses in the normal case, or 49-bit real addresses for the special 4k PFN case. We are doing this in order to be able to move some other PTE bits into the positions where PowerISA V3.0 processors will expect to find them in radix-tree mode. Ultimately we will be able to move the RPN field to lower bit positions and make it larger. Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-4k.h | 1 + arch/powerpc/include/asm/book3s/64/hash-64k.h | 10 ++++++---- arch/powerpc/include/asm/book3s/64/hash.h | 6 +++--- arch/powerpc/include/asm/book3s/64/pgtable.h | 6 +++--- 4 files changed, 13 insertions(+), 10 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index ea0414d..bee3643 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -53,6 +53,7 @@ /* shift to put page number into pte */ #define PTE_RPN_SHIFT (18) +#define PTE_RPN_SIZE (39) /* gives 51-bit real addresses */ #define _PAGE_4K_PFN 0 #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index 849bbec..a8c4c2a 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -39,10 +39,12 @@ /* Shift to put page number into pte. * - * That gives us a max RPN of 34 bits, which means a max of 50 bits - * of addressable physical space, or 46 bits for the special 4k PFNs. + * That gives us a max RPN of 37 bits, which means a max of 53 bits + * of addressable physical space, or 49 bits for the special 4k PFNs. */ -#define PTE_RPN_SHIFT (30) +#define PTE_RPN_SHIFT (20) +#define PTE_RPN_SIZE (37) + /* * we support 16 fragments per PTE page of 64K size. */ @@ -120,7 +122,7 @@ extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index); (((pte) & _PAGE_COMBO)? MMU_PAGE_4K: MMU_PAGE_64K) #define remap_4k_pfn(vma, addr, pfn, prot) \ - (WARN_ON(((pfn) >= (1UL << (64 - PTE_RPN_SHIFT)))) ? -EINVAL : \ + (WARN_ON(((pfn) >= (1UL << PTE_RPN_SIZE))) ? -EINVAL : \ remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, \ __pgprot(pgprot_val((prot)) | _PAGE_4K_PFN))) diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index 9a0a4ef..64eff40 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -131,7 +131,7 @@ * The mask convered by the RPN must be a ULL on 32-bit platforms with * 64-bit PTEs */ -#define PTE_RPN_MASK (~((1UL << PTE_RPN_SHIFT) - 1)) +#define PTE_RPN_MASK (((1UL << PTE_RPN_SIZE) - 1) << PTE_RPN_SHIFT) /* * _PAGE_CHG_MASK masks of bits that are to be preserved across * pgprot changes @@ -412,13 +412,13 @@ static inline int pte_present(pte_t pte) */ static inline pte_t pfn_pte(unsigned long pfn, pgprot_t pgprot) { - return __pte(((pte_basic_t)(pfn) << PTE_RPN_SHIFT) | + return __pte((((pte_basic_t)(pfn) << PTE_RPN_SHIFT) & PTE_RPN_MASK) | pgprot_val(pgprot)); } static inline unsigned long pte_pfn(pte_t pte) { - return pte_val(pte) >> PTE_RPN_SHIFT; + return (pte_val(pte) & PTE_RPN_MASK) >> PTE_RPN_SHIFT; } /* Generic modifiers for PTE bits */ diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index ac07a30..c8240b7 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -154,10 +154,10 @@ static inline void pgd_set(pgd_t *pgdp, unsigned long val) #define SWP_TYPE_BITS 5 #define __swp_type(x) (((x).val >> _PAGE_BIT_SWAP_TYPE) \ & ((1UL << SWP_TYPE_BITS) - 1)) -#define __swp_offset(x) ((x).val >> PTE_RPN_SHIFT) +#define __swp_offset(x) (((x).val & PTE_RPN_MASK) >> PTE_RPN_SHIFT) #define __swp_entry(type, offset) ((swp_entry_t) { \ - ((type) << _PAGE_BIT_SWAP_TYPE) \ - | ((offset) << PTE_RPN_SHIFT) }) + ((type) << _PAGE_BIT_SWAP_TYPE) \ + | (((offset) << PTE_RPN_SHIFT) & PTE_RPN_MASK)}) /* * swp_entry_t must be independent of pte bits. We build a swp_entry_t from * swap type and offset we get from swap and convert that to pte to find a -- cgit v1.1 From c61a8843124e353f4ba27c073133868da00e0335 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Tue, 23 Feb 2016 13:36:17 +1100 Subject: powerpc/mm/book3s-64: Use physical addresses in upper page table tree levels This changes the Linux page tables to store physical addresses rather than kernel virtual addresses in the upper levels of the tree (pgd, pud and pmd) for 64-bit Book 3S machines. This also changes the hugepd pointers used to implement hugepages when the base page size is 4k to store physical addresses rather than virtual addresses (again just for 64-bit Book3S machines). This frees up some high order bits, and will be needed with PowerISA v3.0 machines which read the page table tree in hardware in radix mode. Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-4k.h | 2 +- arch/powerpc/include/asm/book3s/64/hash.h | 13 +++++++------ 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index bee3643..0425d3e 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -64,7 +64,7 @@ #define pgd_none(pgd) (!pgd_val(pgd)) #define pgd_bad(pgd) (pgd_val(pgd) == 0) #define pgd_present(pgd) (pgd_val(pgd) != 0) -#define pgd_page_vaddr(pgd) (pgd_val(pgd) & ~PGD_MASKED_BITS) +#define pgd_page_vaddr(pgd) __va(pgd_val(pgd) & ~PGD_MASKED_BITS) static inline void pgd_clear(pgd_t *pgdp) { diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index 64eff40..5b8ba60 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -222,13 +222,14 @@ #define PUD_BAD_BITS (PMD_TABLE_SIZE-1) #ifndef __ASSEMBLY__ -#define pmd_bad(pmd) (!is_kernel_addr(pmd_val(pmd)) \ - || (pmd_val(pmd) & PMD_BAD_BITS)) -#define pmd_page_vaddr(pmd) (pmd_val(pmd) & ~PMD_MASKED_BITS) +#define pmd_bad(pmd) (pmd_val(pmd) & PMD_BAD_BITS) +#define pmd_page_vaddr(pmd) __va(pmd_val(pmd) & ~PMD_MASKED_BITS) -#define pud_bad(pud) (!is_kernel_addr(pud_val(pud)) \ - || (pud_val(pud) & PUD_BAD_BITS)) -#define pud_page_vaddr(pud) (pud_val(pud) & ~PUD_MASKED_BITS) +#define pud_bad(pud) (pud_val(pud) & PUD_BAD_BITS) +#define pud_page_vaddr(pud) __va(pud_val(pud) & ~PUD_MASKED_BITS) + +/* Pointers in the page table tree are physical addresses */ +#define __pgtable_ptr_val(ptr) __pa(ptr) #define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1)) #define pmd_index(address) (((address) >> (PMD_SHIFT)) & (PTRS_PER_PMD - 1)) -- cgit v1.1 From 849f86a630e9c84bf4c9d5dcbfe59dc94b2e15ce Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Mon, 22 Feb 2016 13:41:15 +1100 Subject: powerpc/mm/book3s-64: Move _PAGE_PRESENT to the most significant bit This changes _PAGE_PRESENT for 64-bit Book 3S processors from 0x2 to 0x8000_0000_0000_0000, because that is where PowerISA v3.0 CPUs in radix mode will expect to find it. Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-64k.h | 10 +++++----- arch/powerpc/include/asm/book3s/64/hash.h | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index a8c4c2a..ed390e1 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -210,30 +210,30 @@ static inline char *get_hpte_slot_array(pmd_t *pmdp) /* * The linux hugepage PMD now include the pmd entries followed by the address * to the stashed pgtable_t. The stashed pgtable_t contains the hpte bits. - * [ 1 bit secondary | 3 bit hidx | 1 bit valid | 000]. We use one byte per + * [ 000 | 1 bit secondary | 3 bit hidx | 1 bit valid]. We use one byte per * each HPTE entry. With 16MB hugepage and 64K HPTE we need 256 entries and * with 4K HPTE we need 4096 entries. Both will fit in a 4K pgtable_t. * - * The last three bits are intentionally left to zero. This memory location + * The top three bits are intentionally left as zero. This memory location * are also used as normal page PTE pointers. So if we have any pointers * left around while we collapse a hugepage, we need to make sure * _PAGE_PRESENT bit of that is zero when we look at them */ static inline unsigned int hpte_valid(unsigned char *hpte_slot_array, int index) { - return (hpte_slot_array[index] >> 3) & 0x1; + return hpte_slot_array[index] & 0x1; } static inline unsigned int hpte_hash_index(unsigned char *hpte_slot_array, int index) { - return hpte_slot_array[index] >> 4; + return hpte_slot_array[index] >> 1; } static inline void mark_hpte_slot_valid(unsigned char *hpte_slot_array, unsigned int index, unsigned int hidx) { - hpte_slot_array[index] = hidx << 4 | 0x1 << 3; + hpte_slot_array[index] = (hidx << 1) | 0x1; } /* diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index 5b8ba60..36ff107 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -14,7 +14,6 @@ * combinations that newer processors provide but we currently don't. */ #define _PAGE_PTE 0x00001 /* distinguishes PTEs from pointers */ -#define _PAGE_PRESENT 0x00002 /* software: pte contains a translation */ #define _PAGE_BIT_SWAP_TYPE 2 #define _PAGE_USER 0x00004 /* page may be accessed by userspace */ #define _PAGE_EXEC 0x00008 /* execute permission */ @@ -39,6 +38,8 @@ #define _PAGE_SOFT_DIRTY 0x00000 #endif +#define _PAGE_PRESENT (1ul << 63) /* pte contains a translation */ + /* * We need to differentiate between explicit huge page and THP huge * page, since THP huge page also need to track real subpage details @@ -402,7 +403,7 @@ static inline int pte_protnone(pte_t pte) static inline int pte_present(pte_t pte) { - return pte_val(pte) & _PAGE_PRESENT; + return !!(pte_val(pte) & _PAGE_PRESENT); } /* Conversion functions: convert a page and protection to a page entry, -- cgit v1.1 From 84c957560a7a8f548aaf6b6b2b15e6d03b7249e2 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Mon, 22 Feb 2016 13:41:16 +1100 Subject: powerpc/mm/book3s-64: Move _PAGE_PTE to 2nd most significant bit This changes _PAGE_PTE for 64-bit Book 3S processors from 0x1 to 0x4000_0000_0000_0000, because that bit is used as the L (leaf) bit by PowerISA v3.0 CPUs in radix mode. The "leaf" bit indicates that the PTE points to a page directly rather than another radix level, which is what the _PAGE_PTE bit means. Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index 36ff107..14cfd49 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -13,7 +13,6 @@ * We could create separate kernel read-only if we used the 3 PP bits * combinations that newer processors provide but we currently don't. */ -#define _PAGE_PTE 0x00001 /* distinguishes PTEs from pointers */ #define _PAGE_BIT_SWAP_TYPE 2 #define _PAGE_USER 0x00004 /* page may be accessed by userspace */ #define _PAGE_EXEC 0x00008 /* execute permission */ @@ -38,6 +37,7 @@ #define _PAGE_SOFT_DIRTY 0x00000 #endif +#define _PAGE_PTE (1ul << 62) /* distinguishes PTEs from pointers */ #define _PAGE_PRESENT (1ul << 63) /* pte contains a translation */ /* -- cgit v1.1 From a9d4996df1f49dc7b45e55060436cace87f1e0d4 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Mon, 22 Feb 2016 13:41:17 +1100 Subject: powerpc/mm/book3s-64: Move HPTE-related bits in PTE to upper end This moves the _PAGE_HASHPTE, _PAGE_F_GIX and _PAGE_F_SECOND fields in the Linux PTE on 64-bit Book 3S systems to the most significant byte. Of the 5 bits, one is a software-use bit and the other four are reserved bit positions in the PowerISA v3.0 radix PTE format. Using these bits is OK because these bits are all to do with tracking the HPTE(s) associated with the Linux PTE, and therefore won't be needed in radix mode. This frees up bit positions in the lower two bytes. Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash.h | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index 14cfd49..a59cfae 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -24,11 +24,7 @@ #define _PAGE_DIRTY 0x00080 /* C: page changed */ #define _PAGE_ACCESSED 0x00100 /* R: page referenced */ #define _PAGE_RW 0x00200 /* software: user write access allowed */ -#define _PAGE_HASHPTE 0x00400 /* software: pte has an associated HPTE */ #define _PAGE_BUSY 0x00800 /* software: PTE & hash are busy */ -#define _PAGE_F_GIX 0x07000 /* full page: hidx bits */ -#define _PAGE_F_GIX_SHIFT 12 -#define _PAGE_F_SECOND 0x08000 /* Whether to use secondary hash or not */ #define _PAGE_SPECIAL 0x10000 /* software: special page */ #ifdef CONFIG_MEM_SOFT_DIRTY @@ -37,6 +33,10 @@ #define _PAGE_SOFT_DIRTY 0x00000 #endif +#define _PAGE_F_GIX_SHIFT 57 +#define _PAGE_F_GIX (7ul << 57) /* HPTE index within HPTEG */ +#define _PAGE_F_SECOND (1ul << 60) /* HPTE is in 2ndary HPTEG */ +#define _PAGE_HASHPTE (1ul << 61) /* PTE has associated HPTE */ #define _PAGE_PTE (1ul << 62) /* distinguishes PTEs from pointers */ #define _PAGE_PRESENT (1ul << 63) /* pte contains a translation */ -- cgit v1.1 From c915df162d09f0957baf56fd3e2f03b8992699a6 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Mon, 22 Feb 2016 13:41:18 +1100 Subject: powerpc/mm/book3s-64: Shuffle read, write, execute and user bits in PTE This moves the _PAGE_EXEC, _PAGE_RW and _PAGE_USER bits around in the Linux PTE on 64-bit Book 3S systems to correspond with the bit positions used in radix mode by PowerISA v3.0 CPUs. This also adds a _PAGE_READ bit corresponding to the read permission bit in the radix PTE. _PAGE_READ is currently unused but could possibly be used in future to improve pte_protnone(). Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index a59cfae..cd4bf95 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -13,9 +13,12 @@ * We could create separate kernel read-only if we used the 3 PP bits * combinations that newer processors provide but we currently don't. */ -#define _PAGE_BIT_SWAP_TYPE 2 -#define _PAGE_USER 0x00004 /* page may be accessed by userspace */ -#define _PAGE_EXEC 0x00008 /* execute permission */ +#define _PAGE_BIT_SWAP_TYPE 0 + +#define _PAGE_EXEC 0x00001 /* execute permission */ +#define _PAGE_RW 0x00002 /* read & write access allowed */ +#define _PAGE_READ 0x00004 /* read access allowed */ +#define _PAGE_USER 0x00008 /* page may be accessed by userspace */ #define _PAGE_GUARDED 0x00010 /* G: guarded (side-effect) page */ /* M (memory coherence) is always set in the HPTE, so we don't need it here */ #define _PAGE_COHERENT 0x0 @@ -23,7 +26,6 @@ #define _PAGE_WRITETHRU 0x00040 /* W: cache write-through */ #define _PAGE_DIRTY 0x00080 /* C: page changed */ #define _PAGE_ACCESSED 0x00100 /* R: page referenced */ -#define _PAGE_RW 0x00200 /* software: user write access allowed */ #define _PAGE_BUSY 0x00800 /* software: PTE & hash are busy */ #define _PAGE_SPECIAL 0x10000 /* software: special page */ -- cgit v1.1 From e726202f06cbfda4e42d2b32138cf288445ec80d Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Mon, 22 Feb 2016 13:41:19 +1100 Subject: powerpc/mm/book3s-64: Move software-used bits in PTE This moves the _PAGE_SPECIAL and _PAGE_SOFT_DIRTY bits in the Linux PTE on 64-bit Book 3S systems to bit positions which are designated for software use in the radix PTE format used by PowerISA v3.0 CPUs in radix mode. Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index cd4bf95..ef9bd68 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -26,13 +26,13 @@ #define _PAGE_WRITETHRU 0x00040 /* W: cache write-through */ #define _PAGE_DIRTY 0x00080 /* C: page changed */ #define _PAGE_ACCESSED 0x00100 /* R: page referenced */ +#define _PAGE_SPECIAL 0x00400 /* software: special page */ #define _PAGE_BUSY 0x00800 /* software: PTE & hash are busy */ -#define _PAGE_SPECIAL 0x10000 /* software: special page */ #ifdef CONFIG_MEM_SOFT_DIRTY -#define _PAGE_SOFT_DIRTY 0x20000 /* software: software dirty tracking */ +#define _PAGE_SOFT_DIRTY 0x200 /* software: software dirty tracking */ #else -#define _PAGE_SOFT_DIRTY 0x00000 +#define _PAGE_SOFT_DIRTY 0x000 #endif #define _PAGE_F_GIX_SHIFT 57 -- cgit v1.1 From 8daf51f55febbc755fe20e74320a194e175811a9 Mon Sep 17 00:00:00 2001 From: Paul Mackerras Date: Mon, 22 Feb 2016 13:41:20 +1100 Subject: powerpc/mm/book3s-64: Expand the real page number field of the Linux PTE Now that other PTE fields have been moved out of the way, we can expand the RPN field of the PTE on 64-bit Book 3S systems and align it with the RPN field in the radix PTE format used by PowerISA v3.0 CPUs in radix mode. For 64k page size, this means we need to move the _PAGE_COMBO and _PAGE_4K_PFN bits. Signed-off-by: Paul Mackerras Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-4k.h | 4 ++-- arch/powerpc/include/asm/book3s/64/hash-64k.h | 12 ++++++------ 2 files changed, 8 insertions(+), 8 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index 0425d3e..7f60f7e 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -52,8 +52,8 @@ _PAGE_F_SECOND | _PAGE_F_GIX) /* shift to put page number into pte */ -#define PTE_RPN_SHIFT (18) -#define PTE_RPN_SIZE (39) /* gives 51-bit real addresses */ +#define PTE_RPN_SHIFT (12) +#define PTE_RPN_SIZE (45) /* gives 57-bit real addresses */ #define _PAGE_4K_PFN 0 #ifndef __ASSEMBLY__ diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index ed390e1..8bb0325 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -25,8 +25,8 @@ #define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) -#define _PAGE_COMBO 0x00040000 /* this is a combo 4k page */ -#define _PAGE_4K_PFN 0x00080000 /* PFN is for a single 4k page */ +#define _PAGE_COMBO 0x00001000 /* this is a combo 4k page */ +#define _PAGE_4K_PFN 0x00002000 /* PFN is for a single 4k page */ /* * Used to track subpage group valid if _PAGE_COMBO is set * This overloads _PAGE_F_GIX and _PAGE_F_SECOND @@ -39,11 +39,11 @@ /* Shift to put page number into pte. * - * That gives us a max RPN of 37 bits, which means a max of 53 bits - * of addressable physical space, or 49 bits for the special 4k PFNs. + * That gives us a max RPN of 41 bits, which means a max of 57 bits + * of addressable physical space, or 53 bits for the special 4k PFNs. */ -#define PTE_RPN_SHIFT (20) -#define PTE_RPN_SIZE (37) +#define PTE_RPN_SHIFT (16) +#define PTE_RPN_SIZE (41) /* * we support 16 fragments per PTE page of 64K size. -- cgit v1.1 From ae9a71afa4d7cf29a816fbc387bfd59ad7c292b6 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Tue, 1 Mar 2016 09:45:12 +0530 Subject: powerpc/mm: Don't have conditional defines for real_pte_t We remove real_pte_t out of STRICT_MM_TYPESCHECK. Reviewed-by: Paul Mackerras Signed-off-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/pgtable.h | 5 ----- 1 file changed, 5 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index c8240b7..7482f69 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -43,13 +43,8 @@ */ #ifndef __real_pte -#ifdef CONFIG_STRICT_MM_TYPECHECKS #define __real_pte(e,p) ((real_pte_t){(e)}) #define __rpte_to_pte(r) ((r).pte) -#else -#define __real_pte(e,p) (e) -#define __rpte_to_pte(r) (__pte(r)) -#endif #define __rpte_to_hidx(r,index) (pte_val(__rpte_to_pte(r)) >>_PAGE_F_GIX_SHIFT) #define pte_iterate_hashed_subpages(rpte, psize, va, index, shift) \ -- cgit v1.1 From 368ced78e6ed3d72c2acc61233b58487071ec289 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Tue, 1 Mar 2016 09:45:13 +0530 Subject: powerpc/mm: Switch book3s 64 with 64K page size to 4 level page table This is needed so that we can support both hash and radix page table using single kernel. Radix kernel uses a 4 level table. We now use physical address in upper page table tree levels. Even though they are aligned to their size, for the masked bits we use the bit positions as per PowerISA 3.0. Signed-off-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/hash-4k.h | 33 +-------------------------- arch/powerpc/include/asm/book3s/64/hash-64k.h | 33 ++++++++++++++------------- arch/powerpc/include/asm/book3s/64/hash.h | 11 +++++++++ arch/powerpc/include/asm/book3s/64/pgtable.h | 25 +++++++++++++++++++- 4 files changed, 53 insertions(+), 49 deletions(-) (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/hash-4k.h b/arch/powerpc/include/asm/book3s/64/hash-4k.h index 7f60f7e..5f08a08 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-4k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-4k.h @@ -58,39 +58,8 @@ #define _PAGE_4K_PFN 0 #ifndef __ASSEMBLY__ /* - * 4-level page tables related bits + * On all 4K setups, remap_4k_pfn() equates to remap_pfn_range() */ - -#define pgd_none(pgd) (!pgd_val(pgd)) -#define pgd_bad(pgd) (pgd_val(pgd) == 0) -#define pgd_present(pgd) (pgd_val(pgd) != 0) -#define pgd_page_vaddr(pgd) __va(pgd_val(pgd) & ~PGD_MASKED_BITS) - -static inline void pgd_clear(pgd_t *pgdp) -{ - *pgdp = __pgd(0); -} - -static inline pte_t pgd_pte(pgd_t pgd) -{ - return __pte(pgd_val(pgd)); -} - -static inline pgd_t pte_pgd(pte_t pte) -{ - return __pgd(pte_val(pte)); -} -extern struct page *pgd_page(pgd_t pgd); - -#define pud_offset(pgdp, addr) \ - (((pud_t *) pgd_page_vaddr(*(pgdp))) + \ - (((addr) >> PUD_SHIFT) & (PTRS_PER_PUD - 1))) - -#define pud_ERROR(e) \ - pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) - -/* - * On all 4K setups, remap_4k_pfn() equates to remap_pfn_range() */ #define remap_4k_pfn(vma, addr, pfn, prot) \ remap_pfn_range((vma), (addr), (pfn), PAGE_SIZE, (prot)) diff --git a/arch/powerpc/include/asm/book3s/64/hash-64k.h b/arch/powerpc/include/asm/book3s/64/hash-64k.h index 8bb0325..0a7956a 100644 --- a/arch/powerpc/include/asm/book3s/64/hash-64k.h +++ b/arch/powerpc/include/asm/book3s/64/hash-64k.h @@ -1,15 +1,14 @@ #ifndef _ASM_POWERPC_BOOK3S_64_HASH_64K_H #define _ASM_POWERPC_BOOK3S_64_HASH_64K_H -#include - #define PTE_INDEX_SIZE 8 -#define PMD_INDEX_SIZE 10 -#define PUD_INDEX_SIZE 0 +#define PMD_INDEX_SIZE 5 +#define PUD_INDEX_SIZE 5 #define PGD_INDEX_SIZE 12 #define PTRS_PER_PTE (1 << PTE_INDEX_SIZE) #define PTRS_PER_PMD (1 << PMD_INDEX_SIZE) +#define PTRS_PER_PUD (1 << PUD_INDEX_SIZE) #define PTRS_PER_PGD (1 << PGD_INDEX_SIZE) /* With 4k base page size, hugepage PTEs go at the PMD level */ @@ -20,8 +19,13 @@ #define PMD_SIZE (1UL << PMD_SHIFT) #define PMD_MASK (~(PMD_SIZE-1)) -/* PGDIR_SHIFT determines what a third-level page table entry can map */ -#define PGDIR_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE) +/* PUD_SHIFT determines what a third-level page table entry can map */ +#define PUD_SHIFT (PMD_SHIFT + PMD_INDEX_SIZE) +#define PUD_SIZE (1UL << PUD_SHIFT) +#define PUD_MASK (~(PUD_SIZE-1)) + +/* PGDIR_SHIFT determines what a fourth-level page table entry can map */ +#define PGDIR_SHIFT (PUD_SHIFT + PUD_INDEX_SIZE) #define PGDIR_SIZE (1UL << PGDIR_SHIFT) #define PGDIR_MASK (~(PGDIR_SIZE-1)) @@ -56,13 +60,12 @@ #define PTE_FRAG_SIZE_SHIFT 12 #define PTE_FRAG_SIZE (1UL << PTE_FRAG_SIZE_SHIFT) -/* - * Bits to mask out from a PMD to get to the PTE page - * PMDs point to PTE table fragments which are PTE_FRAG_SIZE aligned. - */ -#define PMD_MASKED_BITS (PTE_FRAG_SIZE - 1) -/* Bits to mask out from a PGD/PUD to get to the PMD page */ -#define PUD_MASKED_BITS 0x1ff +/* Bits to mask out from a PMD to get to the PTE page */ +#define PMD_MASKED_BITS 0xc0000000000000ffUL +/* Bits to mask out from a PUD to get to the PMD page */ +#define PUD_MASKED_BITS 0xc0000000000000ffUL +/* Bits to mask out from a PGD to get to the PUD page */ +#define PGD_MASKED_BITS 0xc0000000000000ffUL #ifndef __ASSEMBLY__ @@ -132,11 +135,9 @@ extern bool __rpte_sub_valid(real_pte_t rpte, unsigned long index); #else #define PMD_TABLE_SIZE (sizeof(pmd_t) << PMD_INDEX_SIZE) #endif +#define PUD_TABLE_SIZE (sizeof(pud_t) << PUD_INDEX_SIZE) #define PGD_TABLE_SIZE (sizeof(pgd_t) << PGD_INDEX_SIZE) -#define pgd_pte(pgd) (pud_pte(((pud_t){ pgd }))) -#define pte_pgd(pte) ((pgd_t)pte_pud(pte)) - #ifdef CONFIG_HUGETLB_PAGE /* * We have PGD_INDEX_SIZ = 12 and PTE_INDEX_SIZE = 8, so that we can have diff --git a/arch/powerpc/include/asm/book3s/64/hash.h b/arch/powerpc/include/asm/book3s/64/hash.h index ef9bd68..d0ee6fc 100644 --- a/arch/powerpc/include/asm/book3s/64/hash.h +++ b/arch/powerpc/include/asm/book3s/64/hash.h @@ -235,6 +235,7 @@ #define __pgtable_ptr_val(ptr) __pa(ptr) #define pgd_index(address) (((address) >> (PGDIR_SHIFT)) & (PTRS_PER_PGD - 1)) +#define pud_index(address) (((address) >> (PUD_SHIFT)) & (PTRS_PER_PUD - 1)) #define pmd_index(address) (((address) >> (PMD_SHIFT)) & (PTRS_PER_PMD - 1)) #define pte_index(address) (((address) >> (PAGE_SHIFT)) & (PTRS_PER_PTE - 1)) @@ -363,8 +364,18 @@ static inline void __ptep_set_access_flags(pte_t *ptep, pte_t entry) :"cc"); } +static inline int pgd_bad(pgd_t pgd) +{ + return (pgd_val(pgd) == 0); +} + #define __HAVE_ARCH_PTE_SAME #define pte_same(A,B) (((pte_val(A) ^ pte_val(B)) & ~_PAGE_HPTEFLAGS) == 0) +static inline unsigned long pgd_page_vaddr(pgd_t pgd) +{ + return (unsigned long)__va(pgd_val(pgd) & ~PGD_MASKED_BITS); +} + /* Generic accessors to PTE bits */ static inline int pte_write(pte_t pte) { return !!(pte_val(pte) & _PAGE_RW);} diff --git a/arch/powerpc/include/asm/book3s/64/pgtable.h b/arch/powerpc/include/asm/book3s/64/pgtable.h index 7482f69..77d3ce0 100644 --- a/arch/powerpc/include/asm/book3s/64/pgtable.h +++ b/arch/powerpc/include/asm/book3s/64/pgtable.h @@ -106,6 +106,26 @@ static inline void pgd_set(pgd_t *pgdp, unsigned long val) *pgdp = __pgd(val); } +static inline void pgd_clear(pgd_t *pgdp) +{ + *pgdp = __pgd(0); +} + +#define pgd_none(pgd) (!pgd_val(pgd)) +#define pgd_present(pgd) (!pgd_none(pgd)) + +static inline pte_t pgd_pte(pgd_t pgd) +{ + return __pte(pgd_val(pgd)); +} + +static inline pgd_t pte_pgd(pte_t pte) +{ + return __pgd(pte_val(pte)); +} + +extern struct page *pgd_page(pgd_t pgd); + /* * Find an entry in a page-table-directory. We combine the address region * (the high order N bits) and the pgd portion of the address. @@ -113,9 +133,10 @@ static inline void pgd_set(pgd_t *pgdp, unsigned long val) #define pgd_offset(mm, address) ((mm)->pgd + pgd_index(address)) +#define pud_offset(pgdp, addr) \ + (((pud_t *) pgd_page_vaddr(*(pgdp))) + pud_index(addr)) #define pmd_offset(pudp,addr) \ (((pmd_t *) pud_page_vaddr(*(pudp))) + pmd_index(addr)) - #define pte_offset_kernel(dir,addr) \ (((pte_t *) pmd_page_vaddr(*(dir))) + pte_index(addr)) @@ -130,6 +151,8 @@ static inline void pgd_set(pgd_t *pgdp, unsigned long val) pr_err("%s:%d: bad pte %08lx.\n", __FILE__, __LINE__, pte_val(e)) #define pmd_ERROR(e) \ pr_err("%s:%d: bad pmd %08lx.\n", __FILE__, __LINE__, pmd_val(e)) +#define pud_ERROR(e) \ + pr_err("%s:%d: bad pud %08lx.\n", __FILE__, __LINE__, pud_val(e)) #define pgd_ERROR(e) \ pr_err("%s:%d: bad pgd %08lx.\n", __FILE__, __LINE__, pgd_val(e)) -- cgit v1.1 From f64e8084c94bb0449177364856d8117e2f14c4c0 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Tue, 1 Mar 2016 12:59:20 +0530 Subject: powerpc/mm: Move hash related mmu-*.h headers to book3s/ No code changes. Signed-off-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/32/mmu-hash.h | 93 ++++ arch/powerpc/include/asm/book3s/64/mmu-hash.h | 616 ++++++++++++++++++++++++++ 2 files changed, 709 insertions(+) create mode 100644 arch/powerpc/include/asm/book3s/32/mmu-hash.h create mode 100644 arch/powerpc/include/asm/book3s/64/mmu-hash.h (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/32/mmu-hash.h b/arch/powerpc/include/asm/book3s/32/mmu-hash.h new file mode 100644 index 0000000..16f513e --- /dev/null +++ b/arch/powerpc/include/asm/book3s/32/mmu-hash.h @@ -0,0 +1,93 @@ +#ifndef _ASM_POWERPC_MMU_HASH32_H_ +#define _ASM_POWERPC_MMU_HASH32_H_ +/* + * 32-bit hash table MMU support + */ + +/* + * BATs + */ + +/* Block size masks */ +#define BL_128K 0x000 +#define BL_256K 0x001 +#define BL_512K 0x003 +#define BL_1M 0x007 +#define BL_2M 0x00F +#define BL_4M 0x01F +#define BL_8M 0x03F +#define BL_16M 0x07F +#define BL_32M 0x0FF +#define BL_64M 0x1FF +#define BL_128M 0x3FF +#define BL_256M 0x7FF + +/* BAT Access Protection */ +#define BPP_XX 0x00 /* No access */ +#define BPP_RX 0x01 /* Read only */ +#define BPP_RW 0x02 /* Read/write */ + +#ifndef __ASSEMBLY__ +/* Contort a phys_addr_t into the right format/bits for a BAT */ +#ifdef CONFIG_PHYS_64BIT +#define BAT_PHYS_ADDR(x) ((u32)((x & 0x00000000fffe0000ULL) | \ + ((x & 0x0000000e00000000ULL) >> 24) | \ + ((x & 0x0000000100000000ULL) >> 30))) +#else +#define BAT_PHYS_ADDR(x) (x) +#endif + +struct ppc_bat { + u32 batu; + u32 batl; +}; +#endif /* !__ASSEMBLY__ */ + +/* + * Hash table + */ + +/* Values for PP (assumes Ks=0, Kp=1) */ +#define PP_RWXX 0 /* Supervisor read/write, User none */ +#define PP_RWRX 1 /* Supervisor read/write, User read */ +#define PP_RWRW 2 /* Supervisor read/write, User read/write */ +#define PP_RXRX 3 /* Supervisor read, User read */ + +#ifndef __ASSEMBLY__ + +/* + * Hardware Page Table Entry + * Note that the xpn and x bitfields are used only by processors that + * support extended addressing; otherwise, those bits are reserved. + */ +struct hash_pte { + unsigned long v:1; /* Entry is valid */ + unsigned long vsid:24; /* Virtual segment identifier */ + unsigned long h:1; /* Hash algorithm indicator */ + unsigned long api:6; /* Abbreviated page index */ + unsigned long rpn:20; /* Real (physical) page number */ + unsigned long xpn:3; /* Real page number bits 0-2, optional */ + unsigned long r:1; /* Referenced */ + unsigned long c:1; /* Changed */ + unsigned long w:1; /* Write-thru cache mode */ + unsigned long i:1; /* Cache inhibited */ + unsigned long m:1; /* Memory coherence */ + unsigned long g:1; /* Guarded */ + unsigned long x:1; /* Real page number bit 3, optional */ + unsigned long pp:2; /* Page protection */ +}; + +typedef struct { + unsigned long id; + unsigned long vdso_base; +} mm_context_t; + +#endif /* !__ASSEMBLY__ */ + +/* We happily ignore the smaller BATs on 601, we don't actually use + * those definitions on hash32 at the moment anyway + */ +#define mmu_virtual_psize MMU_PAGE_4K +#define mmu_linear_psize MMU_PAGE_256M + +#endif /* _ASM_POWERPC_MMU_HASH32_H_ */ diff --git a/arch/powerpc/include/asm/book3s/64/mmu-hash.h b/arch/powerpc/include/asm/book3s/64/mmu-hash.h new file mode 100644 index 0000000..0cea480 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/mmu-hash.h @@ -0,0 +1,616 @@ +#ifndef _ASM_POWERPC_MMU_HASH64_H_ +#define _ASM_POWERPC_MMU_HASH64_H_ +/* + * PowerPC64 memory management structures + * + * Dave Engebretsen & Mike Corrigan <{engebret|mikejc}@us.ibm.com> + * PPC64 rework. + * + * This program is free software; you can redistribute it and/or + * modify it under the terms of the GNU General Public License + * as published by the Free Software Foundation; either version + * 2 of the License, or (at your option) any later version. + */ + +#include +#include +#include + +/* + * This is necessary to get the definition of PGTABLE_RANGE which we + * need for various slices related matters. Note that this isn't the + * complete pgtable.h but only a portion of it. + */ +#include +#include +#include + +/* + * SLB + */ + +#define SLB_NUM_BOLTED 3 +#define SLB_CACHE_ENTRIES 8 +#define SLB_MIN_SIZE 32 + +/* Bits in the SLB ESID word */ +#define SLB_ESID_V ASM_CONST(0x0000000008000000) /* valid */ + +/* Bits in the SLB VSID word */ +#define SLB_VSID_SHIFT 12 +#define SLB_VSID_SHIFT_1T 24 +#define SLB_VSID_SSIZE_SHIFT 62 +#define SLB_VSID_B ASM_CONST(0xc000000000000000) +#define SLB_VSID_B_256M ASM_CONST(0x0000000000000000) +#define SLB_VSID_B_1T ASM_CONST(0x4000000000000000) +#define SLB_VSID_KS ASM_CONST(0x0000000000000800) +#define SLB_VSID_KP ASM_CONST(0x0000000000000400) +#define SLB_VSID_N ASM_CONST(0x0000000000000200) /* no-execute */ +#define SLB_VSID_L ASM_CONST(0x0000000000000100) +#define SLB_VSID_C ASM_CONST(0x0000000000000080) /* class */ +#define SLB_VSID_LP ASM_CONST(0x0000000000000030) +#define SLB_VSID_LP_00 ASM_CONST(0x0000000000000000) +#define SLB_VSID_LP_01 ASM_CONST(0x0000000000000010) +#define SLB_VSID_LP_10 ASM_CONST(0x0000000000000020) +#define SLB_VSID_LP_11 ASM_CONST(0x0000000000000030) +#define SLB_VSID_LLP (SLB_VSID_L|SLB_VSID_LP) + +#define SLB_VSID_KERNEL (SLB_VSID_KP) +#define SLB_VSID_USER (SLB_VSID_KP|SLB_VSID_KS|SLB_VSID_C) + +#define SLBIE_C (0x08000000) +#define SLBIE_SSIZE_SHIFT 25 + +/* + * Hash table + */ + +#define HPTES_PER_GROUP 8 + +#define HPTE_V_SSIZE_SHIFT 62 +#define HPTE_V_AVPN_SHIFT 7 +#define HPTE_V_AVPN ASM_CONST(0x3fffffffffffff80) +#define HPTE_V_AVPN_VAL(x) (((x) & HPTE_V_AVPN) >> HPTE_V_AVPN_SHIFT) +#define HPTE_V_COMPARE(x,y) (!(((x) ^ (y)) & 0xffffffffffffff80UL)) +#define HPTE_V_BOLTED ASM_CONST(0x0000000000000010) +#define HPTE_V_LOCK ASM_CONST(0x0000000000000008) +#define HPTE_V_LARGE ASM_CONST(0x0000000000000004) +#define HPTE_V_SECONDARY ASM_CONST(0x0000000000000002) +#define HPTE_V_VALID ASM_CONST(0x0000000000000001) + +#define HPTE_R_PP0 ASM_CONST(0x8000000000000000) +#define HPTE_R_TS ASM_CONST(0x4000000000000000) +#define HPTE_R_KEY_HI ASM_CONST(0x3000000000000000) +#define HPTE_R_RPN_SHIFT 12 +#define HPTE_R_RPN ASM_CONST(0x0ffffffffffff000) +#define HPTE_R_PP ASM_CONST(0x0000000000000003) +#define HPTE_R_N ASM_CONST(0x0000000000000004) +#define HPTE_R_G ASM_CONST(0x0000000000000008) +#define HPTE_R_M ASM_CONST(0x0000000000000010) +#define HPTE_R_I ASM_CONST(0x0000000000000020) +#define HPTE_R_W ASM_CONST(0x0000000000000040) +#define HPTE_R_WIMG ASM_CONST(0x0000000000000078) +#define HPTE_R_C ASM_CONST(0x0000000000000080) +#define HPTE_R_R ASM_CONST(0x0000000000000100) +#define HPTE_R_KEY_LO ASM_CONST(0x0000000000000e00) + +#define HPTE_V_1TB_SEG ASM_CONST(0x4000000000000000) +#define HPTE_V_VRMA_MASK ASM_CONST(0x4001ffffff000000) + +/* Values for PP (assumes Ks=0, Kp=1) */ +#define PP_RWXX 0 /* Supervisor read/write, User none */ +#define PP_RWRX 1 /* Supervisor read/write, User read */ +#define PP_RWRW 2 /* Supervisor read/write, User read/write */ +#define PP_RXRX 3 /* Supervisor read, User read */ +#define PP_RXXX (HPTE_R_PP0 | 2) /* Supervisor read, user none */ + +/* Fields for tlbiel instruction in architecture 2.06 */ +#define TLBIEL_INVAL_SEL_MASK 0xc00 /* invalidation selector */ +#define TLBIEL_INVAL_PAGE 0x000 /* invalidate a single page */ +#define TLBIEL_INVAL_SET_LPID 0x800 /* invalidate a set for current LPID */ +#define TLBIEL_INVAL_SET 0xc00 /* invalidate a set for all LPIDs */ +#define TLBIEL_INVAL_SET_MASK 0xfff000 /* set number to inval. */ +#define TLBIEL_INVAL_SET_SHIFT 12 + +#define POWER7_TLB_SETS 128 /* # sets in POWER7 TLB */ +#define POWER8_TLB_SETS 512 /* # sets in POWER8 TLB */ +#define POWER9_TLB_SETS_HASH 256 /* # sets in POWER9 TLB Hash mode */ + +#ifndef __ASSEMBLY__ + +struct hash_pte { + __be64 v; + __be64 r; +}; + +extern struct hash_pte *htab_address; +extern unsigned long htab_size_bytes; +extern unsigned long htab_hash_mask; + +/* + * Page size definition + * + * shift : is the "PAGE_SHIFT" value for that page size + * sllp : is a bit mask with the value of SLB L || LP to be or'ed + * directly to a slbmte "vsid" value + * penc : is the HPTE encoding mask for the "LP" field: + * + */ +struct mmu_psize_def +{ + unsigned int shift; /* number of bits */ + int penc[MMU_PAGE_COUNT]; /* HPTE encoding */ + unsigned int tlbiel; /* tlbiel supported for that page size */ + unsigned long avpnm; /* bits to mask out in AVPN in the HPTE */ + unsigned long sllp; /* SLB L||LP (exact mask to use in slbmte) */ +}; +extern struct mmu_psize_def mmu_psize_defs[MMU_PAGE_COUNT]; + +static inline int shift_to_mmu_psize(unsigned int shift) +{ + int psize; + + for (psize = 0; psize < MMU_PAGE_COUNT; ++psize) + if (mmu_psize_defs[psize].shift == shift) + return psize; + return -1; +} + +static inline unsigned int mmu_psize_to_shift(unsigned int mmu_psize) +{ + if (mmu_psize_defs[mmu_psize].shift) + return mmu_psize_defs[mmu_psize].shift; + BUG(); +} + +#endif /* __ASSEMBLY__ */ + +/* + * Segment sizes. + * These are the values used by hardware in the B field of + * SLB entries and the first dword of MMU hashtable entries. + * The B field is 2 bits; the values 2 and 3 are unused and reserved. + */ +#define MMU_SEGSIZE_256M 0 +#define MMU_SEGSIZE_1T 1 + +/* + * encode page number shift. + * in order to fit the 78 bit va in a 64 bit variable we shift the va by + * 12 bits. This enable us to address upto 76 bit va. + * For hpt hash from a va we can ignore the page size bits of va and for + * hpte encoding we ignore up to 23 bits of va. So ignoring lower 12 bits ensure + * we work in all cases including 4k page size. + */ +#define VPN_SHIFT 12 + +/* + * HPTE Large Page (LP) details + */ +#define LP_SHIFT 12 +#define LP_BITS 8 +#define LP_MASK(i) ((0xFF >> (i)) << LP_SHIFT) + +#ifndef __ASSEMBLY__ + +static inline int slb_vsid_shift(int ssize) +{ + if (ssize == MMU_SEGSIZE_256M) + return SLB_VSID_SHIFT; + return SLB_VSID_SHIFT_1T; +} + +static inline int segment_shift(int ssize) +{ + if (ssize == MMU_SEGSIZE_256M) + return SID_SHIFT; + return SID_SHIFT_1T; +} + +/* + * The current system page and segment sizes + */ +extern int mmu_linear_psize; +extern int mmu_virtual_psize; +extern int mmu_vmalloc_psize; +extern int mmu_vmemmap_psize; +extern int mmu_io_psize; +extern int mmu_kernel_ssize; +extern int mmu_highuser_ssize; +extern u16 mmu_slb_size; +extern unsigned long tce_alloc_start, tce_alloc_end; + +/* + * If the processor supports 64k normal pages but not 64k cache + * inhibited pages, we have to be prepared to switch processes + * to use 4k pages when they create cache-inhibited mappings. + * If this is the case, mmu_ci_restrictions will be set to 1. + */ +extern int mmu_ci_restrictions; + +/* + * This computes the AVPN and B fields of the first dword of a HPTE, + * for use when we want to match an existing PTE. The bottom 7 bits + * of the returned value are zero. + */ +static inline unsigned long hpte_encode_avpn(unsigned long vpn, int psize, + int ssize) +{ + unsigned long v; + /* + * The AVA field omits the low-order 23 bits of the 78 bits VA. + * These bits are not needed in the PTE, because the + * low-order b of these bits are part of the byte offset + * into the virtual page and, if b < 23, the high-order + * 23-b of these bits are always used in selecting the + * PTEGs to be searched + */ + v = (vpn >> (23 - VPN_SHIFT)) & ~(mmu_psize_defs[psize].avpnm); + v <<= HPTE_V_AVPN_SHIFT; + v |= ((unsigned long) ssize) << HPTE_V_SSIZE_SHIFT; + return v; +} + +/* + * This function sets the AVPN and L fields of the HPTE appropriately + * using the base page size and actual page size. + */ +static inline unsigned long hpte_encode_v(unsigned long vpn, int base_psize, + int actual_psize, int ssize) +{ + unsigned long v; + v = hpte_encode_avpn(vpn, base_psize, ssize); + if (actual_psize != MMU_PAGE_4K) + v |= HPTE_V_LARGE; + return v; +} + +/* + * This function sets the ARPN, and LP fields of the HPTE appropriately + * for the page size. We assume the pa is already "clean" that is properly + * aligned for the requested page size + */ +static inline unsigned long hpte_encode_r(unsigned long pa, int base_psize, + int actual_psize) +{ + /* A 4K page needs no special encoding */ + if (actual_psize == MMU_PAGE_4K) + return pa & HPTE_R_RPN; + else { + unsigned int penc = mmu_psize_defs[base_psize].penc[actual_psize]; + unsigned int shift = mmu_psize_defs[actual_psize].shift; + return (pa & ~((1ul << shift) - 1)) | (penc << LP_SHIFT); + } +} + +/* + * Build a VPN_SHIFT bit shifted va given VSID, EA and segment size. + */ +static inline unsigned long hpt_vpn(unsigned long ea, + unsigned long vsid, int ssize) +{ + unsigned long mask; + int s_shift = segment_shift(ssize); + + mask = (1ul << (s_shift - VPN_SHIFT)) - 1; + return (vsid << (s_shift - VPN_SHIFT)) | ((ea >> VPN_SHIFT) & mask); +} + +/* + * This hashes a virtual address + */ +static inline unsigned long hpt_hash(unsigned long vpn, + unsigned int shift, int ssize) +{ + int mask; + unsigned long hash, vsid; + + /* VPN_SHIFT can be atmost 12 */ + if (ssize == MMU_SEGSIZE_256M) { + mask = (1ul << (SID_SHIFT - VPN_SHIFT)) - 1; + hash = (vpn >> (SID_SHIFT - VPN_SHIFT)) ^ + ((vpn & mask) >> (shift - VPN_SHIFT)); + } else { + mask = (1ul << (SID_SHIFT_1T - VPN_SHIFT)) - 1; + vsid = vpn >> (SID_SHIFT_1T - VPN_SHIFT); + hash = vsid ^ (vsid << 25) ^ + ((vpn & mask) >> (shift - VPN_SHIFT)) ; + } + return hash & 0x7fffffffffUL; +} + +#define HPTE_LOCAL_UPDATE 0x1 +#define HPTE_NOHPTE_UPDATE 0x2 + +extern int __hash_page_4K(unsigned long ea, unsigned long access, + unsigned long vsid, pte_t *ptep, unsigned long trap, + unsigned long flags, int ssize, int subpage_prot); +extern int __hash_page_64K(unsigned long ea, unsigned long access, + unsigned long vsid, pte_t *ptep, unsigned long trap, + unsigned long flags, int ssize); +struct mm_struct; +unsigned int hash_page_do_lazy_icache(unsigned int pp, pte_t pte, int trap); +extern int hash_page_mm(struct mm_struct *mm, unsigned long ea, + unsigned long access, unsigned long trap, + unsigned long flags); +extern int hash_page(unsigned long ea, unsigned long access, unsigned long trap, + unsigned long dsisr); +int __hash_page_huge(unsigned long ea, unsigned long access, unsigned long vsid, + pte_t *ptep, unsigned long trap, unsigned long flags, + int ssize, unsigned int shift, unsigned int mmu_psize); +#ifdef CONFIG_TRANSPARENT_HUGEPAGE +extern int __hash_page_thp(unsigned long ea, unsigned long access, + unsigned long vsid, pmd_t *pmdp, unsigned long trap, + unsigned long flags, int ssize, unsigned int psize); +#else +static inline int __hash_page_thp(unsigned long ea, unsigned long access, + unsigned long vsid, pmd_t *pmdp, + unsigned long trap, unsigned long flags, + int ssize, unsigned int psize) +{ + BUG(); + return -1; +} +#endif +extern void hash_failure_debug(unsigned long ea, unsigned long access, + unsigned long vsid, unsigned long trap, + int ssize, int psize, int lpsize, + unsigned long pte); +extern int htab_bolt_mapping(unsigned long vstart, unsigned long vend, + unsigned long pstart, unsigned long prot, + int psize, int ssize); +int htab_remove_mapping(unsigned long vstart, unsigned long vend, + int psize, int ssize); +extern void add_gpage(u64 addr, u64 page_size, unsigned long number_of_pages); +extern void demote_segment_4k(struct mm_struct *mm, unsigned long addr); + +extern void hpte_init_native(void); +extern void hpte_init_lpar(void); +extern void hpte_init_beat(void); +extern void hpte_init_beat_v3(void); + +extern void slb_initialize(void); +extern void slb_flush_and_rebolt(void); + +extern void slb_vmalloc_update(void); +extern void slb_set_size(u16 size); +#endif /* __ASSEMBLY__ */ + +/* + * VSID allocation (256MB segment) + * + * We first generate a 37-bit "proto-VSID". Proto-VSIDs are generated + * from mmu context id and effective segment id of the address. + * + * For user processes max context id is limited to ((1ul << 19) - 5) + * for kernel space, we use the top 4 context ids to map address as below + * NOTE: each context only support 64TB now. + * 0x7fffc - [ 0xc000000000000000 - 0xc0003fffffffffff ] + * 0x7fffd - [ 0xd000000000000000 - 0xd0003fffffffffff ] + * 0x7fffe - [ 0xe000000000000000 - 0xe0003fffffffffff ] + * 0x7ffff - [ 0xf000000000000000 - 0xf0003fffffffffff ] + * + * The proto-VSIDs are then scrambled into real VSIDs with the + * multiplicative hash: + * + * VSID = (proto-VSID * VSID_MULTIPLIER) % VSID_MODULUS + * + * VSID_MULTIPLIER is prime, so in particular it is + * co-prime to VSID_MODULUS, making this a 1:1 scrambling function. + * Because the modulus is 2^n-1 we can compute it efficiently without + * a divide or extra multiply (see below). The scramble function gives + * robust scattering in the hash table (at least based on some initial + * results). + * + * We also consider VSID 0 special. We use VSID 0 for slb entries mapping + * bad address. This enables us to consolidate bad address handling in + * hash_page. + * + * We also need to avoid the last segment of the last context, because that + * would give a protovsid of 0x1fffffffff. That will result in a VSID 0 + * because of the modulo operation in vsid scramble. But the vmemmap + * (which is what uses region 0xf) will never be close to 64TB in size + * (it's 56 bytes per page of system memory). + */ + +#define CONTEXT_BITS 19 +#define ESID_BITS 18 +#define ESID_BITS_1T 6 + +/* + * 256MB segment + * The proto-VSID space has 2^(CONTEX_BITS + ESID_BITS) - 1 segments + * available for user + kernel mapping. The top 4 contexts are used for + * kernel mapping. Each segment contains 2^28 bytes. Each + * context maps 2^46 bytes (64TB) so we can support 2^19-1 contexts + * (19 == 37 + 28 - 46). + */ +#define MAX_USER_CONTEXT ((ASM_CONST(1) << CONTEXT_BITS) - 5) + +/* + * This should be computed such that protovosid * vsid_mulitplier + * doesn't overflow 64 bits. It should also be co-prime to vsid_modulus + */ +#define VSID_MULTIPLIER_256M ASM_CONST(12538073) /* 24-bit prime */ +#define VSID_BITS_256M (CONTEXT_BITS + ESID_BITS) +#define VSID_MODULUS_256M ((1UL<= \ + * 2^36-1, then r3+1 has the 2^36 bit set. So, if r3+1 has \ + * the bit clear, r3 already has the answer we want, if it \ + * doesn't, the answer is the low 36 bits of r3+1. So in all \ + * cases the answer is the low 36 bits of (r3 + ((r3+1) >> 36))*/\ + addi rx,rt,1; \ + srdi rx,rx,VSID_BITS_##size; /* extract 2^VSID_BITS bit */ \ + add rt,rt,rx + +/* 4 bits per slice and we have one slice per 1TB */ +#define SLICE_ARRAY_SIZE (PGTABLE_RANGE >> 41) + +#ifndef __ASSEMBLY__ + +#ifdef CONFIG_PPC_SUBPAGE_PROT +/* + * For the sub-page protection option, we extend the PGD with one of + * these. Basically we have a 3-level tree, with the top level being + * the protptrs array. To optimize speed and memory consumption when + * only addresses < 4GB are being protected, pointers to the first + * four pages of sub-page protection words are stored in the low_prot + * array. + * Each page of sub-page protection words protects 1GB (4 bytes + * protects 64k). For the 3-level tree, each page of pointers then + * protects 8TB. + */ +struct subpage_prot_table { + unsigned long maxaddr; /* only addresses < this are protected */ + unsigned int **protptrs[(TASK_SIZE_USER64 >> 43)]; + unsigned int *low_prot[4]; +}; + +#define SBP_L1_BITS (PAGE_SHIFT - 2) +#define SBP_L2_BITS (PAGE_SHIFT - 3) +#define SBP_L1_COUNT (1 << SBP_L1_BITS) +#define SBP_L2_COUNT (1 << SBP_L2_BITS) +#define SBP_L2_SHIFT (PAGE_SHIFT + SBP_L1_BITS) +#define SBP_L3_SHIFT (SBP_L2_SHIFT + SBP_L2_BITS) + +extern void subpage_prot_free(struct mm_struct *mm); +extern void subpage_prot_init_new_context(struct mm_struct *mm); +#else +static inline void subpage_prot_free(struct mm_struct *mm) {} +static inline void subpage_prot_init_new_context(struct mm_struct *mm) { } +#endif /* CONFIG_PPC_SUBPAGE_PROT */ + +typedef unsigned long mm_context_id_t; +struct spinlock; + +typedef struct { + mm_context_id_t id; + u16 user_psize; /* page size index */ + +#ifdef CONFIG_PPC_MM_SLICES + u64 low_slices_psize; /* SLB page size encodings */ + unsigned char high_slices_psize[SLICE_ARRAY_SIZE]; +#else + u16 sllp; /* SLB page size encoding */ +#endif + unsigned long vdso_base; +#ifdef CONFIG_PPC_SUBPAGE_PROT + struct subpage_prot_table spt; +#endif /* CONFIG_PPC_SUBPAGE_PROT */ +#ifdef CONFIG_PPC_ICSWX + struct spinlock *cop_lockp; /* guard acop and cop_pid */ + unsigned long acop; /* mask of enabled coprocessor types */ + unsigned int cop_pid; /* pid value used with coprocessors */ +#endif /* CONFIG_PPC_ICSWX */ +#ifdef CONFIG_PPC_64K_PAGES + /* for 4K PTE fragment support */ + void *pte_frag; +#endif +#ifdef CONFIG_SPAPR_TCE_IOMMU + struct list_head iommu_group_mem_list; +#endif +} mm_context_t; + + +#if 0 +/* + * The code below is equivalent to this function for arguments + * < 2^VSID_BITS, which is all this should ever be called + * with. However gcc is not clever enough to compute the + * modulus (2^n-1) without a second multiply. + */ +#define vsid_scramble(protovsid, size) \ + ((((protovsid) * VSID_MULTIPLIER_##size) % VSID_MODULUS_##size)) + +#else /* 1 */ +#define vsid_scramble(protovsid, size) \ + ({ \ + unsigned long x; \ + x = (protovsid) * VSID_MULTIPLIER_##size; \ + x = (x >> VSID_BITS_##size) + (x & VSID_MODULUS_##size); \ + (x + ((x+1) >> VSID_BITS_##size)) & VSID_MODULUS_##size; \ + }) +#endif /* 1 */ + +/* Returns the segment size indicator for a user address */ +static inline int user_segment_size(unsigned long addr) +{ + /* Use 1T segments if possible for addresses >= 1T */ + if (addr >= (1UL << SID_SHIFT_1T)) + return mmu_highuser_ssize; + return MMU_SEGSIZE_256M; +} + +static inline unsigned long get_vsid(unsigned long context, unsigned long ea, + int ssize) +{ + /* + * Bad address. We return VSID 0 for that + */ + if ((ea & ~REGION_MASK) >= PGTABLE_RANGE) + return 0; + + if (ssize == MMU_SEGSIZE_256M) + return vsid_scramble((context << ESID_BITS) + | (ea >> SID_SHIFT), 256M); + return vsid_scramble((context << ESID_BITS_1T) + | (ea >> SID_SHIFT_1T), 1T); +} + +/* + * This is only valid for addresses >= PAGE_OFFSET + * + * For kernel space, we use the top 4 context ids to map address as below + * 0x7fffc - [ 0xc000000000000000 - 0xc0003fffffffffff ] + * 0x7fffd - [ 0xd000000000000000 - 0xd0003fffffffffff ] + * 0x7fffe - [ 0xe000000000000000 - 0xe0003fffffffffff ] + * 0x7ffff - [ 0xf000000000000000 - 0xf0003fffffffffff ] + */ +static inline unsigned long get_kernel_vsid(unsigned long ea, int ssize) +{ + unsigned long context; + + /* + * kernel take the top 4 context from the available range + */ + context = (MAX_USER_CONTEXT) + ((ea >> 60) - 0xc) + 1; + return get_vsid(context, ea, ssize); +} + +unsigned htab_shift_for_mem_size(unsigned long mem_size); + +#endif /* __ASSEMBLY__ */ + +#endif /* _ASM_POWERPC_MMU_HASH64_H_ */ -- cgit v1.1 From ee3b93ebfbed6279f7a329001433c75c50ddfcc9 Mon Sep 17 00:00:00 2001 From: "Aneesh Kumar K.V" Date: Tue, 1 Mar 2016 12:59:21 +0530 Subject: powerpc/mm: Move hash64 tlbflush code into a new header No code changes. Signed-off-by: Aneesh Kumar K.V Signed-off-by: Michael Ellerman --- arch/powerpc/include/asm/book3s/64/tlbflush-hash.h | 94 ++++++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 arch/powerpc/include/asm/book3s/64/tlbflush-hash.h (limited to 'arch/powerpc/include/asm/book3s') diff --git a/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h new file mode 100644 index 0000000..1b753f9 --- /dev/null +++ b/arch/powerpc/include/asm/book3s/64/tlbflush-hash.h @@ -0,0 +1,94 @@ +#ifndef _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H +#define _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H + +#define MMU_NO_CONTEXT 0 + +/* + * TLB flushing for 64-bit hash-MMU CPUs + */ + +#include +#include + +#define PPC64_TLB_BATCH_NR 192 + +struct ppc64_tlb_batch { + int active; + unsigned long index; + struct mm_struct *mm; + real_pte_t pte[PPC64_TLB_BATCH_NR]; + unsigned long vpn[PPC64_TLB_BATCH_NR]; + unsigned int psize; + int ssize; +}; +DECLARE_PER_CPU(struct ppc64_tlb_batch, ppc64_tlb_batch); + +extern void __flush_tlb_pending(struct ppc64_tlb_batch *batch); + +#define __HAVE_ARCH_ENTER_LAZY_MMU_MODE + +static inline void arch_enter_lazy_mmu_mode(void) +{ + struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch); + + batch->active = 1; +} + +static inline void arch_leave_lazy_mmu_mode(void) +{ + struct ppc64_tlb_batch *batch = this_cpu_ptr(&ppc64_tlb_batch); + + if (batch->index) + __flush_tlb_pending(batch); + batch->active = 0; +} + +#define arch_flush_lazy_mmu_mode() do {} while (0) + + +extern void flush_hash_page(unsigned long vpn, real_pte_t pte, int psize, + int ssize, unsigned long flags); +extern void flush_hash_range(unsigned long number, int local); +extern void flush_hash_hugepage(unsigned long vsid, unsigned long addr, + pmd_t *pmdp, unsigned int psize, int ssize, + unsigned long flags); + +static inline void local_flush_tlb_mm(struct mm_struct *mm) +{ +} + +static inline void flush_tlb_mm(struct mm_struct *mm) +{ +} + +static inline void local_flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ +} + +static inline void flush_tlb_page(struct vm_area_struct *vma, + unsigned long vmaddr) +{ +} + +static inline void flush_tlb_page_nohash(struct vm_area_struct *vma, + unsigned long vmaddr) +{ +} + +static inline void flush_tlb_range(struct vm_area_struct *vma, + unsigned long start, unsigned long end) +{ +} + +static inline void flush_tlb_kernel_range(unsigned long start, + unsigned long end) +{ +} + +/* Private function for use by PCI IO mapping code */ +extern void __flush_hash_table_range(struct mm_struct *mm, unsigned long start, + unsigned long end); +extern void flush_tlb_pmd_range(struct mm_struct *mm, pmd_t *pmd, + unsigned long addr); +#endif /* _ASM_POWERPC_BOOK3S_64_TLBFLUSH_HASH_H */ -- cgit v1.1