diff options
author | Heiko Carstens <heiko.carstens@de.ibm.com> | 2014-09-23 21:29:20 +0200 |
---|---|---|
committer | Martin Schwidefsky <schwidefsky@de.ibm.com> | 2014-09-30 10:19:29 +0200 |
commit | cfb0b24143b4f587ff3e3bd829f9f471285d097b (patch) | |
tree | 217a325aaba02fa8b6a0c91d8dd2abe7b96fcb66 | |
parent | 242a112af62ea73ce507cbe76c2c944c23b6a1e3 (diff) | |
download | op-kernel-dev-cfb0b24143b4f587ff3e3bd829f9f471285d097b.zip op-kernel-dev-cfb0b24143b4f587ff3e3bd829f9f471285d097b.tar.gz |
s390/mm: make use of ipte range facility
Invalidate several pte entries at once if the ipte range facility
is available. Currently this works only for DEBUG_PAGE_ALLOC where
several up to 2 ^ MAX_ORDER may be invalidated at once.
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>
-rw-r--r-- | arch/s390/include/asm/pgtable.h | 16 | ||||
-rw-r--r-- | arch/s390/mm/pageattr.c | 38 |
2 files changed, 47 insertions, 7 deletions
diff --git a/arch/s390/include/asm/pgtable.h b/arch/s390/include/asm/pgtable.h index 7c4af56..2de229c 100644 --- a/arch/s390/include/asm/pgtable.h +++ b/arch/s390/include/asm/pgtable.h @@ -1053,6 +1053,22 @@ static inline void __ptep_ipte_local(unsigned long address, pte_t *ptep) : "=m" (*ptep) : "m" (*ptep), "a" (pto), "a" (address)); } +static inline void __ptep_ipte_range(unsigned long address, int nr, pte_t *ptep) +{ + unsigned long pto = (unsigned long) ptep; + +#ifndef CONFIG_64BIT + /* pto in ESA mode must point to the start of the segment table */ + pto &= 0x7ffffc00; +#endif + /* Invalidate a range of ptes + global TLB flush of the ptes */ + do { + asm volatile( + " .insn rrf,0xb2210000,%2,%0,%1,0" + : "+a" (address), "+a" (nr) : "a" (pto) : "memory"); + } while (nr != 255); +} + static inline void ptep_flush_direct(struct mm_struct *mm, unsigned long address, pte_t *ptep) { diff --git a/arch/s390/mm/pageattr.c b/arch/s390/mm/pageattr.c index 8400f49..3fef3b2 100644 --- a/arch/s390/mm/pageattr.c +++ b/arch/s390/mm/pageattr.c @@ -6,6 +6,7 @@ #include <linux/module.h> #include <linux/mm.h> #include <asm/cacheflush.h> +#include <asm/facility.h> #include <asm/pgtable.h> #include <asm/page.h> @@ -103,27 +104,50 @@ int set_memory_x(unsigned long addr, int numpages) } #ifdef CONFIG_DEBUG_PAGEALLOC + +static void ipte_range(pte_t *pte, unsigned long address, int nr) +{ + int i; + + if (test_facility(13) && IS_ENABLED(CONFIG_64BIT)) { + __ptep_ipte_range(address, nr - 1, pte); + return; + } + for (i = 0; i < nr; i++) { + __ptep_ipte(address, pte); + address += PAGE_SIZE; + pte++; + } +} + void kernel_map_pages(struct page *page, int numpages, int enable) { unsigned long address; + int nr, i, j; pgd_t *pgd; pud_t *pud; pmd_t *pmd; pte_t *pte; - int i; - for (i = 0; i < numpages; i++) { + for (i = 0; i < numpages;) { address = page_to_phys(page + i); pgd = pgd_offset_k(address); pud = pud_offset(pgd, address); pmd = pmd_offset(pud, address); pte = pte_offset_kernel(pmd, address); - if (!enable) { - __ptep_ipte(address, pte); - pte_val(*pte) = _PAGE_INVALID; - continue; + nr = (unsigned long)pte >> ilog2(sizeof(long)); + nr = PTRS_PER_PTE - (nr & (PTRS_PER_PTE - 1)); + nr = min(numpages - i, nr); + if (enable) { + for (j = 0; j < nr; j++) { + pte_val(*pte) = __pa(address); + address += PAGE_SIZE; + pte++; + } + } else { + ipte_range(pte, address, nr); } - pte_val(*pte) = __pa(address); + i += nr; } } |