summaryrefslogtreecommitdiffstats
path: root/arch/sh/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/sh/mm')
-rw-r--r--arch/sh/mm/Kconfig222
-rw-r--r--arch/sh/mm/Makefile40
-rw-r--r--arch/sh/mm/Makefile_3236
-rw-r--r--arch/sh/mm/Makefile_6444
-rw-r--r--arch/sh/mm/cache-debugfs.c9
-rw-r--r--arch/sh/mm/cache-sh4.c14
-rw-r--r--arch/sh/mm/cache-sh5.c1029
-rw-r--r--arch/sh/mm/cache-sh7705.c12
-rw-r--r--arch/sh/mm/clear_page.S152
-rw-r--r--arch/sh/mm/consistent.c174
-rw-r--r--arch/sh/mm/copy_page.S388
-rw-r--r--arch/sh/mm/extable_32.c (renamed from arch/sh/mm/extable.c)0
-rw-r--r--arch/sh/mm/extable_64.c82
-rw-r--r--arch/sh/mm/fault_32.c (renamed from arch/sh/mm/fault.c)2
-rw-r--r--arch/sh/mm/fault_64.c275
-rw-r--r--arch/sh/mm/init.c68
-rw-r--r--arch/sh/mm/ioremap_32.c (renamed from arch/sh/mm/ioremap.c)0
-rw-r--r--arch/sh/mm/ioremap_64.c404
-rw-r--r--arch/sh/mm/pg-nommu.c4
-rw-r--r--arch/sh/mm/pmb.c26
-rw-r--r--arch/sh/mm/tlb-nommu.c10
-rw-r--r--arch/sh/mm/tlb-sh4.c7
-rw-r--r--arch/sh/mm/tlb-sh5.c164
-rw-r--r--arch/sh/mm/tlbflush_32.c (renamed from arch/sh/mm/tlb-flush.c)0
-rw-r--r--arch/sh/mm/tlbflush_64.c475
25 files changed, 2774 insertions, 863 deletions
diff --git a/arch/sh/mm/Kconfig b/arch/sh/mm/Kconfig
index 1265f20..f549b8c 100644
--- a/arch/sh/mm/Kconfig
+++ b/arch/sh/mm/Kconfig
@@ -1,193 +1,3 @@
-#
-# Processor families
-#
-config CPU_SH2
- bool
-
-config CPU_SH2A
- bool
- select CPU_SH2
-
-config CPU_SH3
- bool
- select CPU_HAS_INTEVT
- select CPU_HAS_SR_RB
-
-config CPU_SH4
- bool
- select CPU_HAS_INTEVT
- select CPU_HAS_SR_RB
- select CPU_HAS_PTEA if !CPU_SH4A || CPU_SHX2
- select CPU_HAS_FPU if !CPU_SH4AL_DSP
-
-config CPU_SH4A
- bool
- select CPU_SH4
-
-config CPU_SH4AL_DSP
- bool
- select CPU_SH4A
- select CPU_HAS_DSP
-
-config CPU_SHX2
- bool
-
-config CPU_SHX3
- bool
-
-choice
- prompt "Processor sub-type selection"
-
-#
-# Processor subtypes
-#
-
-# SH-2 Processor Support
-
-config CPU_SUBTYPE_SH7619
- bool "Support SH7619 processor"
- select CPU_SH2
-
-# SH-2A Processor Support
-
-config CPU_SUBTYPE_SH7206
- bool "Support SH7206 processor"
- select CPU_SH2A
-
-# SH-3 Processor Support
-
-config CPU_SUBTYPE_SH7705
- bool "Support SH7705 processor"
- select CPU_SH3
-
-config CPU_SUBTYPE_SH7706
- bool "Support SH7706 processor"
- select CPU_SH3
- help
- Select SH7706 if you have a 133 Mhz SH-3 HD6417706 CPU.
-
-config CPU_SUBTYPE_SH7707
- bool "Support SH7707 processor"
- select CPU_SH3
- help
- Select SH7707 if you have a 60 Mhz SH-3 HD6417707 CPU.
-
-config CPU_SUBTYPE_SH7708
- bool "Support SH7708 processor"
- select CPU_SH3
- help
- Select SH7708 if you have a 60 Mhz SH-3 HD6417708S or
- if you have a 100 Mhz SH-3 HD6417708R CPU.
-
-config CPU_SUBTYPE_SH7709
- bool "Support SH7709 processor"
- select CPU_SH3
- help
- Select SH7709 if you have a 80 Mhz SH-3 HD6417709 CPU.
-
-config CPU_SUBTYPE_SH7710
- bool "Support SH7710 processor"
- select CPU_SH3
- select CPU_HAS_DSP
- help
- Select SH7710 if you have a SH3-DSP SH7710 CPU.
-
-config CPU_SUBTYPE_SH7712
- bool "Support SH7712 processor"
- select CPU_SH3
- select CPU_HAS_DSP
- help
- Select SH7712 if you have a SH3-DSP SH7712 CPU.
-
-config CPU_SUBTYPE_SH7720
- bool "Support SH7720 processor"
- select CPU_SH3
- select CPU_HAS_DSP
- help
- Select SH7720 if you have a SH3-DSP SH7720 CPU.
-
-# SH-4 Processor Support
-
-config CPU_SUBTYPE_SH7750
- bool "Support SH7750 processor"
- select CPU_SH4
- help
- Select SH7750 if you have a 200 Mhz SH-4 HD6417750 CPU.
-
-config CPU_SUBTYPE_SH7091
- bool "Support SH7091 processor"
- select CPU_SH4
- help
- Select SH7091 if you have an SH-4 based Sega device (such as
- the Dreamcast, Naomi, and Naomi 2).
-
-config CPU_SUBTYPE_SH7750R
- bool "Support SH7750R processor"
- select CPU_SH4
-
-config CPU_SUBTYPE_SH7750S
- bool "Support SH7750S processor"
- select CPU_SH4
-
-config CPU_SUBTYPE_SH7751
- bool "Support SH7751 processor"
- select CPU_SH4
- help
- Select SH7751 if you have a 166 Mhz SH-4 HD6417751 CPU,
- or if you have a HD6417751R CPU.
-
-config CPU_SUBTYPE_SH7751R
- bool "Support SH7751R processor"
- select CPU_SH4
-
-config CPU_SUBTYPE_SH7760
- bool "Support SH7760 processor"
- select CPU_SH4
-
-config CPU_SUBTYPE_SH4_202
- bool "Support SH4-202 processor"
- select CPU_SH4
-
-# SH-4A Processor Support
-
-config CPU_SUBTYPE_SH7770
- bool "Support SH7770 processor"
- select CPU_SH4A
-
-config CPU_SUBTYPE_SH7780
- bool "Support SH7780 processor"
- select CPU_SH4A
-
-config CPU_SUBTYPE_SH7785
- bool "Support SH7785 processor"
- select CPU_SH4A
- select CPU_SHX2
- select ARCH_SPARSEMEM_ENABLE
- select SYS_SUPPORTS_NUMA
-
-config CPU_SUBTYPE_SHX3
- bool "Support SH-X3 processor"
- select CPU_SH4A
- select CPU_SHX3
- select ARCH_SPARSEMEM_ENABLE
- select SYS_SUPPORTS_NUMA
- select SYS_SUPPORTS_SMP
-
-# SH4AL-DSP Processor Support
-
-config CPU_SUBTYPE_SH7343
- bool "Support SH7343 processor"
- select CPU_SH4AL_DSP
-
-config CPU_SUBTYPE_SH7722
- bool "Support SH7722 processor"
- select CPU_SH4AL_DSP
- select CPU_SHX2
- select ARCH_SPARSEMEM_ENABLE
- select SYS_SUPPORTS_NUMA
-
-endchoice
-
menu "Memory management options"
config QUICKLIST
@@ -207,7 +17,8 @@ config MMU
config PAGE_OFFSET
hex
- default "0x80000000" if MMU
+ default "0x80000000" if MMU && SUPERH32
+ default "0x20000000" if MMU && SUPERH64
default "0x00000000"
config MEMORY_START
@@ -228,17 +39,28 @@ config MEMORY_START
config MEMORY_SIZE
hex "Physical memory size"
- default "0x00400000"
+ default "0x04000000"
help
This sets the default memory size assumed by your SH kernel. It can
be overridden as normal by the 'mem=' argument on the kernel command
line. If unsure, consult your board specifications or just leave it
- as 0x00400000 which was the default value before this became
+ as 0x04000000 which was the default value before this became
configurable.
+# Physical addressing modes
+
+config 29BIT
+ def_bool !32BIT
+ depends on SUPERH32
+
config 32BIT
+ bool
+ default y if CPU_SH5
+
+config PMB
bool "Support 32-bit physical addressing through PMB"
depends on MMU && (CPU_SUBTYPE_SH7780 || CPU_SUBTYPE_SH7785)
+ select 32BIT
default y
help
If you say Y here, physical addressing will be extended to
@@ -256,7 +78,7 @@ config X2TLB
config VSYSCALL
bool "Support vsyscall page"
- depends on MMU
+ depends on MMU && (CPU_SH3 || CPU_SH4)
default y
help
This will enable support for the kernel mapping a vDSO page
@@ -335,7 +157,7 @@ config PAGE_SIZE_8KB
config PAGE_SIZE_64KB
bool "64kB"
- depends on CPU_SH4
+ depends on CPU_SH4 || CPU_SH5
help
This enables support for 64kB pages, possible on all SH-4
CPUs and later.
@@ -344,7 +166,7 @@ endchoice
choice
prompt "HugeTLB page size"
- depends on HUGETLB_PAGE && CPU_SH4 && MMU
+ depends on HUGETLB_PAGE && (CPU_SH4 || CPU_SH5) && MMU
default HUGETLB_PAGE_SIZE_64K
config HUGETLB_PAGE_SIZE_64K
@@ -365,6 +187,10 @@ config HUGETLB_PAGE_SIZE_64MB
bool "64MB"
depends on X2TLB
+config HUGETLB_PAGE_SIZE_512MB
+ bool "512MB"
+ depends on CPU_SH5
+
endchoice
source "mm/Kconfig"
@@ -392,12 +218,12 @@ config SH_DIRECT_MAPPED
choice
prompt "Cache mode"
- default CACHE_WRITEBACK if CPU_SH2A || CPU_SH3 || CPU_SH4
+ default CACHE_WRITEBACK if CPU_SH2A || CPU_SH3 || CPU_SH4 || CPU_SH5
default CACHE_WRITETHROUGH if (CPU_SH2 && !CPU_SH2A)
config CACHE_WRITEBACK
bool "Write-back"
- depends on CPU_SH2A || CPU_SH3 || CPU_SH4
+ depends on CPU_SH2A || CPU_SH3 || CPU_SH4 || CPU_SH5
config CACHE_WRITETHROUGH
bool "Write-through"
diff --git a/arch/sh/mm/Makefile b/arch/sh/mm/Makefile
index aa44607..9f4bc3d 100644
--- a/arch/sh/mm/Makefile
+++ b/arch/sh/mm/Makefile
@@ -1,37 +1,5 @@
-#
-# Makefile for the Linux SuperH-specific parts of the memory manager.
-#
-
-obj-y := init.o extable.o consistent.o
-
-ifndef CONFIG_CACHE_OFF
-obj-$(CONFIG_CPU_SH2) += cache-sh2.o
-obj-$(CONFIG_CPU_SH3) += cache-sh3.o
-obj-$(CONFIG_CPU_SH4) += cache-sh4.o
-obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o
+ifeq ($(CONFIG_SUPERH32),y)
+include ${srctree}/arch/sh/mm/Makefile_32
+else
+include ${srctree}/arch/sh/mm/Makefile_64
endif
-
-mmu-y := tlb-nommu.o pg-nommu.o
-mmu-$(CONFIG_MMU) := fault.o clear_page.o copy_page.o tlb-flush.o \
- ioremap.o
-
-obj-y += $(mmu-y)
-
-ifdef CONFIG_DEBUG_FS
-obj-$(CONFIG_CPU_SH4) += cache-debugfs.o
-endif
-
-ifdef CONFIG_MMU
-obj-$(CONFIG_CPU_SH3) += tlb-sh3.o
-obj-$(CONFIG_CPU_SH4) += tlb-sh4.o
-ifndef CONFIG_CACHE_OFF
-obj-$(CONFIG_CPU_SH4) += pg-sh4.o
-obj-$(CONFIG_SH7705_CACHE_32KB) += pg-sh7705.o
-endif
-endif
-
-obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
-obj-$(CONFIG_32BIT) += pmb.o
-obj-$(CONFIG_NUMA) += numa.o
-
-EXTRA_CFLAGS += -Werror
diff --git a/arch/sh/mm/Makefile_32 b/arch/sh/mm/Makefile_32
new file mode 100644
index 0000000..e295db6
--- /dev/null
+++ b/arch/sh/mm/Makefile_32
@@ -0,0 +1,36 @@
+#
+# Makefile for the Linux SuperH-specific parts of the memory manager.
+#
+
+obj-y := init.o extable_32.o consistent.o
+
+ifndef CONFIG_CACHE_OFF
+obj-$(CONFIG_CPU_SH2) += cache-sh2.o
+obj-$(CONFIG_CPU_SH3) += cache-sh3.o
+obj-$(CONFIG_CPU_SH4) += cache-sh4.o
+obj-$(CONFIG_SH7705_CACHE_32KB) += cache-sh7705.o
+endif
+
+mmu-y := tlb-nommu.o pg-nommu.o
+mmu-$(CONFIG_MMU) := fault_32.o tlbflush_32.o ioremap_32.o
+
+obj-y += $(mmu-y)
+
+ifdef CONFIG_DEBUG_FS
+obj-$(CONFIG_CPU_SH4) += cache-debugfs.o
+endif
+
+ifdef CONFIG_MMU
+obj-$(CONFIG_CPU_SH3) += tlb-sh3.o
+obj-$(CONFIG_CPU_SH4) += tlb-sh4.o
+ifndef CONFIG_CACHE_OFF
+obj-$(CONFIG_CPU_SH4) += pg-sh4.o
+obj-$(CONFIG_SH7705_CACHE_32KB) += pg-sh7705.o
+endif
+endif
+
+obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
+obj-$(CONFIG_PMB) += pmb.o
+obj-$(CONFIG_NUMA) += numa.o
+
+EXTRA_CFLAGS += -Werror
diff --git a/arch/sh/mm/Makefile_64 b/arch/sh/mm/Makefile_64
new file mode 100644
index 0000000..cbd6aa3
--- /dev/null
+++ b/arch/sh/mm/Makefile_64
@@ -0,0 +1,44 @@
+#
+# Makefile for the Linux SuperH-specific parts of the memory manager.
+#
+
+obj-y := init.o extable_64.o consistent.o
+
+mmu-y := tlb-nommu.o pg-nommu.o
+mmu-$(CONFIG_MMU) := fault_64.o ioremap_64.o tlbflush_64.o tlb-sh5.o
+
+ifndef CONFIG_CACHE_OFF
+obj-y += cache-sh5.o
+endif
+
+obj-y += $(mmu-y)
+
+obj-$(CONFIG_HUGETLB_PAGE) += hugetlbpage.o
+obj-$(CONFIG_NUMA) += numa.o
+
+EXTRA_CFLAGS += -Werror
+
+# Special flags for fault_64.o. This puts restrictions on the number of
+# caller-save registers that the compiler can target when building this file.
+# This is required because the code is called from a context in entry.S where
+# very few registers have been saved in the exception handler (for speed
+# reasons).
+# The caller save registers that have been saved and which can be used are
+# r2,r3,r4,r5 : argument passing
+# r15, r18 : SP and LINK
+# tr0-4 : allow all caller-save TR's. The compiler seems to be able to make
+# use of them, so it's probably beneficial to performance to save them
+# and have them available for it.
+#
+# The resources not listed below are callee save, i.e. the compiler is free to
+# use any of them and will spill them to the stack itself.
+
+CFLAGS_fault_64.o += -ffixed-r7 \
+ -ffixed-r8 -ffixed-r9 -ffixed-r10 -ffixed-r11 -ffixed-r12 \
+ -ffixed-r13 -ffixed-r14 -ffixed-r16 -ffixed-r17 -ffixed-r19 \
+ -ffixed-r20 -ffixed-r21 -ffixed-r22 -ffixed-r23 \
+ -ffixed-r24 -ffixed-r25 -ffixed-r26 -ffixed-r27 \
+ -ffixed-r36 -ffixed-r37 -ffixed-r38 -ffixed-r39 -ffixed-r40 \
+ -ffixed-r41 -ffixed-r42 -ffixed-r43 \
+ -ffixed-r60 -ffixed-r61 -ffixed-r62 \
+ -fomit-frame-pointer
diff --git a/arch/sh/mm/cache-debugfs.c b/arch/sh/mm/cache-debugfs.c
index de6d2c9..db6d950 100644
--- a/arch/sh/mm/cache-debugfs.c
+++ b/arch/sh/mm/cache-debugfs.c
@@ -22,7 +22,8 @@ enum cache_type {
CACHE_TYPE_UNIFIED,
};
-static int cache_seq_show(struct seq_file *file, void *iter)
+static int __uses_jump_to_uncached cache_seq_show(struct seq_file *file,
+ void *iter)
{
unsigned int cache_type = (unsigned int)file->private;
struct cache_info *cache;
@@ -34,11 +35,11 @@ static int cache_seq_show(struct seq_file *file, void *iter)
* Go uncached immediately so we don't skew the results any
* more than we already are..
*/
- jump_to_P2();
+ jump_to_uncached();
ccr = ctrl_inl(CCR);
if ((ccr & CCR_CACHE_ENABLE) == 0) {
- back_to_P1();
+ back_to_cached();
seq_printf(file, "disabled\n");
return 0;
@@ -104,7 +105,7 @@ static int cache_seq_show(struct seq_file *file, void *iter)
addrstart += cache->way_incr;
}
- back_to_P1();
+ back_to_cached();
return 0;
}
diff --git a/arch/sh/mm/cache-sh4.c b/arch/sh/mm/cache-sh4.c
index 226b190..43d7ff6b 100644
--- a/arch/sh/mm/cache-sh4.c
+++ b/arch/sh/mm/cache-sh4.c
@@ -190,7 +190,7 @@ void flush_icache_range(unsigned long start, unsigned long end)
* .. which happens to be the same behavior as flush_icache_range().
* So, we simply flush out a line.
*/
-void flush_cache_sigtramp(unsigned long addr)
+void __uses_jump_to_uncached flush_cache_sigtramp(unsigned long addr)
{
unsigned long v, index;
unsigned long flags;
@@ -205,13 +205,13 @@ void flush_cache_sigtramp(unsigned long addr)
(v & boot_cpu_data.icache.entry_mask);
local_irq_save(flags);
- jump_to_P2();
+ jump_to_uncached();
for (i = 0; i < boot_cpu_data.icache.ways;
i++, index += boot_cpu_data.icache.way_incr)
ctrl_outl(0, index); /* Clear out Valid-bit */
- back_to_P1();
+ back_to_cached();
wmb();
local_irq_restore(flags);
}
@@ -256,12 +256,12 @@ void flush_dcache_page(struct page *page)
}
/* TODO: Selective icache invalidation through IC address array.. */
-static inline void flush_icache_all(void)
+static inline void __uses_jump_to_uncached flush_icache_all(void)
{
unsigned long flags, ccr;
local_irq_save(flags);
- jump_to_P2();
+ jump_to_uncached();
/* Flush I-cache */
ccr = ctrl_inl(CCR);
@@ -269,11 +269,11 @@ static inline void flush_icache_all(void)
ctrl_outl(ccr, CCR);
/*
- * back_to_P1() will take care of the barrier for us, don't add
+ * back_to_cached() will take care of the barrier for us, don't add
* another one!
*/
- back_to_P1();
+ back_to_cached();
local_irq_restore(flags);
}
diff --git a/arch/sh/mm/cache-sh5.c b/arch/sh/mm/cache-sh5.c
new file mode 100644
index 0000000..4617e3a
--- /dev/null
+++ b/arch/sh/mm/cache-sh5.c
@@ -0,0 +1,1029 @@
+/*
+ * arch/sh/mm/cache-sh5.c
+ *
+ * Original version Copyright (C) 2000, 2001 Paolo Alberelli
+ * Second version Copyright (C) benedict.gaster@superh.com 2002
+ * Third version Copyright Richard.Curnow@superh.com 2003
+ * Hacks to third version Copyright (C) 2003 Paul Mundt
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/init.h>
+#include <linux/mman.h>
+#include <linux/mm.h>
+#include <linux/threads.h>
+#include <asm/page.h>
+#include <asm/pgtable.h>
+#include <asm/processor.h>
+#include <asm/cache.h>
+#include <asm/tlb.h>
+#include <asm/io.h>
+#include <asm/uaccess.h>
+#include <asm/mmu_context.h>
+#include <asm/pgalloc.h> /* for flush_itlb_range */
+
+#include <linux/proc_fs.h>
+
+/* This function is in entry.S */
+extern unsigned long switch_and_save_asid(unsigned long new_asid);
+
+/* Wired TLB entry for the D-cache */
+static unsigned long long dtlb_cache_slot;
+
+/**
+ * sh64_cache_init()
+ *
+ * This is pretty much just a straightforward clone of the SH
+ * detect_cpu_and_cache_system().
+ *
+ * This function is responsible for setting up all of the cache
+ * info dynamically as well as taking care of CPU probing and
+ * setting up the relevant subtype data.
+ *
+ * FIXME: For the time being, we only really support the SH5-101
+ * out of the box, and don't support dynamic probing for things
+ * like the SH5-103 or even cut2 of the SH5-101. Implement this
+ * later!
+ */
+int __init sh64_cache_init(void)
+{
+ /*
+ * First, setup some sane values for the I-cache.
+ */
+ cpu_data->icache.ways = 4;
+ cpu_data->icache.sets = 256;
+ cpu_data->icache.linesz = L1_CACHE_BYTES;
+
+ /*
+ * FIXME: This can probably be cleaned up a bit as well.. for example,
+ * do we really need the way shift _and_ the way_step_shift ?? Judging
+ * by the existing code, I would guess no.. is there any valid reason
+ * why we need to be tracking this around?
+ */
+ cpu_data->icache.way_shift = 13;
+ cpu_data->icache.entry_shift = 5;
+ cpu_data->icache.set_shift = 4;
+ cpu_data->icache.way_step_shift = 16;
+ cpu_data->icache.asid_shift = 2;
+
+ /*
+ * way offset = cache size / associativity, so just don't factor in
+ * associativity in the first place..
+ */
+ cpu_data->icache.way_ofs = cpu_data->icache.sets *
+ cpu_data->icache.linesz;
+
+ cpu_data->icache.asid_mask = 0x3fc;
+ cpu_data->icache.idx_mask = 0x1fe0;
+ cpu_data->icache.epn_mask = 0xffffe000;
+ cpu_data->icache.flags = 0;
+
+ /*
+ * Next, setup some sane values for the D-cache.
+ *
+ * On the SH5, these are pretty consistent with the I-cache settings,
+ * so we just copy over the existing definitions.. these can be fixed
+ * up later, especially if we add runtime CPU probing.
+ *
+ * Though in the meantime it saves us from having to duplicate all of
+ * the above definitions..
+ */
+ cpu_data->dcache = cpu_data->icache;
+
+ /*
+ * Setup any cache-related flags here
+ */
+#if defined(CONFIG_DCACHE_WRITE_THROUGH)
+ set_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags));
+#elif defined(CONFIG_DCACHE_WRITE_BACK)
+ set_bit(SH_CACHE_MODE_WB, &(cpu_data->dcache.flags));
+#endif
+
+ /*
+ * We also need to reserve a slot for the D-cache in the DTLB, so we
+ * do this now ..
+ */
+ dtlb_cache_slot = sh64_get_wired_dtlb_entry();
+
+ return 0;
+}
+
+#ifdef CONFIG_DCACHE_DISABLED
+#define sh64_dcache_purge_all() do { } while (0)
+#define sh64_dcache_purge_coloured_phy_page(paddr, eaddr) do { } while (0)
+#define sh64_dcache_purge_user_range(mm, start, end) do { } while (0)
+#define sh64_dcache_purge_phy_page(paddr) do { } while (0)
+#define sh64_dcache_purge_virt_page(mm, eaddr) do { } while (0)
+#define sh64_dcache_purge_kernel_range(start, end) do { } while (0)
+#define sh64_dcache_wback_current_user_range(start, end) do { } while (0)
+#endif
+
+/*##########################################################################*/
+
+/* From here onwards, a rewrite of the implementation,
+ by Richard.Curnow@superh.com.
+
+ The major changes in this compared to the old version are;
+ 1. use more selective purging through OCBP instead of using ALLOCO to purge
+ by natural replacement. This avoids purging out unrelated cache lines
+ that happen to be in the same set.
+ 2. exploit the APIs copy_user_page and clear_user_page better
+ 3. be more selective about I-cache purging, in particular use invalidate_all
+ more sparingly.
+
+ */
+
+/*##########################################################################
+ SUPPORT FUNCTIONS
+ ##########################################################################*/
+
+/****************************************************************************/
+/* The following group of functions deal with mapping and unmapping a temporary
+ page into the DTLB slot that have been set aside for our exclusive use. */
+/* In order to accomplish this, we use the generic interface for adding and
+ removing a wired slot entry as defined in arch/sh/mm/tlb-sh5.c */
+/****************************************************************************/
+
+static unsigned long slot_own_flags;
+
+static inline void sh64_setup_dtlb_cache_slot(unsigned long eaddr, unsigned long asid, unsigned long paddr)
+{
+ local_irq_save(slot_own_flags);
+ sh64_setup_tlb_slot(dtlb_cache_slot, eaddr, asid, paddr);
+}
+
+static inline void sh64_teardown_dtlb_cache_slot(void)
+{
+ sh64_teardown_tlb_slot(dtlb_cache_slot);
+ local_irq_restore(slot_own_flags);
+}
+
+/****************************************************************************/
+
+#ifndef CONFIG_ICACHE_DISABLED
+
+static void __inline__ sh64_icache_inv_all(void)
+{
+ unsigned long long addr, flag, data;
+ unsigned int flags;
+
+ addr=ICCR0;
+ flag=ICCR0_ICI;
+ data=0;
+
+ /* Make this a critical section for safety (probably not strictly necessary.) */
+ local_irq_save(flags);
+
+ /* Without %1 it gets unexplicably wrong */
+ asm volatile("getcfg %3, 0, %0\n\t"
+ "or %0, %2, %0\n\t"
+ "putcfg %3, 0, %0\n\t"
+ "synci"
+ : "=&r" (data)
+ : "0" (data), "r" (flag), "r" (addr));
+
+ local_irq_restore(flags);
+}
+
+static void sh64_icache_inv_kernel_range(unsigned long start, unsigned long end)
+{
+ /* Invalidate range of addresses [start,end] from the I-cache, where
+ * the addresses lie in the kernel superpage. */
+
+ unsigned long long ullend, addr, aligned_start;
+#if (NEFF == 32)
+ aligned_start = (unsigned long long)(signed long long)(signed long) start;
+#else
+#error "NEFF != 32"
+#endif
+ aligned_start &= L1_CACHE_ALIGN_MASK;
+ addr = aligned_start;
+#if (NEFF == 32)
+ ullend = (unsigned long long) (signed long long) (signed long) end;
+#else
+#error "NEFF != 32"
+#endif
+ while (addr <= ullend) {
+ asm __volatile__ ("icbi %0, 0" : : "r" (addr));
+ addr += L1_CACHE_BYTES;
+ }
+}
+
+static void sh64_icache_inv_user_page(struct vm_area_struct *vma, unsigned long eaddr)
+{
+ /* If we get called, we know that vma->vm_flags contains VM_EXEC.
+ Also, eaddr is page-aligned. */
+
+ unsigned long long addr, end_addr;
+ unsigned long flags = 0;
+ unsigned long running_asid, vma_asid;
+ addr = eaddr;
+ end_addr = addr + PAGE_SIZE;
+
+ /* Check whether we can use the current ASID for the I-cache
+ invalidation. For example, if we're called via
+ access_process_vm->flush_cache_page->here, (e.g. when reading from
+ /proc), 'running_asid' will be that of the reader, not of the
+ victim.
+
+ Also, note the risk that we might get pre-empted between the ASID
+ compare and blocking IRQs, and before we regain control, the
+ pid->ASID mapping changes. However, the whole cache will get
+ invalidated when the mapping is renewed, so the worst that can
+ happen is that the loop below ends up invalidating somebody else's
+ cache entries.
+ */
+
+ running_asid = get_asid();
+ vma_asid = (vma->vm_mm->context & MMU_CONTEXT_ASID_MASK);
+ if (running_asid != vma_asid) {
+ local_irq_save(flags);
+ switch_and_save_asid(vma_asid);
+ }
+ while (addr < end_addr) {
+ /* Worth unrolling a little */
+ asm __volatile__("icbi %0, 0" : : "r" (addr));
+ asm __volatile__("icbi %0, 32" : : "r" (addr));
+ asm __volatile__("icbi %0, 64" : : "r" (addr));
+ asm __volatile__("icbi %0, 96" : : "r" (addr));
+ addr += 128;
+ }
+ if (running_asid != vma_asid) {
+ switch_and_save_asid(running_asid);
+ local_irq_restore(flags);
+ }
+}
+
+/****************************************************************************/
+
+static void sh64_icache_inv_user_page_range(struct mm_struct *mm,
+ unsigned long start, unsigned long end)
+{
+ /* Used for invalidating big chunks of I-cache, i.e. assume the range
+ is whole pages. If 'start' or 'end' is not page aligned, the code
+ is conservative and invalidates to the ends of the enclosing pages.
+ This is functionally OK, just a performance loss. */
+
+ /* See the comments below in sh64_dcache_purge_user_range() regarding
+ the choice of algorithm. However, for the I-cache option (2) isn't
+ available because there are no physical tags so aliases can't be
+ resolved. The icbi instruction has to be used through the user
+ mapping. Because icbi is cheaper than ocbp on a cache hit, it
+ would be cheaper to use the selective code for a large range than is
+ possible with the D-cache. Just assume 64 for now as a working
+ figure.
+ */
+
+ int n_pages;
+
+ if (!mm) return;
+
+ n_pages = ((end - start) >> PAGE_SHIFT);
+ if (n_pages >= 64) {
+ sh64_icache_inv_all();
+ } else {
+ unsigned long aligned_start;
+ unsigned long eaddr;
+ unsigned long after_last_page_start;
+ unsigned long mm_asid, current_asid;
+ unsigned long long flags = 0ULL;
+
+ mm_asid = mm->context & MMU_CONTEXT_ASID_MASK;
+ current_asid = get_asid();
+
+ if (mm_asid != current_asid) {
+ /* Switch ASID and run the invalidate loop under cli */
+ local_irq_save(flags);
+ switch_and_save_asid(mm_asid);
+ }
+
+ aligned_start = start & PAGE_MASK;
+ after_last_page_start = PAGE_SIZE + ((end - 1) & PAGE_MASK);
+
+ while (aligned_start < after_last_page_start) {
+ struct vm_area_struct *vma;
+ unsigned long vma_end;
+ vma = find_vma(mm, aligned_start);
+ if (!vma || (aligned_start <= vma->vm_end)) {
+ /* Avoid getting stuck in an error condition */
+ aligned_start += PAGE_SIZE;
+ continue;
+ }
+ vma_end = vma->vm_end;
+ if (vma->vm_flags & VM_EXEC) {
+ /* Executable */
+ eaddr = aligned_start;
+ while (eaddr < vma_end) {
+ sh64_icache_inv_user_page(vma, eaddr);
+ eaddr += PAGE_SIZE;
+ }
+ }
+ aligned_start = vma->vm_end; /* Skip to start of next region */
+ }
+ if (mm_asid != current_asid) {
+ switch_and_save_asid(current_asid);
+ local_irq_restore(flags);
+ }
+ }
+}
+
+static void sh64_icache_inv_user_small_range(struct mm_struct *mm,
+ unsigned long start, int len)
+{
+
+ /* Invalidate a small range of user context I-cache, not necessarily
+ page (or even cache-line) aligned. */
+
+ unsigned long long eaddr = start;
+ unsigned long long eaddr_end = start + len;
+ unsigned long current_asid, mm_asid;
+ unsigned long long flags;
+ unsigned long long epage_start;
+
+ /* Since this is used inside ptrace, the ASID in the mm context
+ typically won't match current_asid. We'll have to switch ASID to do
+ this. For safety, and given that the range will be small, do all
+ this under cli.
+
+ Note, there is a hazard that the ASID in mm->context is no longer
+ actually associated with mm, i.e. if the mm->context has started a
+ new cycle since mm was last active. However, this is just a
+ performance issue: all that happens is that we invalidate lines
+ belonging to another mm, so the owning process has to refill them
+ when that mm goes live again. mm itself can't have any cache
+ entries because there will have been a flush_cache_all when the new
+ mm->context cycle started. */
+
+ /* Align to start of cache line. Otherwise, suppose len==8 and start
+ was at 32N+28 : the last 4 bytes wouldn't get invalidated. */
+ eaddr = start & L1_CACHE_ALIGN_MASK;
+ eaddr_end = start + len;
+
+ local_irq_save(flags);
+ mm_asid = mm->context & MMU_CONTEXT_ASID_MASK;
+ current_asid = switch_and_save_asid(mm_asid);
+
+ epage_start = eaddr & PAGE_MASK;
+
+ while (eaddr < eaddr_end)
+ {
+ asm __volatile__("icbi %0, 0" : : "r" (eaddr));
+ eaddr += L1_CACHE_BYTES;
+ }
+ switch_and_save_asid(current_asid);
+ local_irq_restore(flags);
+}
+
+static void sh64_icache_inv_current_user_range(unsigned long start, unsigned long end)
+{
+ /* The icbi instruction never raises ITLBMISS. i.e. if there's not a
+ cache hit on the virtual tag the instruction ends there, without a
+ TLB lookup. */
+
+ unsigned long long aligned_start;
+ unsigned long long ull_end;
+ unsigned long long addr;
+
+ ull_end = end;
+
+ /* Just invalidate over the range using the natural addresses. TLB
+ miss handling will be OK (TBC). Since it's for the current process,
+ either we're already in the right ASID context, or the ASIDs have
+ been recycled since we were last active in which case we might just
+ invalidate another processes I-cache entries : no worries, just a
+ performance drop for him. */
+ aligned_start = start & L1_CACHE_ALIGN_MASK;
+ addr = aligned_start;
+ while (addr < ull_end) {
+ asm __volatile__ ("icbi %0, 0" : : "r" (addr));
+ asm __volatile__ ("nop");
+ asm __volatile__ ("nop");
+ addr += L1_CACHE_BYTES;
+ }
+}
+
+#endif /* !CONFIG_ICACHE_DISABLED */
+
+/****************************************************************************/
+
+#ifndef CONFIG_DCACHE_DISABLED
+
+/* Buffer used as the target of alloco instructions to purge data from cache
+ sets by natural eviction. -- RPC */
+#define DUMMY_ALLOCO_AREA_SIZE L1_CACHE_SIZE_BYTES + (1024 * 4)
+static unsigned char dummy_alloco_area[DUMMY_ALLOCO_AREA_SIZE] __cacheline_aligned = { 0, };
+
+/****************************************************************************/
+
+static void __inline__ sh64_dcache_purge_sets(int sets_to_purge_base, int n_sets)
+{
+ /* Purge all ways in a particular block of sets, specified by the base
+ set number and number of sets. Can handle wrap-around, if that's
+ needed. */
+
+ int dummy_buffer_base_set;
+ unsigned long long eaddr, eaddr0, eaddr1;
+ int j;
+ int set_offset;
+
+ dummy_buffer_base_set = ((int)&dummy_alloco_area & cpu_data->dcache.idx_mask) >> cpu_data->dcache.entry_shift;
+ set_offset = sets_to_purge_base - dummy_buffer_base_set;
+
+ for (j=0; j<n_sets; j++, set_offset++) {
+ set_offset &= (cpu_data->dcache.sets - 1);
+ eaddr0 = (unsigned long long)dummy_alloco_area + (set_offset << cpu_data->dcache.entry_shift);
+
+ /* Do one alloco which hits the required set per cache way. For
+ write-back mode, this will purge the #ways resident lines. There's
+ little point unrolling this loop because the allocos stall more if
+ they're too close together. */
+ eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways;
+ for (eaddr=eaddr0; eaddr<eaddr1; eaddr+=cpu_data->dcache.way_ofs) {
+ asm __volatile__ ("alloco %0, 0" : : "r" (eaddr));
+ asm __volatile__ ("synco"); /* TAKum03020 */
+ }
+
+ eaddr1 = eaddr0 + cpu_data->dcache.way_ofs * cpu_data->dcache.ways;
+ for (eaddr=eaddr0; eaddr<eaddr1; eaddr+=cpu_data->dcache.way_ofs) {
+ /* Load from each address. Required because alloco is a NOP if
+ the cache is write-through. Write-through is a config option. */
+ if (test_bit(SH_CACHE_MODE_WT, &(cpu_data->dcache.flags)))
+ *(volatile unsigned char *)(int)eaddr;
+ }
+ }
+
+ /* Don't use OCBI to invalidate the lines. That costs cycles directly.
+ If the dummy block is just left resident, it will naturally get
+ evicted as required. */
+
+ return;
+}
+
+/****************************************************************************/
+
+static void sh64_dcache_purge_all(void)
+{
+ /* Purge the entire contents of the dcache. The most efficient way to
+ achieve this is to use alloco instructions on a region of unused
+ memory equal in size to the cache, thereby causing the current
+ contents to be discarded by natural eviction. The alternative,
+ namely reading every tag, setting up a mapping for the corresponding
+ page and doing an OCBP for the line, would be much more expensive.
+ */
+
+ sh64_dcache_purge_sets(0, cpu_data->dcache.sets);
+
+ return;
+
+}
+
+/****************************************************************************/
+
+static void sh64_dcache_purge_kernel_range(unsigned long start, unsigned long end)
+{
+ /* Purge the range of addresses [start,end] from the D-cache. The
+ addresses lie in the superpage mapping. There's no harm if we
+ overpurge at either end - just a small performance loss. */
+ unsigned long long ullend, addr, aligned_start;
+#if (NEFF == 32)
+ aligned_start = (unsigned long long)(signed long long)(signed long) start;
+#else
+#error "NEFF != 32"
+#endif
+ aligned_start &= L1_CACHE_ALIGN_MASK;
+ addr = aligned_start;
+#if (NEFF == 32)
+ ullend = (unsigned long long) (signed long long) (signed long) end;
+#else
+#error "NEFF != 32"
+#endif
+ while (addr <= ullend) {
+ asm __volatile__ ("ocbp %0, 0" : : "r" (addr));
+ addr += L1_CACHE_BYTES;
+ }
+ return;
+}
+
+/* Assumes this address (+ (2**n_synbits) pages up from it) aren't used for
+ anything else in the kernel */
+#define MAGIC_PAGE0_START 0xffffffffec000000ULL
+
+static void sh64_dcache_purge_coloured_phy_page(unsigned long paddr, unsigned long eaddr)
+{
+ /* Purge the physical page 'paddr' from the cache. It's known that any
+ cache lines requiring attention have the same page colour as the the
+ address 'eaddr'.
+
+ This relies on the fact that the D-cache matches on physical tags
+ when no virtual tag matches. So we create an alias for the original
+ page and purge through that. (Alternatively, we could have done
+ this by switching ASID to match the original mapping and purged
+ through that, but that involves ASID switching cost + probably a
+ TLBMISS + refill anyway.)
+ */
+
+ unsigned long long magic_page_start;
+ unsigned long long magic_eaddr, magic_eaddr_end;
+
+ magic_page_start = MAGIC_PAGE0_START + (eaddr & CACHE_OC_SYN_MASK);
+
+ /* As long as the kernel is not pre-emptible, this doesn't need to be
+ under cli/sti. */
+
+ sh64_setup_dtlb_cache_slot(magic_page_start, get_asid(), paddr);
+
+ magic_eaddr = magic_page_start;
+ magic_eaddr_end = magic_eaddr + PAGE_SIZE;
+ while (magic_eaddr < magic_eaddr_end) {
+ /* Little point in unrolling this loop - the OCBPs are blocking
+ and won't go any quicker (i.e. the loop overhead is parallel
+ to part of the OCBP execution.) */
+ asm __volatile__ ("ocbp %0, 0" : : "r" (magic_eaddr));
+ magic_eaddr += L1_CACHE_BYTES;
+ }
+
+ sh64_teardown_dtlb_cache_slot();
+}
+
+/****************************************************************************/
+
+static void sh64_dcache_purge_phy_page(unsigned long paddr)
+{
+ /* Pure a page given its physical start address, by creating a
+ temporary 1 page mapping and purging across that. Even if we know
+ the virtual address (& vma or mm) of the page, the method here is
+ more elegant because it avoids issues of coping with page faults on
+ the purge instructions (i.e. no special-case code required in the
+ critical path in the TLB miss handling). */
+
+ unsigned long long eaddr_start, eaddr, eaddr_end;
+ int i;
+
+ /* As long as the kernel is not pre-emptible, this doesn't need to be
+ under cli/sti. */
+
+ eaddr_start = MAGIC_PAGE0_START;
+ for (i=0; i < (1 << CACHE_OC_N_SYNBITS); i++) {
+ sh64_setup_dtlb_cache_slot(eaddr_start, get_asid(), paddr);
+
+ eaddr = eaddr_start;
+ eaddr_end = eaddr + PAGE_SIZE;
+ while (eaddr < eaddr_end) {
+ asm __volatile__ ("ocbp %0, 0" : : "r" (eaddr));
+ eaddr += L1_CACHE_BYTES;
+ }
+
+ sh64_teardown_dtlb_cache_slot();
+ eaddr_start += PAGE_SIZE;
+ }
+}
+
+static void sh64_dcache_purge_user_pages(struct mm_struct *mm,
+ unsigned long addr, unsigned long end)
+{
+ pgd_t *pgd;
+ pmd_t *pmd;
+ pte_t *pte;
+ pte_t entry;
+ spinlock_t *ptl;
+ unsigned long paddr;
+
+ if (!mm)
+ return; /* No way to find physical address of page */
+
+ pgd = pgd_offset(mm, addr);
+ if (pgd_bad(*pgd))
+ return;
+
+ pmd = pmd_offset(pgd, addr);
+ if (pmd_none(*pmd) || pmd_bad(*pmd))
+ return;
+
+ pte = pte_offset_map_lock(mm, pmd, addr, &ptl);
+ do {
+ entry = *pte;
+ if (pte_none(entry) || !pte_present(entry))
+ continue;
+ paddr = pte_val(entry) & PAGE_MASK;
+ sh64_dcache_purge_coloured_phy_page(paddr, addr);
+ } while (pte++, addr += PAGE_SIZE, addr != end);
+ pte_unmap_unlock(pte - 1, ptl);
+}
+/****************************************************************************/
+
+static void sh64_dcache_purge_user_range(struct mm_struct *mm,
+ unsigned long start, unsigned long end)
+{
+ /* There are at least 5 choices for the implementation of this, with
+ pros (+), cons(-), comments(*):
+
+ 1. ocbp each line in the range through the original user's ASID
+ + no lines spuriously evicted
+ - tlbmiss handling (must either handle faults on demand => extra
+ special-case code in tlbmiss critical path), or map the page in
+ advance (=> flush_tlb_range in advance to avoid multiple hits)
+ - ASID switching
+ - expensive for large ranges
+
+ 2. temporarily map each page in the range to a special effective
+ address and ocbp through the temporary mapping; relies on the
+ fact that SH-5 OCB* always do TLB lookup and match on ptags (they
+ never look at the etags)
+ + no spurious evictions
+ - expensive for large ranges
+ * surely cheaper than (1)
+
+ 3. walk all the lines in the cache, check the tags, if a match
+ occurs create a page mapping to ocbp the line through
+ + no spurious evictions
+ - tag inspection overhead
+ - (especially for small ranges)
+ - potential cost of setting up/tearing down page mapping for
+ every line that matches the range
+ * cost partly independent of range size
+
+ 4. walk all the lines in the cache, check the tags, if a match
+ occurs use 4 * alloco to purge the line (+3 other probably
+ innocent victims) by natural eviction
+ + no tlb mapping overheads
+ - spurious evictions
+ - tag inspection overhead
+
+ 5. implement like flush_cache_all
+ + no tag inspection overhead
+ - spurious evictions
+ - bad for small ranges
+
+ (1) can be ruled out as more expensive than (2). (2) appears best
+ for small ranges. The choice between (3), (4) and (5) for large
+ ranges and the range size for the large/small boundary need
+ benchmarking to determine.
+
+ For now use approach (2) for small ranges and (5) for large ones.
+
+ */
+
+ int n_pages;
+
+ n_pages = ((end - start) >> PAGE_SHIFT);
+ if (n_pages >= 64 || ((start ^ (end - 1)) & PMD_MASK)) {
+#if 1
+ sh64_dcache_purge_all();
+#else
+ unsigned long long set, way;
+ unsigned long mm_asid = mm->context & MMU_CONTEXT_ASID_MASK;
+ for (set = 0; set < cpu_data->dcache.sets; set++) {
+ unsigned long long set_base_config_addr = CACHE_OC_ADDRESS_ARRAY + (set << cpu_data->dcache.set_shift);
+ for (way = 0; way < cpu_data->dcache.ways; way++) {
+ unsigned long long config_addr = set_base_config_addr + (way << cpu_data->dcache.way_step_shift);
+ unsigned long long tag0;
+ unsigned long line_valid;
+
+ asm __volatile__("getcfg %1, 0, %0" : "=r" (tag0) : "r" (config_addr));
+ line_valid = tag0 & SH_CACHE_VALID;
+ if (line_valid) {
+ unsigned long cache_asid;
+ unsigned long epn;
+
+ cache_asid = (tag0 & cpu_data->dcache.asid_mask) >> cpu_data->dcache.asid_shift;
+ /* The next line needs some
+ explanation. The virtual tags
+ encode bits [31:13] of the virtual
+ address, bit [12] of the 'tag' being
+ implied by the cache set index. */
+ epn = (tag0 & cpu_data->dcache.epn_mask) | ((set & 0x80) << cpu_data->dcache.entry_shift);
+
+ if ((cache_asid == mm_asid) && (start <= epn) && (epn < end)) {
+ /* TODO : could optimise this
+ call by batching multiple
+ adjacent sets together. */
+ sh64_dcache_purge_sets(set, 1);
+ break; /* Don't waste time inspecting other ways for this set */
+ }
+ }
+ }
+ }
+#endif
+ } else {
+ /* Small range, covered by a single page table page */
+ start &= PAGE_MASK; /* should already be so */
+ end = PAGE_ALIGN(end); /* should already be so */
+ sh64_dcache_purge_user_pages(mm, start, end);
+ }
+ return;
+}
+
+static void sh64_dcache_wback_current_user_range(unsigned long start, unsigned long end)
+{
+ unsigned long long aligned_start;
+ unsigned long long ull_end;
+ unsigned long long addr;
+
+ ull_end = end;
+
+ /* Just wback over the range using the natural addresses. TLB miss
+ handling will be OK (TBC) : the range has just been written to by
+ the signal frame setup code, so the PTEs must exist.
+
+ Note, if we have CONFIG_PREEMPT and get preempted inside this loop,
+ it doesn't matter, even if the pid->ASID mapping changes whilst
+ we're away. In that case the cache will have been flushed when the
+ mapping was renewed. So the writebacks below will be nugatory (and
+ we'll doubtless have to fault the TLB entry/ies in again with the
+ new ASID), but it's a rare case.
+ */
+ aligned_start = start & L1_CACHE_ALIGN_MASK;
+ addr = aligned_start;
+ while (addr < ull_end) {
+ asm __volatile__ ("ocbwb %0, 0" : : "r" (addr));
+ addr += L1_CACHE_BYTES;
+ }
+}
+
+/****************************************************************************/
+
+/* These *MUST* lie in an area of virtual address space that's otherwise unused. */
+#define UNIQUE_EADDR_START 0xe0000000UL
+#define UNIQUE_EADDR_END 0xe8000000UL
+
+static unsigned long sh64_make_unique_eaddr(unsigned long user_eaddr, unsigned long paddr)
+{
+ /* Given a physical address paddr, and a user virtual address
+ user_eaddr which will eventually be mapped to it, create a one-off
+ kernel-private eaddr mapped to the same paddr. This is used for
+ creating special destination pages for copy_user_page and
+ clear_user_page */
+
+ static unsigned long current_pointer = UNIQUE_EADDR_START;
+ unsigned long coloured_pointer;
+
+ if (current_pointer == UNIQUE_EADDR_END) {
+ sh64_dcache_purge_all();
+ current_pointer = UNIQUE_EADDR_START;
+ }
+
+ coloured_pointer = (current_pointer & ~CACHE_OC_SYN_MASK) | (user_eaddr & CACHE_OC_SYN_MASK);
+ sh64_setup_dtlb_cache_slot(coloured_pointer, get_asid(), paddr);
+
+ current_pointer += (PAGE_SIZE << CACHE_OC_N_SYNBITS);
+
+ return coloured_pointer;
+}
+
+/****************************************************************************/
+
+static void sh64_copy_user_page_coloured(void *to, void *from, unsigned long address)
+{
+ void *coloured_to;
+
+ /* Discard any existing cache entries of the wrong colour. These are
+ present quite often, if the kernel has recently used the page
+ internally, then given it up, then it's been allocated to the user.
+ */
+ sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to);
+
+ coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to));
+ sh64_page_copy(from, coloured_to);
+
+ sh64_teardown_dtlb_cache_slot();
+}
+
+static void sh64_clear_user_page_coloured(void *to, unsigned long address)
+{
+ void *coloured_to;
+
+ /* Discard any existing kernel-originated lines of the wrong colour (as
+ above) */
+ sh64_dcache_purge_coloured_phy_page(__pa(to), (unsigned long) to);
+
+ coloured_to = (void *) sh64_make_unique_eaddr(address, __pa(to));
+ sh64_page_clear(coloured_to);
+
+ sh64_teardown_dtlb_cache_slot();
+}
+
+#endif /* !CONFIG_DCACHE_DISABLED */
+
+/****************************************************************************/
+
+/*##########################################################################
+ EXTERNALLY CALLABLE API.
+ ##########################################################################*/
+
+/* These functions are described in Documentation/cachetlb.txt.
+ Each one of these functions varies in behaviour depending on whether the
+ I-cache and/or D-cache are configured out.
+
+ Note that the Linux term 'flush' corresponds to what is termed 'purge' in
+ the sh/sh64 jargon for the D-cache, i.e. write back dirty data then
+ invalidate the cache lines, and 'invalidate' for the I-cache.
+ */
+
+#undef FLUSH_TRACE
+
+void flush_cache_all(void)
+{
+ /* Invalidate the entire contents of both caches, after writing back to
+ memory any dirty data from the D-cache. */
+ sh64_dcache_purge_all();
+ sh64_icache_inv_all();
+}
+
+/****************************************************************************/
+
+void flush_cache_mm(struct mm_struct *mm)
+{
+ /* Invalidate an entire user-address space from both caches, after
+ writing back dirty data (e.g. for shared mmap etc). */
+
+ /* This could be coded selectively by inspecting all the tags then
+ doing 4*alloco on any set containing a match (as for
+ flush_cache_range), but fork/exit/execve (where this is called from)
+ are expensive anyway. */
+
+ /* Have to do a purge here, despite the comments re I-cache below.
+ There could be odd-coloured dirty data associated with the mm still
+ in the cache - if this gets written out through natural eviction
+ after the kernel has reused the page there will be chaos.
+ */
+
+ sh64_dcache_purge_all();
+
+ /* The mm being torn down won't ever be active again, so any Icache
+ lines tagged with its ASID won't be visible for the rest of the
+ lifetime of this ASID cycle. Before the ASID gets reused, there
+ will be a flush_cache_all. Hence we don't need to touch the
+ I-cache. This is similar to the lack of action needed in
+ flush_tlb_mm - see fault.c. */
+}
+
+/****************************************************************************/
+
+void flush_cache_range(struct vm_area_struct *vma, unsigned long start,
+ unsigned long end)
+{
+ struct mm_struct *mm = vma->vm_mm;
+
+ /* Invalidate (from both caches) the range [start,end) of virtual
+ addresses from the user address space specified by mm, after writing
+ back any dirty data.
+
+ Note, 'end' is 1 byte beyond the end of the range to flush. */
+
+ sh64_dcache_purge_user_range(mm, start, end);
+ sh64_icache_inv_user_page_range(mm, start, end);
+}
+
+/****************************************************************************/
+
+void flush_cache_page(struct vm_area_struct *vma, unsigned long eaddr, unsigned long pfn)
+{
+ /* Invalidate any entries in either cache for the vma within the user
+ address space vma->vm_mm for the page starting at virtual address
+ 'eaddr'. This seems to be used primarily in breaking COW. Note,
+ the I-cache must be searched too in case the page in question is
+ both writable and being executed from (e.g. stack trampolines.)
+
+ Note, this is called with pte lock held.
+ */
+
+ sh64_dcache_purge_phy_page(pfn << PAGE_SHIFT);
+
+ if (vma->vm_flags & VM_EXEC) {
+ sh64_icache_inv_user_page(vma, eaddr);
+ }
+}
+
+/****************************************************************************/
+
+#ifndef CONFIG_DCACHE_DISABLED
+
+void copy_user_page(void *to, void *from, unsigned long address, struct page *page)
+{
+ /* 'from' and 'to' are kernel virtual addresses (within the superpage
+ mapping of the physical RAM). 'address' is the user virtual address
+ where the copy 'to' will be mapped after. This allows a custom
+ mapping to be used to ensure that the new copy is placed in the
+ right cache sets for the user to see it without having to bounce it
+ out via memory. Note however : the call to flush_page_to_ram in
+ (generic)/mm/memory.c:(break_cow) undoes all this good work in that one
+ very important case!
+
+ TBD : can we guarantee that on every call, any cache entries for
+ 'from' are in the same colour sets as 'address' also? i.e. is this
+ always used just to deal with COW? (I suspect not). */
+
+ /* There are two possibilities here for when the page 'from' was last accessed:
+ * by the kernel : this is OK, no purge required.
+ * by the/a user (e.g. for break_COW) : need to purge.
+
+ If the potential user mapping at 'address' is the same colour as
+ 'from' there is no need to purge any cache lines from the 'from'
+ page mapped into cache sets of colour 'address'. (The copy will be
+ accessing the page through 'from').
+ */
+
+ if (((address ^ (unsigned long) from) & CACHE_OC_SYN_MASK) != 0) {
+ sh64_dcache_purge_coloured_phy_page(__pa(from), address);
+ }
+
+ if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) {
+ /* No synonym problem on destination */
+ sh64_page_copy(from, to);
+ } else {
+ sh64_copy_user_page_coloured(to, from, address);
+ }
+
+ /* Note, don't need to flush 'from' page from the cache again - it's
+ done anyway by the generic code */
+}
+
+void clear_user_page(void *to, unsigned long address, struct page *page)
+{
+ /* 'to' is a kernel virtual address (within the superpage
+ mapping of the physical RAM). 'address' is the user virtual address
+ where the 'to' page will be mapped after. This allows a custom
+ mapping to be used to ensure that the new copy is placed in the
+ right cache sets for the user to see it without having to bounce it
+ out via memory.
+ */
+
+ if (((address ^ (unsigned long) to) & CACHE_OC_SYN_MASK) == 0) {
+ /* No synonym problem on destination */
+ sh64_page_clear(to);
+ } else {
+ sh64_clear_user_page_coloured(to, address);
+ }
+}
+
+#endif /* !CONFIG_DCACHE_DISABLED */
+
+/****************************************************************************/
+
+void flush_dcache_page(struct page *page)
+{
+ sh64_dcache_purge_phy_page(page_to_phys(page));
+ wmb();
+}
+
+/****************************************************************************/
+
+void flush_icache_range(unsigned long start, unsigned long end)
+{
+ /* Flush the range [start,end] of kernel virtual adddress space from
+ the I-cache. The corresponding range must be purged from the
+ D-cache also because the SH-5 doesn't have cache snooping between
+ the caches. The addresses will be visible through the superpage
+ mapping, therefore it's guaranteed that there no cache entries for
+ the range in cache sets of the wrong colour.
+
+ Primarily used for cohering the I-cache after a module has
+ been loaded. */
+
+ /* We also make sure to purge the same range from the D-cache since
+ flush_page_to_ram() won't be doing this for us! */
+
+ sh64_dcache_purge_kernel_range(start, end);
+ wmb();
+ sh64_icache_inv_kernel_range(start, end);
+}
+
+/****************************************************************************/
+
+void flush_icache_user_range(struct vm_area_struct *vma,
+ struct page *page, unsigned long addr, int len)
+{
+ /* Flush the range of user (defined by vma->vm_mm) address space
+ starting at 'addr' for 'len' bytes from the cache. The range does
+ not straddle a page boundary, the unique physical page containing
+ the range is 'page'. This seems to be used mainly for invalidating
+ an address range following a poke into the program text through the
+ ptrace() call from another process (e.g. for BRK instruction
+ insertion). */
+
+ sh64_dcache_purge_coloured_phy_page(page_to_phys(page), addr);
+ mb();
+
+ if (vma->vm_flags & VM_EXEC) {
+ sh64_icache_inv_user_small_range(vma->vm_mm, addr, len);
+ }
+}
+
+/*##########################################################################
+ ARCH/SH64 PRIVATE CALLABLE API.
+ ##########################################################################*/
+
+void flush_cache_sigtramp(unsigned long start, unsigned long end)
+{
+ /* For the address range [start,end), write back the data from the
+ D-cache and invalidate the corresponding region of the I-cache for
+ the current process. Used to flush signal trampolines on the stack
+ to make them executable. */
+
+ sh64_dcache_wback_current_user_range(start, end);
+ wmb();
+ sh64_icache_inv_current_user_range(start, end);
+}
+
diff --git a/arch/sh/mm/cache-sh7705.c b/arch/sh/mm/cache-sh7705.c
index 4896d73..22dacc7 100644
--- a/arch/sh/mm/cache-sh7705.c
+++ b/arch/sh/mm/cache-sh7705.c
@@ -71,7 +71,7 @@ void flush_icache_range(unsigned long start, unsigned long end)
/*
* Writeback&Invalidate the D-cache of the page
*/
-static void __flush_dcache_page(unsigned long phys)
+static void __uses_jump_to_uncached __flush_dcache_page(unsigned long phys)
{
unsigned long ways, waysize, addrstart;
unsigned long flags;
@@ -92,7 +92,7 @@ static void __flush_dcache_page(unsigned long phys)
* possible.
*/
local_irq_save(flags);
- jump_to_P2();
+ jump_to_uncached();
ways = current_cpu_data.dcache.ways;
waysize = current_cpu_data.dcache.sets;
@@ -118,7 +118,7 @@ static void __flush_dcache_page(unsigned long phys)
addrstart += current_cpu_data.dcache.way_incr;
} while (--ways);
- back_to_P1();
+ back_to_cached();
local_irq_restore(flags);
}
@@ -132,15 +132,15 @@ void flush_dcache_page(struct page *page)
__flush_dcache_page(PHYSADDR(page_address(page)));
}
-void flush_cache_all(void)
+void __uses_jump_to_uncached flush_cache_all(void)
{
unsigned long flags;
local_irq_save(flags);
- jump_to_P2();
+ jump_to_uncached();
cache_wback_all();
- back_to_P1();
+ back_to_cached();
local_irq_restore(flags);
}
diff --git a/arch/sh/mm/clear_page.S b/arch/sh/mm/clear_page.S
deleted file mode 100644
index 7a7c81e..0000000
--- a/arch/sh/mm/clear_page.S
+++ /dev/null
@@ -1,152 +0,0 @@
-/*
- * __clear_user_page, __clear_user, clear_page implementation of SuperH
- *
- * Copyright (C) 2001 Kaz Kojima
- * Copyright (C) 2001, 2002 Niibe Yutaka
- * Copyright (C) 2006 Paul Mundt
- */
-#include <linux/linkage.h>
-#include <asm/page.h>
-
-/*
- * clear_page_slow
- * @to: P1 address
- *
- * void clear_page_slow(void *to)
- */
-
-/*
- * r0 --- scratch
- * r4 --- to
- * r5 --- to + PAGE_SIZE
- */
-ENTRY(clear_page_slow)
- mov r4,r5
- mov.l .Llimit,r0
- add r0,r5
- mov #0,r0
- !
-1:
-#if defined(CONFIG_CPU_SH3)
- mov.l r0,@r4
-#elif defined(CONFIG_CPU_SH4)
- movca.l r0,@r4
- mov r4,r1
-#endif
- add #32,r4
- mov.l r0,@-r4
- mov.l r0,@-r4
- mov.l r0,@-r4
- mov.l r0,@-r4
- mov.l r0,@-r4
- mov.l r0,@-r4
- mov.l r0,@-r4
-#if defined(CONFIG_CPU_SH4)
- ocbwb @r1
-#endif
- cmp/eq r5,r4
- bf/s 1b
- add #28,r4
- !
- rts
- nop
-.Llimit: .long (PAGE_SIZE-28)
-
-ENTRY(__clear_user)
- !
- mov #0, r0
- mov #0xe0, r1 ! 0xffffffe0
- !
- ! r4..(r4+31)&~32 -------- not aligned [ Area 0 ]
- ! (r4+31)&~32..(r4+r5)&~32 -------- aligned [ Area 1 ]
- ! (r4+r5)&~32..r4+r5 -------- not aligned [ Area 2 ]
- !
- ! Clear area 0
- mov r4, r2
- !
- tst r1, r5 ! length < 32
- bt .Larea2 ! skip to remainder
- !
- add #31, r2
- and r1, r2
- cmp/eq r4, r2
- bt .Larea1
- mov r2, r3
- sub r4, r3
- mov r3, r7
- mov r4, r2
- !
-.L0: dt r3
-0: mov.b r0, @r2
- bf/s .L0
- add #1, r2
- !
- sub r7, r5
- mov r2, r4
-.Larea1:
- mov r4, r3
- add r5, r3
- and r1, r3
- cmp/hi r2, r3
- bf .Larea2
- !
- ! Clear area 1
-#if defined(CONFIG_CPU_SH4)
-1: movca.l r0, @r2
-#else
-1: mov.l r0, @r2
-#endif
- add #4, r2
-2: mov.l r0, @r2
- add #4, r2
-3: mov.l r0, @r2
- add #4, r2
-4: mov.l r0, @r2
- add #4, r2
-5: mov.l r0, @r2
- add #4, r2
-6: mov.l r0, @r2
- add #4, r2
-7: mov.l r0, @r2
- add #4, r2
-8: mov.l r0, @r2
- add #4, r2
- cmp/hi r2, r3
- bt/s 1b
- nop
- !
- ! Clear area 2
-.Larea2:
- mov r4, r3
- add r5, r3
- cmp/hs r3, r2
- bt/s .Ldone
- sub r2, r3
-.L2: dt r3
-9: mov.b r0, @r2
- bf/s .L2
- add #1, r2
- !
-.Ldone: rts
- mov #0, r0 ! return 0 as normal return
-
- ! return the number of bytes remained
-.Lbad_clear_user:
- mov r4, r0
- add r5, r0
- rts
- sub r2, r0
-
-.section __ex_table,"a"
- .align 2
- .long 0b, .Lbad_clear_user
- .long 1b, .Lbad_clear_user
- .long 2b, .Lbad_clear_user
- .long 3b, .Lbad_clear_user
- .long 4b, .Lbad_clear_user
- .long 5b, .Lbad_clear_user
- .long 6b, .Lbad_clear_user
- .long 7b, .Lbad_clear_user
- .long 8b, .Lbad_clear_user
- .long 9b, .Lbad_clear_user
-.previous
diff --git a/arch/sh/mm/consistent.c b/arch/sh/mm/consistent.c
index e220c29..7b2131c 100644
--- a/arch/sh/mm/consistent.c
+++ b/arch/sh/mm/consistent.c
@@ -1,7 +1,9 @@
/*
* arch/sh/mm/consistent.c
*
- * Copyright (C) 2004 Paul Mundt
+ * Copyright (C) 2004 - 2007 Paul Mundt
+ *
+ * Declared coherent memory functions based on arch/x86/kernel/pci-dma_32.c
*
* This file is subject to the terms and conditions of the GNU General Public
* License. See the file "COPYING" in the main directory of this archive
@@ -13,58 +15,152 @@
#include <asm/addrspace.h>
#include <asm/io.h>
-void *consistent_alloc(gfp_t gfp, size_t size, dma_addr_t *handle)
+struct dma_coherent_mem {
+ void *virt_base;
+ u32 device_base;
+ int size;
+ int flags;
+ unsigned long *bitmap;
+};
+
+void *dma_alloc_coherent(struct device *dev, size_t size,
+ dma_addr_t *dma_handle, gfp_t gfp)
{
- struct page *page, *end, *free;
void *ret;
- int order;
+ struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
+ int order = get_order(size);
- size = PAGE_ALIGN(size);
- order = get_order(size);
+ if (mem) {
+ int page = bitmap_find_free_region(mem->bitmap, mem->size,
+ order);
+ if (page >= 0) {
+ *dma_handle = mem->device_base + (page << PAGE_SHIFT);
+ ret = mem->virt_base + (page << PAGE_SHIFT);
+ memset(ret, 0, size);
+ return ret;
+ }
+ if (mem->flags & DMA_MEMORY_EXCLUSIVE)
+ return NULL;
+ }
- page = alloc_pages(gfp, order);
- if (!page)
- return NULL;
- split_page(page, order);
+ ret = (void *)__get_free_pages(gfp, order);
- ret = page_address(page);
- memset(ret, 0, size);
- *handle = virt_to_phys(ret);
+ if (ret != NULL) {
+ memset(ret, 0, size);
+ /*
+ * Pages from the page allocator may have data present in
+ * cache. So flush the cache before using uncached memory.
+ */
+ dma_cache_sync(NULL, ret, size, DMA_BIDIRECTIONAL);
+ *dma_handle = virt_to_phys(ret);
+ }
+ return ret;
+}
+EXPORT_SYMBOL(dma_alloc_coherent);
- /*
- * We must flush the cache before we pass it on to the device
- */
- __flush_purge_region(ret, size);
+void dma_free_coherent(struct device *dev, size_t size,
+ void *vaddr, dma_addr_t dma_handle)
+{
+ struct dma_coherent_mem *mem = dev ? dev->dma_mem : NULL;
+ int order = get_order(size);
- page = virt_to_page(ret);
- free = page + (size >> PAGE_SHIFT);
- end = page + (1 << order);
+ if (mem && vaddr >= mem->virt_base && vaddr < (mem->virt_base + (mem->size << PAGE_SHIFT))) {
+ int page = (vaddr - mem->virt_base) >> PAGE_SHIFT;
- while (++page < end) {
- /* Free any unused pages */
- if (page >= free) {
- __free_page(page);
- }
+ bitmap_release_region(mem->bitmap, page, order);
+ } else {
+ WARN_ON(irqs_disabled()); /* for portability */
+ BUG_ON(mem && mem->flags & DMA_MEMORY_EXCLUSIVE);
+ free_pages((unsigned long)vaddr, order);
}
+}
+EXPORT_SYMBOL(dma_free_coherent);
+
+int dma_declare_coherent_memory(struct device *dev, dma_addr_t bus_addr,
+ dma_addr_t device_addr, size_t size, int flags)
+{
+ void __iomem *mem_base = NULL;
+ int pages = size >> PAGE_SHIFT;
+ int bitmap_size = BITS_TO_LONGS(pages) * sizeof(long);
+
+ if ((flags & (DMA_MEMORY_MAP | DMA_MEMORY_IO)) == 0)
+ goto out;
+ if (!size)
+ goto out;
+ if (dev->dma_mem)
+ goto out;
+
+ /* FIXME: this routine just ignores DMA_MEMORY_INCLUDES_CHILDREN */
- return P2SEGADDR(ret);
+ mem_base = ioremap_nocache(bus_addr, size);
+ if (!mem_base)
+ goto out;
+
+ dev->dma_mem = kmalloc(sizeof(struct dma_coherent_mem), GFP_KERNEL);
+ if (!dev->dma_mem)
+ goto out;
+ dev->dma_mem->bitmap = kzalloc(bitmap_size, GFP_KERNEL);
+ if (!dev->dma_mem->bitmap)
+ goto free1_out;
+
+ dev->dma_mem->virt_base = mem_base;
+ dev->dma_mem->device_base = device_addr;
+ dev->dma_mem->size = pages;
+ dev->dma_mem->flags = flags;
+
+ if (flags & DMA_MEMORY_MAP)
+ return DMA_MEMORY_MAP;
+
+ return DMA_MEMORY_IO;
+
+ free1_out:
+ kfree(dev->dma_mem);
+ out:
+ if (mem_base)
+ iounmap(mem_base);
+ return 0;
}
+EXPORT_SYMBOL(dma_declare_coherent_memory);
-void consistent_free(void *vaddr, size_t size)
+void dma_release_declared_memory(struct device *dev)
{
- unsigned long addr = P1SEGADDR((unsigned long)vaddr);
- struct page *page=virt_to_page(addr);
- int num_pages=(size+PAGE_SIZE-1) >> PAGE_SHIFT;
- int i;
+ struct dma_coherent_mem *mem = dev->dma_mem;
- for(i=0;i<num_pages;i++) {
- __free_page((page+i));
- }
+ if (!mem)
+ return;
+ dev->dma_mem = NULL;
+ iounmap(mem->virt_base);
+ kfree(mem->bitmap);
+ kfree(mem);
}
+EXPORT_SYMBOL(dma_release_declared_memory);
-void consistent_sync(void *vaddr, size_t size, int direction)
+void *dma_mark_declared_memory_occupied(struct device *dev,
+ dma_addr_t device_addr, size_t size)
{
- void * p1addr = (void*) P1SEGADDR((unsigned long)vaddr);
+ struct dma_coherent_mem *mem = dev->dma_mem;
+ int pages = (size + (device_addr & ~PAGE_MASK) + PAGE_SIZE - 1) >> PAGE_SHIFT;
+ int pos, err;
+
+ if (!mem)
+ return ERR_PTR(-EINVAL);
+
+ pos = (device_addr - mem->device_base) >> PAGE_SHIFT;
+ err = bitmap_allocate_region(mem->bitmap, pos, get_order(pages));
+ if (err != 0)
+ return ERR_PTR(err);
+ return mem->virt_base + (pos << PAGE_SHIFT);
+}
+EXPORT_SYMBOL(dma_mark_declared_memory_occupied);
+
+void dma_cache_sync(struct device *dev, void *vaddr, size_t size,
+ enum dma_data_direction direction)
+{
+#ifdef CONFIG_CPU_SH5
+ void *p1addr = vaddr;
+#else
+ void *p1addr = (void*) P1SEGADDR((unsigned long)vaddr);
+#endif
switch (direction) {
case DMA_FROM_DEVICE: /* invalidate only */
@@ -80,8 +176,4 @@ void consistent_sync(void *vaddr, size_t size, int direction)
BUG();
}
}
-
-EXPORT_SYMBOL(consistent_alloc);
-EXPORT_SYMBOL(consistent_free);
-EXPORT_SYMBOL(consistent_sync);
-
+EXPORT_SYMBOL(dma_cache_sync);
diff --git a/arch/sh/mm/copy_page.S b/arch/sh/mm/copy_page.S
deleted file mode 100644
index 4068501..0000000
--- a/arch/sh/mm/copy_page.S
+++ /dev/null
@@ -1,388 +0,0 @@
-/*
- * copy_page, __copy_user_page, __copy_user implementation of SuperH
- *
- * Copyright (C) 2001 Niibe Yutaka & Kaz Kojima
- * Copyright (C) 2002 Toshinobu Sugioka
- * Copyright (C) 2006 Paul Mundt
- */
-#include <linux/linkage.h>
-#include <asm/page.h>
-
-/*
- * copy_page_slow
- * @to: P1 address
- * @from: P1 address
- *
- * void copy_page_slow(void *to, void *from)
- */
-
-/*
- * r0, r1, r2, r3, r4, r5, r6, r7 --- scratch
- * r8 --- from + PAGE_SIZE
- * r9 --- not used
- * r10 --- to
- * r11 --- from
- */
-ENTRY(copy_page_slow)
- mov.l r8,@-r15
- mov.l r10,@-r15
- mov.l r11,@-r15
- mov r4,r10
- mov r5,r11
- mov r5,r8
- mov.l .Lpsz,r0
- add r0,r8
- !
-1: mov.l @r11+,r0
- mov.l @r11+,r1
- mov.l @r11+,r2
- mov.l @r11+,r3
- mov.l @r11+,r4
- mov.l @r11+,r5
- mov.l @r11+,r6
- mov.l @r11+,r7
-#if defined(CONFIG_CPU_SH3)
- mov.l r0,@r10
-#elif defined(CONFIG_CPU_SH4)
- movca.l r0,@r10
- mov r10,r0
-#endif
- add #32,r10
- mov.l r7,@-r10
- mov.l r6,@-r10
- mov.l r5,@-r10
- mov.l r4,@-r10
- mov.l r3,@-r10
- mov.l r2,@-r10
- mov.l r1,@-r10
-#if defined(CONFIG_CPU_SH4)
- ocbwb @r0
-#endif
- cmp/eq r11,r8
- bf/s 1b
- add #28,r10
- !
- mov.l @r15+,r11
- mov.l @r15+,r10
- mov.l @r15+,r8
- rts
- nop
-
- .align 2
-.Lpsz: .long PAGE_SIZE
-/*
- * __kernel_size_t __copy_user(void *to, const void *from, __kernel_size_t n);
- * Return the number of bytes NOT copied
- */
-#define EX(...) \
- 9999: __VA_ARGS__ ; \
- .section __ex_table, "a"; \
- .long 9999b, 6000f ; \
- .previous
-ENTRY(__copy_user)
- ! Check if small number of bytes
- mov #11,r0
- mov r4,r3
- cmp/gt r0,r6 ! r6 (len) > r0 (11)
- bf/s .L_cleanup_loop_no_pop
- add r6,r3 ! last destination address
-
- ! Calculate bytes needed to align to src
- mov.l r11,@-r15
- neg r5,r0
- mov.l r10,@-r15
- add #4,r0
- mov.l r9,@-r15
- and #3,r0
- mov.l r8,@-r15
- tst r0,r0
- bt 2f
-
-1:
- ! Copy bytes to long word align src
-EX( mov.b @r5+,r1 )
- dt r0
- add #-1,r6
-EX( mov.b r1,@r4 )
- bf/s 1b
- add #1,r4
-
- ! Jump to appropriate routine depending on dest
-2: mov #3,r1
- mov r6, r2
- and r4,r1
- shlr2 r2
- shll2 r1
- mova .L_jump_tbl,r0
- mov.l @(r0,r1),r1
- jmp @r1
- nop
-
- .align 2
-.L_jump_tbl:
- .long .L_dest00
- .long .L_dest01
- .long .L_dest10
- .long .L_dest11
-
-/*
- * Come here if there are less than 12 bytes to copy
- *
- * Keep the branch target close, so the bf/s callee doesn't overflow
- * and result in a more expensive branch being inserted. This is the
- * fast-path for small copies, the jump via the jump table will hit the
- * default slow-path cleanup. -PFM.
- */
-.L_cleanup_loop_no_pop:
- tst r6,r6 ! Check explicitly for zero
- bt 1f
-
-2:
-EX( mov.b @r5+,r0 )
- dt r6
-EX( mov.b r0,@r4 )
- bf/s 2b
- add #1,r4
-
-1: mov #0,r0 ! normal return
-5000:
-
-# Exception handler:
-.section .fixup, "ax"
-6000:
- mov.l 8000f,r1
- mov r3,r0
- jmp @r1
- sub r4,r0
- .align 2
-8000: .long 5000b
-
-.previous
- rts
- nop
-
-! Destination = 00
-
-.L_dest00:
- ! Skip the large copy for small transfers
- mov #(32+32-4), r0
- cmp/gt r6, r0 ! r0 (60) > r6 (len)
- bt 1f
-
- ! Align dest to a 32 byte boundary
- neg r4,r0
- add #0x20, r0
- and #0x1f, r0
- tst r0, r0
- bt 2f
-
- sub r0, r6
- shlr2 r0
-3:
-EX( mov.l @r5+,r1 )
- dt r0
-EX( mov.l r1,@r4 )
- bf/s 3b
- add #4,r4
-
-2:
-EX( mov.l @r5+,r0 )
-EX( mov.l @r5+,r1 )
-EX( mov.l @r5+,r2 )
-EX( mov.l @r5+,r7 )
-EX( mov.l @r5+,r8 )
-EX( mov.l @r5+,r9 )
-EX( mov.l @r5+,r10 )
-EX( mov.l @r5+,r11 )
-#ifdef CONFIG_CPU_SH4
-EX( movca.l r0,@r4 )
-#else
-EX( mov.l r0,@r4 )
-#endif
- add #-32, r6
-EX( mov.l r1,@(4,r4) )
- mov #32, r0
-EX( mov.l r2,@(8,r4) )
- cmp/gt r6, r0 ! r0 (32) > r6 (len)
-EX( mov.l r7,@(12,r4) )
-EX( mov.l r8,@(16,r4) )
-EX( mov.l r9,@(20,r4) )
-EX( mov.l r10,@(24,r4) )
-EX( mov.l r11,@(28,r4) )
- bf/s 2b
- add #32,r4
-
-1: mov r6, r0
- shlr2 r0
- tst r0, r0
- bt .L_cleanup
-1:
-EX( mov.l @r5+,r1 )
- dt r0
-EX( mov.l r1,@r4 )
- bf/s 1b
- add #4,r4
-
- bra .L_cleanup
- nop
-
-! Destination = 10
-
-.L_dest10:
- mov r2,r7
- shlr2 r7
- shlr r7
- tst r7,r7
- mov #7,r0
- bt/s 1f
- and r0,r2
-2:
- dt r7
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
-EX( mov.l @r5+,r0 )
-EX( mov.l @r5+,r1 )
-EX( mov.l @r5+,r8 )
-EX( mov.l @r5+,r9 )
-EX( mov.l @r5+,r10 )
-EX( mov.w r0,@r4 )
- add #2,r4
- xtrct r1,r0
- xtrct r8,r1
- xtrct r9,r8
- xtrct r10,r9
-
-EX( mov.l r0,@r4 )
-EX( mov.l r1,@(4,r4) )
-EX( mov.l r8,@(8,r4) )
-EX( mov.l r9,@(12,r4) )
-
-EX( mov.l @r5+,r1 )
-EX( mov.l @r5+,r8 )
-EX( mov.l @r5+,r0 )
- xtrct r1,r10
- xtrct r8,r1
- xtrct r0,r8
- shlr16 r0
-EX( mov.l r10,@(16,r4) )
-EX( mov.l r1,@(20,r4) )
-EX( mov.l r8,@(24,r4) )
-EX( mov.w r0,@(28,r4) )
- bf/s 2b
- add #30,r4
-#else
-EX( mov.l @(28,r5),r0 )
-EX( mov.l @(24,r5),r8 )
-EX( mov.l @(20,r5),r9 )
-EX( mov.l @(16,r5),r10 )
-EX( mov.w r0,@(30,r4) )
- add #-2,r4
- xtrct r8,r0
- xtrct r9,r8
- xtrct r10,r9
-EX( mov.l r0,@(28,r4) )
-EX( mov.l r8,@(24,r4) )
-EX( mov.l r9,@(20,r4) )
-
-EX( mov.l @(12,r5),r0 )
-EX( mov.l @(8,r5),r8 )
- xtrct r0,r10
-EX( mov.l @(4,r5),r9 )
- mov.l r10,@(16,r4)
-EX( mov.l @r5,r10 )
- xtrct r8,r0
- xtrct r9,r8
- xtrct r10,r9
-EX( mov.l r0,@(12,r4) )
-EX( mov.l r8,@(8,r4) )
- swap.w r10,r0
-EX( mov.l r9,@(4,r4) )
-EX( mov.w r0,@(2,r4) )
-
- add #32,r5
- bf/s 2b
- add #34,r4
-#endif
- tst r2,r2
- bt .L_cleanup
-
-1: ! Read longword, write two words per iteration
-EX( mov.l @r5+,r0 )
- dt r2
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
-EX( mov.w r0,@r4 )
- shlr16 r0
-EX( mov.w r0,@(2,r4) )
-#else
-EX( mov.w r0,@(2,r4) )
- shlr16 r0
-EX( mov.w r0,@r4 )
-#endif
- bf/s 1b
- add #4,r4
-
- bra .L_cleanup
- nop
-
-! Destination = 01 or 11
-
-.L_dest01:
-.L_dest11:
- ! Read longword, write byte, word, byte per iteration
-EX( mov.l @r5+,r0 )
- dt r2
-#ifdef CONFIG_CPU_LITTLE_ENDIAN
-EX( mov.b r0,@r4 )
- shlr8 r0
- add #1,r4
-EX( mov.w r0,@r4 )
- shlr16 r0
-EX( mov.b r0,@(2,r4) )
- bf/s .L_dest01
- add #3,r4
-#else
-EX( mov.b r0,@(3,r4) )
- shlr8 r0
- swap.w r0,r7
-EX( mov.b r7,@r4 )
- add #1,r4
-EX( mov.w r0,@r4 )
- bf/s .L_dest01
- add #3,r4
-#endif
-
-! Cleanup last few bytes
-.L_cleanup:
- mov r6,r0
- and #3,r0
- tst r0,r0
- bt .L_exit
- mov r0,r6
-
-.L_cleanup_loop:
-EX( mov.b @r5+,r0 )
- dt r6
-EX( mov.b r0,@r4 )
- bf/s .L_cleanup_loop
- add #1,r4
-
-.L_exit:
- mov #0,r0 ! normal return
-
-5000:
-
-# Exception handler:
-.section .fixup, "ax"
-6000:
- mov.l 8000f,r1
- mov r3,r0
- jmp @r1
- sub r4,r0
- .align 2
-8000: .long 5000b
-
-.previous
- mov.l @r15+,r8
- mov.l @r15+,r9
- mov.l @r15+,r10
- rts
- mov.l @r15+,r11
diff --git a/arch/sh/mm/extable.c b/arch/sh/mm/extable_32.c
index c1cf446..c1cf446 100644
--- a/arch/sh/mm/extable.c
+++ b/arch/sh/mm/extable_32.c
diff --git a/arch/sh/mm/extable_64.c b/arch/sh/mm/extable_64.c
new file mode 100644
index 0000000..f054996
--- /dev/null
+++ b/arch/sh/mm/extable_64.c
@@ -0,0 +1,82 @@
+/*
+ * arch/sh/mm/extable_64.c
+ *
+ * Copyright (C) 2003 Richard Curnow
+ * Copyright (C) 2003, 2004 Paul Mundt
+ *
+ * Cloned from the 2.5 SH version..
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/rwsem.h>
+#include <linux/module.h>
+#include <asm/uaccess.h>
+
+extern unsigned long copy_user_memcpy, copy_user_memcpy_end;
+extern void __copy_user_fixup(void);
+
+static const struct exception_table_entry __copy_user_fixup_ex = {
+ .fixup = (unsigned long)&__copy_user_fixup,
+};
+
+/*
+ * Some functions that may trap due to a bad user-mode address have too
+ * many loads and stores in them to make it at all practical to label
+ * each one and put them all in the main exception table.
+ *
+ * In particular, the fast memcpy routine is like this. It's fix-up is
+ * just to fall back to a slow byte-at-a-time copy, which is handled the
+ * conventional way. So it's functionally OK to just handle any trap
+ * occurring in the fast memcpy with that fixup.
+ */
+static const struct exception_table_entry *check_exception_ranges(unsigned long addr)
+{
+ if ((addr >= (unsigned long)&copy_user_memcpy) &&
+ (addr <= (unsigned long)&copy_user_memcpy_end))
+ return &__copy_user_fixup_ex;
+
+ return NULL;
+}
+
+/* Simple binary search */
+const struct exception_table_entry *
+search_extable(const struct exception_table_entry *first,
+ const struct exception_table_entry *last,
+ unsigned long value)
+{
+ const struct exception_table_entry *mid;
+
+ mid = check_exception_ranges(value);
+ if (mid)
+ return mid;
+
+ while (first <= last) {
+ long diff;
+
+ mid = (last - first) / 2 + first;
+ diff = mid->insn - value;
+ if (diff == 0)
+ return mid;
+ else if (diff < 0)
+ first = mid+1;
+ else
+ last = mid-1;
+ }
+
+ return NULL;
+}
+
+int fixup_exception(struct pt_regs *regs)
+{
+ const struct exception_table_entry *fixup;
+
+ fixup = search_exception_tables(regs->pc);
+ if (fixup) {
+ regs->pc = fixup->fixup;
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/arch/sh/mm/fault.c b/arch/sh/mm/fault_32.c
index 60d74f7..33b43d2 100644
--- a/arch/sh/mm/fault.c
+++ b/arch/sh/mm/fault_32.c
@@ -172,7 +172,7 @@ no_context:
bust_spinlocks(1);
if (oops_may_print()) {
- __typeof__(pte_val(__pte(0))) page;
+ unsigned long page;
if (address < PAGE_SIZE)
printk(KERN_ALERT "Unable to handle kernel NULL "
diff --git a/arch/sh/mm/fault_64.c b/arch/sh/mm/fault_64.c
new file mode 100644
index 0000000..399d537
--- /dev/null
+++ b/arch/sh/mm/fault_64.c
@@ -0,0 +1,275 @@
+/*
+ * The SH64 TLB miss.
+ *
+ * Original code from fault.c
+ * Copyright (C) 2000, 2001 Paolo Alberelli
+ *
+ * Fast PTE->TLB refill path
+ * Copyright (C) 2003 Richard.Curnow@superh.com
+ *
+ * IMPORTANT NOTES :
+ * The do_fast_page_fault function is called from a context in entry.S
+ * where very few registers have been saved. In particular, the code in
+ * this file must be compiled not to use ANY caller-save registers that
+ * are not part of the restricted save set. Also, it means that code in
+ * this file must not make calls to functions elsewhere in the kernel, or
+ * else the excepting context will see corruption in its caller-save
+ * registers. Plus, the entry.S save area is non-reentrant, so this code
+ * has to run with SR.BL==1, i.e. no interrupts taken inside it and panic
+ * on any exception.
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/signal.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/ptrace.h>
+#include <linux/mman.h>
+#include <linux/mm.h>
+#include <linux/smp.h>
+#include <linux/interrupt.h>
+#include <asm/system.h>
+#include <asm/tlb.h>
+#include <asm/io.h>
+#include <asm/uaccess.h>
+#include <asm/pgalloc.h>
+#include <asm/mmu_context.h>
+#include <asm/cpu/registers.h>
+
+/* Callable from fault.c, so not static */
+inline void __do_tlb_refill(unsigned long address,
+ unsigned long long is_text_not_data, pte_t *pte)
+{
+ unsigned long long ptel;
+ unsigned long long pteh=0;
+ struct tlb_info *tlbp;
+ unsigned long long next;
+
+ /* Get PTEL first */
+ ptel = pte_val(*pte);
+
+ /*
+ * Set PTEH register
+ */
+ pteh = address & MMU_VPN_MASK;
+
+ /* Sign extend based on neff. */
+#if (NEFF == 32)
+ /* Faster sign extension */
+ pteh = (unsigned long long)(signed long long)(signed long)pteh;
+#else
+ /* General case */
+ pteh = (pteh & NEFF_SIGN) ? (pteh | NEFF_MASK) : pteh;
+#endif
+
+ /* Set the ASID. */
+ pteh |= get_asid() << PTEH_ASID_SHIFT;
+ pteh |= PTEH_VALID;
+
+ /* Set PTEL register, set_pte has performed the sign extension */
+ ptel &= _PAGE_FLAGS_HARDWARE_MASK; /* drop software flags */
+
+ tlbp = is_text_not_data ? &(cpu_data->itlb) : &(cpu_data->dtlb);
+ next = tlbp->next;
+ __flush_tlb_slot(next);
+ asm volatile ("putcfg %0,1,%2\n\n\t"
+ "putcfg %0,0,%1\n"
+ : : "r" (next), "r" (pteh), "r" (ptel) );
+
+ next += TLB_STEP;
+ if (next > tlbp->last) next = tlbp->first;
+ tlbp->next = next;
+
+}
+
+static int handle_vmalloc_fault(struct mm_struct *mm,
+ unsigned long protection_flags,
+ unsigned long long textaccess,
+ unsigned long address)
+{
+ pgd_t *dir;
+ pud_t *pud;
+ pmd_t *pmd;
+ static pte_t *pte;
+ pte_t entry;
+
+ dir = pgd_offset_k(address);
+
+ pud = pud_offset(dir, address);
+ if (pud_none_or_clear_bad(pud))
+ return 0;
+
+ pmd = pmd_offset(pud, address);
+ if (pmd_none_or_clear_bad(pmd))
+ return 0;
+
+ pte = pte_offset_kernel(pmd, address);
+ entry = *pte;
+
+ if (pte_none(entry) || !pte_present(entry))
+ return 0;
+ if ((pte_val(entry) & protection_flags) != protection_flags)
+ return 0;
+
+ __do_tlb_refill(address, textaccess, pte);
+
+ return 1;
+}
+
+static int handle_tlbmiss(struct mm_struct *mm,
+ unsigned long long protection_flags,
+ unsigned long long textaccess,
+ unsigned long address)
+{
+ pgd_t *dir;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+ pte_t entry;
+
+ /* NB. The PGD currently only contains a single entry - there is no
+ page table tree stored for the top half of the address space since
+ virtual pages in that region should never be mapped in user mode.
+ (In kernel mode, the only things in that region are the 512Mb super
+ page (locked in), and vmalloc (modules) + I/O device pages (handled
+ by handle_vmalloc_fault), so no PGD for the upper half is required
+ by kernel mode either).
+
+ See how mm->pgd is allocated and initialised in pgd_alloc to see why
+ the next test is necessary. - RPC */
+ if (address >= (unsigned long) TASK_SIZE)
+ /* upper half - never has page table entries. */
+ return 0;
+
+ dir = pgd_offset(mm, address);
+ if (pgd_none(*dir) || !pgd_present(*dir))
+ return 0;
+ if (!pgd_present(*dir))
+ return 0;
+
+ pud = pud_offset(dir, address);
+ if (pud_none(*pud) || !pud_present(*pud))
+ return 0;
+
+ pmd = pmd_offset(pud, address);
+ if (pmd_none(*pmd) || !pmd_present(*pmd))
+ return 0;
+
+ pte = pte_offset_kernel(pmd, address);
+ entry = *pte;
+
+ if (pte_none(entry) || !pte_present(entry))
+ return 0;
+
+ /*
+ * If the page doesn't have sufficient protection bits set to
+ * service the kind of fault being handled, there's not much
+ * point doing the TLB refill. Punt the fault to the general
+ * handler.
+ */
+ if ((pte_val(entry) & protection_flags) != protection_flags)
+ return 0;
+
+ __do_tlb_refill(address, textaccess, pte);
+
+ return 1;
+}
+
+/*
+ * Put all this information into one structure so that everything is just
+ * arithmetic relative to a single base address. This reduces the number
+ * of movi/shori pairs needed just to load addresses of static data.
+ */
+struct expevt_lookup {
+ unsigned short protection_flags[8];
+ unsigned char is_text_access[8];
+ unsigned char is_write_access[8];
+};
+
+#define PRU (1<<9)
+#define PRW (1<<8)
+#define PRX (1<<7)
+#define PRR (1<<6)
+
+#define DIRTY (_PAGE_DIRTY | _PAGE_ACCESSED)
+#define YOUNG (_PAGE_ACCESSED)
+
+/* Sized as 8 rather than 4 to allow checking the PTE's PRU bit against whether
+ the fault happened in user mode or privileged mode. */
+static struct expevt_lookup expevt_lookup_table = {
+ .protection_flags = {PRX, PRX, 0, 0, PRR, PRR, PRW, PRW},
+ .is_text_access = {1, 1, 0, 0, 0, 0, 0, 0}
+};
+
+/*
+ This routine handles page faults that can be serviced just by refilling a
+ TLB entry from an existing page table entry. (This case represents a very
+ large majority of page faults.) Return 1 if the fault was successfully
+ handled. Return 0 if the fault could not be handled. (This leads into the
+ general fault handling in fault.c which deals with mapping file-backed
+ pages, stack growth, segmentation faults, swapping etc etc)
+ */
+asmlinkage int do_fast_page_fault(unsigned long long ssr_md,
+ unsigned long long expevt,
+ unsigned long address)
+{
+ struct task_struct *tsk;
+ struct mm_struct *mm;
+ unsigned long long textaccess;
+ unsigned long long protection_flags;
+ unsigned long long index;
+ unsigned long long expevt4;
+
+ /* The next few lines implement a way of hashing EXPEVT into a
+ * small array index which can be used to lookup parameters
+ * specific to the type of TLBMISS being handled.
+ *
+ * Note:
+ * ITLBMISS has EXPEVT==0xa40
+ * RTLBMISS has EXPEVT==0x040
+ * WTLBMISS has EXPEVT==0x060
+ */
+ expevt4 = (expevt >> 4);
+ /* TODO : xor ssr_md into this expression too. Then we can check
+ * that PRU is set when it needs to be. */
+ index = expevt4 ^ (expevt4 >> 5);
+ index &= 7;
+ protection_flags = expevt_lookup_table.protection_flags[index];
+ textaccess = expevt_lookup_table.is_text_access[index];
+
+ /* SIM
+ * Note this is now called with interrupts still disabled
+ * This is to cope with being called for a missing IO port
+ * address with interrupts disabled. This should be fixed as
+ * soon as we have a better 'fast path' miss handler.
+ *
+ * Plus take care how you try and debug this stuff.
+ * For example, writing debug data to a port which you
+ * have just faulted on is not going to work.
+ */
+
+ tsk = current;
+ mm = tsk->mm;
+
+ if ((address >= VMALLOC_START && address < VMALLOC_END) ||
+ (address >= IOBASE_VADDR && address < IOBASE_END)) {
+ if (ssr_md)
+ /*
+ * Process-contexts can never have this address
+ * range mapped
+ */
+ if (handle_vmalloc_fault(mm, protection_flags,
+ textaccess, address))
+ return 1;
+ } else if (!in_interrupt() && mm) {
+ if (handle_tlbmiss(mm, protection_flags, textaccess, address))
+ return 1;
+ }
+
+ return 0;
+}
diff --git a/arch/sh/mm/init.c b/arch/sh/mm/init.c
index d5e160d..2918c6b 100644
--- a/arch/sh/mm/init.c
+++ b/arch/sh/mm/init.c
@@ -23,9 +23,7 @@
DEFINE_PER_CPU(struct mmu_gather, mmu_gathers);
pgd_t swapper_pg_dir[PTRS_PER_PGD];
-
-void (*copy_page)(void *from, void *to);
-void (*clear_page)(void *to);
+unsigned long cached_to_uncached = 0;
void show_mem(void)
{
@@ -102,7 +100,8 @@ static void set_pte_phys(unsigned long addr, unsigned long phys, pgprot_t prot)
set_pte(pte, pfn_pte(phys >> PAGE_SHIFT, prot));
- flush_tlb_one(get_asid(), addr);
+ if (cached_to_uncached)
+ flush_tlb_one(get_asid(), addr);
}
/*
@@ -131,6 +130,37 @@ void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
set_pte_phys(address, phys, prot);
}
+
+void __init page_table_range_init(unsigned long start, unsigned long end,
+ pgd_t *pgd_base)
+{
+ pgd_t *pgd;
+ pud_t *pud;
+ pmd_t *pmd;
+ int pgd_idx;
+ unsigned long vaddr;
+
+ vaddr = start & PMD_MASK;
+ end = (end + PMD_SIZE - 1) & PMD_MASK;
+ pgd_idx = pgd_index(vaddr);
+ pgd = pgd_base + pgd_idx;
+
+ for ( ; (pgd_idx < PTRS_PER_PGD) && (vaddr != end); pgd++, pgd_idx++) {
+ BUG_ON(pgd_none(*pgd));
+ pud = pud_offset(pgd, 0);
+ BUG_ON(pud_none(*pud));
+ pmd = pmd_offset(pud, 0);
+
+ if (!pmd_present(*pmd)) {
+ pte_t *pte_table;
+ pte_table = (pte_t *)alloc_bootmem_low_pages(PAGE_SIZE);
+ memset(pte_table, 0, PAGE_SIZE);
+ pmd_populate_kernel(&init_mm, pmd, pte_table);
+ }
+
+ vaddr += PMD_SIZE;
+ }
+}
#endif /* CONFIG_MMU */
/*
@@ -150,6 +180,11 @@ void __init paging_init(void)
* check for a null value. */
set_TTB(swapper_pg_dir);
+ /* Populate the relevant portions of swapper_pg_dir so that
+ * we can use the fixmap entries without calling kmalloc.
+ * pte's will be filled in by __set_fixmap(). */
+ page_table_range_init(FIXADDR_START, FIXADDR_TOP, swapper_pg_dir);
+
memset(max_zone_pfns, 0, sizeof(max_zone_pfns));
for_each_online_node(nid) {
@@ -167,9 +202,22 @@ void __init paging_init(void)
}
free_area_init_nodes(max_zone_pfns);
+
+ /* Set up the uncached fixmap */
+ set_fixmap_nocache(FIX_UNCACHED, __pa(&__uncached_start));
+
+#ifdef CONFIG_29BIT
+ /*
+ * Handle trivial transitions between cached and uncached
+ * segments, making use of the 1:1 mapping relationship in
+ * 512MB lowmem.
+ */
+ cached_to_uncached = P2SEG - P1SEG;
+#endif
}
static struct kcore_list kcore_mem, kcore_vmalloc;
+int after_bootmem = 0;
void __init mem_init(void)
{
@@ -202,17 +250,7 @@ void __init mem_init(void)
memset(empty_zero_page, 0, PAGE_SIZE);
__flush_wback_region(empty_zero_page, PAGE_SIZE);
- /*
- * Setup wrappers for copy/clear_page(), these will get overridden
- * later in the boot process if a better method is available.
- */
-#ifdef CONFIG_MMU
- copy_page = copy_page_slow;
- clear_page = clear_page_slow;
-#else
- copy_page = copy_page_nommu;
- clear_page = clear_page_nommu;
-#endif
+ after_bootmem = 1;
codesize = (unsigned long) &_etext - (unsigned long) &_text;
datasize = (unsigned long) &_edata - (unsigned long) &_etext;
diff --git a/arch/sh/mm/ioremap.c b/arch/sh/mm/ioremap_32.c
index 0c7b7e3..0c7b7e3 100644
--- a/arch/sh/mm/ioremap.c
+++ b/arch/sh/mm/ioremap_32.c
diff --git a/arch/sh/mm/ioremap_64.c b/arch/sh/mm/ioremap_64.c
new file mode 100644
index 0000000..e27d165
--- /dev/null
+++ b/arch/sh/mm/ioremap_64.c
@@ -0,0 +1,404 @@
+/*
+ * arch/sh/mm/ioremap_64.c
+ *
+ * Copyright (C) 2000, 2001 Paolo Alberelli
+ * Copyright (C) 2003 - 2007 Paul Mundt
+ *
+ * Mostly derived from arch/sh/mm/ioremap.c which, in turn is mostly
+ * derived from arch/i386/mm/ioremap.c .
+ *
+ * (C) Copyright 1995 1996 Linus Torvalds
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/vmalloc.h>
+#include <linux/ioport.h>
+#include <linux/module.h>
+#include <linux/mm.h>
+#include <linux/io.h>
+#include <linux/bootmem.h>
+#include <linux/proc_fs.h>
+#include <asm/page.h>
+#include <asm/pgalloc.h>
+#include <asm/addrspace.h>
+#include <asm/cacheflush.h>
+#include <asm/tlbflush.h>
+#include <asm/mmu.h>
+
+static void shmedia_mapioaddr(unsigned long, unsigned long);
+static unsigned long shmedia_ioremap(struct resource *, u32, int);
+
+/*
+ * Generic mapping function (not visible outside):
+ */
+
+/*
+ * Remap an arbitrary physical address space into the kernel virtual
+ * address space. Needed when the kernel wants to access high addresses
+ * directly.
+ *
+ * NOTE! We need to allow non-page-aligned mappings too: we will obviously
+ * have to convert them into an offset in a page-aligned mapping, but the
+ * caller shouldn't need to know that small detail.
+ */
+void *__ioremap(unsigned long phys_addr, unsigned long size,
+ unsigned long flags)
+{
+ void * addr;
+ struct vm_struct * area;
+ unsigned long offset, last_addr;
+ pgprot_t pgprot;
+
+ /* Don't allow wraparound or zero size */
+ last_addr = phys_addr + size - 1;
+ if (!size || last_addr < phys_addr)
+ return NULL;
+
+ pgprot = __pgprot(_PAGE_PRESENT | _PAGE_READ |
+ _PAGE_WRITE | _PAGE_DIRTY |
+ _PAGE_ACCESSED | _PAGE_SHARED | flags);
+
+ /*
+ * Mappings have to be page-aligned
+ */
+ offset = phys_addr & ~PAGE_MASK;
+ phys_addr &= PAGE_MASK;
+ size = PAGE_ALIGN(last_addr + 1) - phys_addr;
+
+ /*
+ * Ok, go for it..
+ */
+ area = get_vm_area(size, VM_IOREMAP);
+ pr_debug("Get vm_area returns %p addr %p\n",area,area->addr);
+ if (!area)
+ return NULL;
+ area->phys_addr = phys_addr;
+ addr = area->addr;
+ if (ioremap_page_range((unsigned long)addr, (unsigned long)addr + size,
+ phys_addr, pgprot)) {
+ vunmap(addr);
+ return NULL;
+ }
+ return (void *) (offset + (char *)addr);
+}
+EXPORT_SYMBOL(__ioremap);
+
+void __iounmap(void *addr)
+{
+ struct vm_struct *area;
+
+ vfree((void *) (PAGE_MASK & (unsigned long) addr));
+ area = remove_vm_area((void *) (PAGE_MASK & (unsigned long) addr));
+ if (!area) {
+ printk(KERN_ERR "iounmap: bad address %p\n", addr);
+ return;
+ }
+
+ kfree(area);
+}
+EXPORT_SYMBOL(__iounmap);
+
+static struct resource shmedia_iomap = {
+ .name = "shmedia_iomap",
+ .start = IOBASE_VADDR + PAGE_SIZE,
+ .end = IOBASE_END - 1,
+};
+
+static void shmedia_mapioaddr(unsigned long pa, unsigned long va);
+static void shmedia_unmapioaddr(unsigned long vaddr);
+static unsigned long shmedia_ioremap(struct resource *res, u32 pa, int sz);
+
+/*
+ * We have the same problem as the SPARC, so lets have the same comment:
+ * Our mini-allocator...
+ * Boy this is gross! We need it because we must map I/O for
+ * timers and interrupt controller before the kmalloc is available.
+ */
+
+#define XNMLN 15
+#define XNRES 10
+
+struct xresource {
+ struct resource xres; /* Must be first */
+ int xflag; /* 1 == used */
+ char xname[XNMLN+1];
+};
+
+static struct xresource xresv[XNRES];
+
+static struct xresource *xres_alloc(void)
+{
+ struct xresource *xrp;
+ int n;
+
+ xrp = xresv;
+ for (n = 0; n < XNRES; n++) {
+ if (xrp->xflag == 0) {
+ xrp->xflag = 1;
+ return xrp;
+ }
+ xrp++;
+ }
+ return NULL;
+}
+
+static void xres_free(struct xresource *xrp)
+{
+ xrp->xflag = 0;
+}
+
+static struct resource *shmedia_find_resource(struct resource *root,
+ unsigned long vaddr)
+{
+ struct resource *res;
+
+ for (res = root->child; res; res = res->sibling)
+ if (res->start <= vaddr && res->end >= vaddr)
+ return res;
+
+ return NULL;
+}
+
+static unsigned long shmedia_alloc_io(unsigned long phys, unsigned long size,
+ const char *name)
+{
+ static int printed_full = 0;
+ struct xresource *xres;
+ struct resource *res;
+ char *tack;
+ int tlen;
+
+ if (name == NULL) name = "???";
+
+ if ((xres = xres_alloc()) != 0) {
+ tack = xres->xname;
+ res = &xres->xres;
+ } else {
+ if (!printed_full) {
+ printk("%s: done with statics, switching to kmalloc\n",
+ __FUNCTION__);
+ printed_full = 1;
+ }
+ tlen = strlen(name);
+ tack = kmalloc(sizeof (struct resource) + tlen + 1, GFP_KERNEL);
+ if (!tack)
+ return -ENOMEM;
+ memset(tack, 0, sizeof(struct resource));
+ res = (struct resource *) tack;
+ tack += sizeof (struct resource);
+ }
+
+ strncpy(tack, name, XNMLN);
+ tack[XNMLN] = 0;
+ res->name = tack;
+
+ return shmedia_ioremap(res, phys, size);
+}
+
+static unsigned long shmedia_ioremap(struct resource *res, u32 pa, int sz)
+{
+ unsigned long offset = ((unsigned long) pa) & (~PAGE_MASK);
+ unsigned long round_sz = (offset + sz + PAGE_SIZE-1) & PAGE_MASK;
+ unsigned long va;
+ unsigned int psz;
+
+ if (allocate_resource(&shmedia_iomap, res, round_sz,
+ shmedia_iomap.start, shmedia_iomap.end,
+ PAGE_SIZE, NULL, NULL) != 0) {
+ panic("alloc_io_res(%s): cannot occupy\n",
+ (res->name != NULL)? res->name: "???");
+ }
+
+ va = res->start;
+ pa &= PAGE_MASK;
+
+ psz = (res->end - res->start + (PAGE_SIZE - 1)) / PAGE_SIZE;
+
+ /* log at boot time ... */
+ printk("mapioaddr: %6s [%2d page%s] va 0x%08lx pa 0x%08x\n",
+ ((res->name != NULL) ? res->name : "???"),
+ psz, psz == 1 ? " " : "s", va, pa);
+
+ for (psz = res->end - res->start + 1; psz != 0; psz -= PAGE_SIZE) {
+ shmedia_mapioaddr(pa, va);
+ va += PAGE_SIZE;
+ pa += PAGE_SIZE;
+ }
+
+ res->start += offset;
+ res->end = res->start + sz - 1; /* not strictly necessary.. */
+
+ return res->start;
+}
+
+static void shmedia_free_io(struct resource *res)
+{
+ unsigned long len = res->end - res->start + 1;
+
+ BUG_ON((len & (PAGE_SIZE - 1)) != 0);
+
+ while (len) {
+ len -= PAGE_SIZE;
+ shmedia_unmapioaddr(res->start + len);
+ }
+
+ release_resource(res);
+}
+
+static __init_refok void *sh64_get_page(void)
+{
+ extern int after_bootmem;
+ void *page;
+
+ if (after_bootmem) {
+ page = (void *)get_zeroed_page(GFP_ATOMIC);
+ } else {
+ page = alloc_bootmem_pages(PAGE_SIZE);
+ }
+
+ if (!page || ((unsigned long)page & ~PAGE_MASK))
+ panic("sh64_get_page: Out of memory already?\n");
+
+ return page;
+}
+
+static void shmedia_mapioaddr(unsigned long pa, unsigned long va)
+{
+ pgd_t *pgdp;
+ pud_t *pudp;
+ pmd_t *pmdp;
+ pte_t *ptep, pte;
+ pgprot_t prot;
+ unsigned long flags = 1; /* 1 = CB0-1 device */
+
+ pr_debug("shmedia_mapiopage pa %08lx va %08lx\n", pa, va);
+
+ pgdp = pgd_offset_k(va);
+ if (pgd_none(*pgdp) || !pgd_present(*pgdp)) {
+ pudp = (pud_t *)sh64_get_page();
+ set_pgd(pgdp, __pgd((unsigned long)pudp | _KERNPG_TABLE));
+ }
+
+ pudp = pud_offset(pgdp, va);
+ if (pud_none(*pudp) || !pud_present(*pudp)) {
+ pmdp = (pmd_t *)sh64_get_page();
+ set_pud(pudp, __pud((unsigned long)pmdp | _KERNPG_TABLE));
+ }
+
+ pmdp = pmd_offset(pudp, va);
+ if (pmd_none(*pmdp) || !pmd_present(*pmdp) ) {
+ ptep = (pte_t *)sh64_get_page();
+ set_pmd(pmdp, __pmd((unsigned long)ptep + _PAGE_TABLE));
+ }
+
+ prot = __pgprot(_PAGE_PRESENT | _PAGE_READ | _PAGE_WRITE |
+ _PAGE_DIRTY | _PAGE_ACCESSED | _PAGE_SHARED | flags);
+
+ pte = pfn_pte(pa >> PAGE_SHIFT, prot);
+ ptep = pte_offset_kernel(pmdp, va);
+
+ if (!pte_none(*ptep) &&
+ pte_val(*ptep) != pte_val(pte))
+ pte_ERROR(*ptep);
+
+ set_pte(ptep, pte);
+
+ flush_tlb_kernel_range(va, PAGE_SIZE);
+}
+
+static void shmedia_unmapioaddr(unsigned long vaddr)
+{
+ pgd_t *pgdp;
+ pud_t *pudp;
+ pmd_t *pmdp;
+ pte_t *ptep;
+
+ pgdp = pgd_offset_k(vaddr);
+ if (pgd_none(*pgdp) || pgd_bad(*pgdp))
+ return;
+
+ pudp = pud_offset(pgdp, vaddr);
+ if (pud_none(*pudp) || pud_bad(*pudp))
+ return;
+
+ pmdp = pmd_offset(pudp, vaddr);
+ if (pmd_none(*pmdp) || pmd_bad(*pmdp))
+ return;
+
+ ptep = pte_offset_kernel(pmdp, vaddr);
+
+ if (pte_none(*ptep) || !pte_present(*ptep))
+ return;
+
+ clear_page((void *)ptep);
+ pte_clear(&init_mm, vaddr, ptep);
+}
+
+unsigned long onchip_remap(unsigned long phys, unsigned long size, const char *name)
+{
+ if (size < PAGE_SIZE)
+ size = PAGE_SIZE;
+
+ return shmedia_alloc_io(phys, size, name);
+}
+
+void onchip_unmap(unsigned long vaddr)
+{
+ struct resource *res;
+ unsigned int psz;
+
+ res = shmedia_find_resource(&shmedia_iomap, vaddr);
+ if (!res) {
+ printk(KERN_ERR "%s: Failed to free 0x%08lx\n",
+ __FUNCTION__, vaddr);
+ return;
+ }
+
+ psz = (res->end - res->start + (PAGE_SIZE - 1)) / PAGE_SIZE;
+
+ printk(KERN_DEBUG "unmapioaddr: %6s [%2d page%s] freed\n",
+ res->name, psz, psz == 1 ? " " : "s");
+
+ shmedia_free_io(res);
+
+ if ((char *)res >= (char *)xresv &&
+ (char *)res < (char *)&xresv[XNRES]) {
+ xres_free((struct xresource *)res);
+ } else {
+ kfree(res);
+ }
+}
+
+#ifdef CONFIG_PROC_FS
+static int
+ioremap_proc_info(char *buf, char **start, off_t fpos, int length, int *eof,
+ void *data)
+{
+ char *p = buf, *e = buf + length;
+ struct resource *r;
+ const char *nm;
+
+ for (r = ((struct resource *)data)->child; r != NULL; r = r->sibling) {
+ if (p + 32 >= e) /* Better than nothing */
+ break;
+ if ((nm = r->name) == 0) nm = "???";
+ p += sprintf(p, "%08lx-%08lx: %s\n",
+ (unsigned long)r->start,
+ (unsigned long)r->end, nm);
+ }
+
+ return p-buf;
+}
+#endif /* CONFIG_PROC_FS */
+
+static int __init register_proc_onchip(void)
+{
+#ifdef CONFIG_PROC_FS
+ create_proc_read_entry("io_map",0,0, ioremap_proc_info, &shmedia_iomap);
+#endif
+ return 0;
+}
+
+__initcall(register_proc_onchip);
diff --git a/arch/sh/mm/pg-nommu.c b/arch/sh/mm/pg-nommu.c
index d15221b..677dd57 100644
--- a/arch/sh/mm/pg-nommu.c
+++ b/arch/sh/mm/pg-nommu.c
@@ -14,12 +14,12 @@
#include <linux/string.h>
#include <asm/page.h>
-void copy_page_nommu(void *to, void *from)
+void copy_page(void *to, void *from)
{
memcpy(to, from, PAGE_SIZE);
}
-void clear_page_nommu(void *to)
+void clear_page(void *to)
{
memset(to, 0, PAGE_SIZE);
}
diff --git a/arch/sh/mm/pmb.c b/arch/sh/mm/pmb.c
index 1d45b82..ab81c60 100644
--- a/arch/sh/mm/pmb.c
+++ b/arch/sh/mm/pmb.c
@@ -27,6 +27,7 @@
#include <asm/pgtable.h>
#include <asm/mmu.h>
#include <asm/io.h>
+#include <asm/mmu_context.h>
#define NR_PMB_ENTRIES 16
@@ -162,18 +163,18 @@ repeat:
return 0;
}
-int set_pmb_entry(struct pmb_entry *pmbe)
+int __uses_jump_to_uncached set_pmb_entry(struct pmb_entry *pmbe)
{
int ret;
- jump_to_P2();
+ jump_to_uncached();
ret = __set_pmb_entry(pmbe->vpn, pmbe->ppn, pmbe->flags, &pmbe->entry);
- back_to_P1();
+ back_to_cached();
return ret;
}
-void clear_pmb_entry(struct pmb_entry *pmbe)
+void __uses_jump_to_uncached clear_pmb_entry(struct pmb_entry *pmbe)
{
unsigned int entry = pmbe->entry;
unsigned long addr;
@@ -187,7 +188,7 @@ void clear_pmb_entry(struct pmb_entry *pmbe)
entry >= NR_PMB_ENTRIES))
return;
- jump_to_P2();
+ jump_to_uncached();
/* Clear V-bit */
addr = mk_pmb_addr(entry);
@@ -196,7 +197,7 @@ void clear_pmb_entry(struct pmb_entry *pmbe)
addr = mk_pmb_data(entry);
ctrl_outl(ctrl_inl(addr) & ~PMB_V, addr);
- back_to_P1();
+ back_to_cached();
clear_bit(entry, &pmb_map);
}
@@ -301,17 +302,17 @@ static void pmb_cache_ctor(struct kmem_cache *cachep, void *pmb)
pmbe->entry = PMB_NO_ENTRY;
}
-static int __init pmb_init(void)
+static int __uses_jump_to_uncached pmb_init(void)
{
unsigned int nr_entries = ARRAY_SIZE(pmb_init_map);
- unsigned int entry;
+ unsigned int entry, i;
BUG_ON(unlikely(nr_entries >= NR_PMB_ENTRIES));
pmb_cache = kmem_cache_create("pmb", sizeof(struct pmb_entry), 0,
SLAB_PANIC, pmb_cache_ctor);
- jump_to_P2();
+ jump_to_uncached();
/*
* Ordering is important, P2 must be mapped in the PMB before we
@@ -329,7 +330,12 @@ static int __init pmb_init(void)
/* PMB.SE and UB[7] */
ctrl_outl((1 << 31) | (1 << 7), PMB_PASCR);
- back_to_P1();
+ /* Flush out the TLB */
+ i = ctrl_inl(MMUCR);
+ i |= MMUCR_TI;
+ ctrl_outl(i, MMUCR);
+
+ back_to_cached();
return 0;
}
diff --git a/arch/sh/mm/tlb-nommu.c b/arch/sh/mm/tlb-nommu.c
index 1ccca7c..15111bc 100644
--- a/arch/sh/mm/tlb-nommu.c
+++ b/arch/sh/mm/tlb-nommu.c
@@ -9,6 +9,7 @@
*/
#include <linux/kernel.h>
#include <linux/mm.h>
+#include <asm/pgtable.h>
/*
* Nothing too terribly exciting here ..
@@ -49,3 +50,12 @@ void update_mmu_cache(struct vm_area_struct * vma,
{
BUG();
}
+
+void __init page_table_range_init(unsigned long start, unsigned long end,
+ pgd_t *pgd_base)
+{
+}
+
+void __set_fixmap(enum fixed_addresses idx, unsigned long phys, pgprot_t prot)
+{
+}
diff --git a/arch/sh/mm/tlb-sh4.c b/arch/sh/mm/tlb-sh4.c
index 2d1dd60..f0c7b73 100644
--- a/arch/sh/mm/tlb-sh4.c
+++ b/arch/sh/mm/tlb-sh4.c
@@ -79,7 +79,8 @@ void update_mmu_cache(struct vm_area_struct * vma,
local_irq_restore(flags);
}
-void local_flush_tlb_one(unsigned long asid, unsigned long page)
+void __uses_jump_to_uncached local_flush_tlb_one(unsigned long asid,
+ unsigned long page)
{
unsigned long addr, data;
@@ -91,7 +92,7 @@ void local_flush_tlb_one(unsigned long asid, unsigned long page)
*/
addr = MMU_UTLB_ADDRESS_ARRAY | MMU_PAGE_ASSOC_BIT;
data = page | asid; /* VALID bit is off */
- jump_to_P2();
+ jump_to_uncached();
ctrl_outl(data, addr);
- back_to_P1();
+ back_to_cached();
}
diff --git a/arch/sh/mm/tlb-sh5.c b/arch/sh/mm/tlb-sh5.c
new file mode 100644
index 0000000..f34274a
--- /dev/null
+++ b/arch/sh/mm/tlb-sh5.c
@@ -0,0 +1,164 @@
+/*
+ * arch/sh/mm/tlb-sh5.c
+ *
+ * Copyright (C) 2003 Paul Mundt <lethal@linux-sh.org>
+ * Copyright (C) 2003 Richard Curnow <richard.curnow@superh.com>
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/mm.h>
+#include <linux/init.h>
+#include <asm/page.h>
+#include <asm/tlb.h>
+#include <asm/mmu_context.h>
+
+/**
+ * sh64_tlb_init
+ *
+ * Perform initial setup for the DTLB and ITLB.
+ */
+int __init sh64_tlb_init(void)
+{
+ /* Assign some sane DTLB defaults */
+ cpu_data->dtlb.entries = 64;
+ cpu_data->dtlb.step = 0x10;
+
+ cpu_data->dtlb.first = DTLB_FIXED | cpu_data->dtlb.step;
+ cpu_data->dtlb.next = cpu_data->dtlb.first;
+
+ cpu_data->dtlb.last = DTLB_FIXED |
+ ((cpu_data->dtlb.entries - 1) *
+ cpu_data->dtlb.step);
+
+ /* And again for the ITLB */
+ cpu_data->itlb.entries = 64;
+ cpu_data->itlb.step = 0x10;
+
+ cpu_data->itlb.first = ITLB_FIXED | cpu_data->itlb.step;
+ cpu_data->itlb.next = cpu_data->itlb.first;
+ cpu_data->itlb.last = ITLB_FIXED |
+ ((cpu_data->itlb.entries - 1) *
+ cpu_data->itlb.step);
+
+ return 0;
+}
+
+/**
+ * sh64_next_free_dtlb_entry
+ *
+ * Find the next available DTLB entry
+ */
+unsigned long long sh64_next_free_dtlb_entry(void)
+{
+ return cpu_data->dtlb.next;
+}
+
+/**
+ * sh64_get_wired_dtlb_entry
+ *
+ * Allocate a wired (locked-in) entry in the DTLB
+ */
+unsigned long long sh64_get_wired_dtlb_entry(void)
+{
+ unsigned long long entry = sh64_next_free_dtlb_entry();
+
+ cpu_data->dtlb.first += cpu_data->dtlb.step;
+ cpu_data->dtlb.next += cpu_data->dtlb.step;
+
+ return entry;
+}
+
+/**
+ * sh64_put_wired_dtlb_entry
+ *
+ * @entry: Address of TLB slot.
+ *
+ * Free a wired (locked-in) entry in the DTLB.
+ *
+ * Works like a stack, last one to allocate must be first one to free.
+ */
+int sh64_put_wired_dtlb_entry(unsigned long long entry)
+{
+ __flush_tlb_slot(entry);
+
+ /*
+ * We don't do any particularly useful tracking of wired entries,
+ * so this approach works like a stack .. last one to be allocated
+ * has to be the first one to be freed.
+ *
+ * We could potentially load wired entries into a list and work on
+ * rebalancing the list periodically (which also entails moving the
+ * contents of a TLB entry) .. though I have a feeling that this is
+ * more trouble than it's worth.
+ */
+
+ /*
+ * Entry must be valid .. we don't want any ITLB addresses!
+ */
+ if (entry <= DTLB_FIXED)
+ return -EINVAL;
+
+ /*
+ * Next, check if we're within range to be freed. (ie, must be the
+ * entry beneath the first 'free' entry!
+ */
+ if (entry < (cpu_data->dtlb.first - cpu_data->dtlb.step))
+ return -EINVAL;
+
+ /* If we are, then bring this entry back into the list */
+ cpu_data->dtlb.first -= cpu_data->dtlb.step;
+ cpu_data->dtlb.next = entry;
+
+ return 0;
+}
+
+/**
+ * sh64_setup_tlb_slot
+ *
+ * @config_addr: Address of TLB slot.
+ * @eaddr: Virtual address.
+ * @asid: Address Space Identifier.
+ * @paddr: Physical address.
+ *
+ * Load up a virtual<->physical translation for @eaddr<->@paddr in the
+ * pre-allocated TLB slot @config_addr (see sh64_get_wired_dtlb_entry).
+ */
+inline void sh64_setup_tlb_slot(unsigned long long config_addr,
+ unsigned long eaddr,
+ unsigned long asid,
+ unsigned long paddr)
+{
+ unsigned long long pteh, ptel;
+
+ /* Sign extension */
+#if (NEFF == 32)
+ pteh = (unsigned long long)(signed long long)(signed long) eaddr;
+#else
+#error "Can't sign extend more than 32 bits yet"
+#endif
+ pteh &= PAGE_MASK;
+ pteh |= (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
+#if (NEFF == 32)
+ ptel = (unsigned long long)(signed long long)(signed long) paddr;
+#else
+#error "Can't sign extend more than 32 bits yet"
+#endif
+ ptel &= PAGE_MASK;
+ ptel |= (_PAGE_CACHABLE | _PAGE_READ | _PAGE_WRITE);
+
+ asm volatile("putcfg %0, 1, %1\n\t"
+ "putcfg %0, 0, %2\n"
+ : : "r" (config_addr), "r" (ptel), "r" (pteh));
+}
+
+/**
+ * sh64_teardown_tlb_slot
+ *
+ * @config_addr: Address of TLB slot.
+ *
+ * Teardown any existing mapping in the TLB slot @config_addr.
+ */
+inline void sh64_teardown_tlb_slot(unsigned long long config_addr)
+ __attribute__ ((alias("__flush_tlb_slot")));
diff --git a/arch/sh/mm/tlb-flush.c b/arch/sh/mm/tlbflush_32.c
index 6f45c1f..6f45c1f 100644
--- a/arch/sh/mm/tlb-flush.c
+++ b/arch/sh/mm/tlbflush_32.c
diff --git a/arch/sh/mm/tlbflush_64.c b/arch/sh/mm/tlbflush_64.c
new file mode 100644
index 0000000..2a98c9e
--- /dev/null
+++ b/arch/sh/mm/tlbflush_64.c
@@ -0,0 +1,475 @@
+/*
+ * arch/sh/mm/tlb-flush_64.c
+ *
+ * Copyright (C) 2000, 2001 Paolo Alberelli
+ * Copyright (C) 2003 Richard Curnow (/proc/tlb, bug fixes)
+ * Copyright (C) 2003 Paul Mundt
+ *
+ * This file is subject to the terms and conditions of the GNU General Public
+ * License. See the file "COPYING" in the main directory of this archive
+ * for more details.
+ */
+#include <linux/signal.h>
+#include <linux/rwsem.h>
+#include <linux/sched.h>
+#include <linux/kernel.h>
+#include <linux/errno.h>
+#include <linux/string.h>
+#include <linux/types.h>
+#include <linux/ptrace.h>
+#include <linux/mman.h>
+#include <linux/mm.h>
+#include <linux/smp.h>
+#include <linux/interrupt.h>
+#include <asm/system.h>
+#include <asm/io.h>
+#include <asm/tlb.h>
+#include <asm/uaccess.h>
+#include <asm/pgalloc.h>
+#include <asm/mmu_context.h>
+
+extern void die(const char *,struct pt_regs *,long);
+
+#define PFLAG(val,flag) (( (val) & (flag) ) ? #flag : "" )
+#define PPROT(flag) PFLAG(pgprot_val(prot),flag)
+
+static inline void print_prots(pgprot_t prot)
+{
+ printk("prot is 0x%08lx\n",pgprot_val(prot));
+
+ printk("%s %s %s %s %s\n",PPROT(_PAGE_SHARED),PPROT(_PAGE_READ),
+ PPROT(_PAGE_EXECUTE),PPROT(_PAGE_WRITE),PPROT(_PAGE_USER));
+}
+
+static inline void print_vma(struct vm_area_struct *vma)
+{
+ printk("vma start 0x%08lx\n", vma->vm_start);
+ printk("vma end 0x%08lx\n", vma->vm_end);
+
+ print_prots(vma->vm_page_prot);
+ printk("vm_flags 0x%08lx\n", vma->vm_flags);
+}
+
+static inline void print_task(struct task_struct *tsk)
+{
+ printk("Task pid %d\n", task_pid_nr(tsk));
+}
+
+static pte_t *lookup_pte(struct mm_struct *mm, unsigned long address)
+{
+ pgd_t *dir;
+ pud_t *pud;
+ pmd_t *pmd;
+ pte_t *pte;
+ pte_t entry;
+
+ dir = pgd_offset(mm, address);
+ if (pgd_none(*dir))
+ return NULL;
+
+ pud = pud_offset(dir, address);
+ if (pud_none(*pud))
+ return NULL;
+
+ pmd = pmd_offset(pud, address);
+ if (pmd_none(*pmd))
+ return NULL;
+
+ pte = pte_offset_kernel(pmd, address);
+ entry = *pte;
+ if (pte_none(entry) || !pte_present(entry))
+ return NULL;
+
+ return pte;
+}
+
+/*
+ * This routine handles page faults. It determines the address,
+ * and the problem, and then passes it off to one of the appropriate
+ * routines.
+ */
+asmlinkage void do_page_fault(struct pt_regs *regs, unsigned long writeaccess,
+ unsigned long textaccess, unsigned long address)
+{
+ struct task_struct *tsk;
+ struct mm_struct *mm;
+ struct vm_area_struct * vma;
+ const struct exception_table_entry *fixup;
+ pte_t *pte;
+ int fault;
+
+ /* SIM
+ * Note this is now called with interrupts still disabled
+ * This is to cope with being called for a missing IO port
+ * address with interrupts disabled. This should be fixed as
+ * soon as we have a better 'fast path' miss handler.
+ *
+ * Plus take care how you try and debug this stuff.
+ * For example, writing debug data to a port which you
+ * have just faulted on is not going to work.
+ */
+
+ tsk = current;
+ mm = tsk->mm;
+
+ /* Not an IO address, so reenable interrupts */
+ local_irq_enable();
+
+ /*
+ * If we're in an interrupt or have no user
+ * context, we must not take the fault..
+ */
+ if (in_atomic() || !mm)
+ goto no_context;
+
+ /* TLB misses upon some cache flushes get done under cli() */
+ down_read(&mm->mmap_sem);
+
+ vma = find_vma(mm, address);
+
+ if (!vma) {
+#ifdef DEBUG_FAULT
+ print_task(tsk);
+ printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
+ __FUNCTION__,__LINE__,
+ address,regs->pc,textaccess,writeaccess);
+ show_regs(regs);
+#endif
+ goto bad_area;
+ }
+ if (vma->vm_start <= address) {
+ goto good_area;
+ }
+
+ if (!(vma->vm_flags & VM_GROWSDOWN)) {
+#ifdef DEBUG_FAULT
+ print_task(tsk);
+ printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
+ __FUNCTION__,__LINE__,
+ address,regs->pc,textaccess,writeaccess);
+ show_regs(regs);
+
+ print_vma(vma);
+#endif
+ goto bad_area;
+ }
+ if (expand_stack(vma, address)) {
+#ifdef DEBUG_FAULT
+ print_task(tsk);
+ printk("%s:%d fault, address is 0x%08x PC %016Lx textaccess %d writeaccess %d\n",
+ __FUNCTION__,__LINE__,
+ address,regs->pc,textaccess,writeaccess);
+ show_regs(regs);
+#endif
+ goto bad_area;
+ }
+/*
+ * Ok, we have a good vm_area for this memory access, so
+ * we can handle it..
+ */
+good_area:
+ if (textaccess) {
+ if (!(vma->vm_flags & VM_EXEC))
+ goto bad_area;
+ } else {
+ if (writeaccess) {
+ if (!(vma->vm_flags & VM_WRITE))
+ goto bad_area;
+ } else {
+ if (!(vma->vm_flags & VM_READ))
+ goto bad_area;
+ }
+ }
+
+ /*
+ * If for any reason at all we couldn't handle the fault,
+ * make sure we exit gracefully rather than endlessly redo
+ * the fault.
+ */
+survive:
+ fault = handle_mm_fault(mm, vma, address, writeaccess);
+ if (unlikely(fault & VM_FAULT_ERROR)) {
+ if (fault & VM_FAULT_OOM)
+ goto out_of_memory;
+ else if (fault & VM_FAULT_SIGBUS)
+ goto do_sigbus;
+ BUG();
+ }
+ if (fault & VM_FAULT_MAJOR)
+ tsk->maj_flt++;
+ else
+ tsk->min_flt++;
+
+ /* If we get here, the page fault has been handled. Do the TLB refill
+ now from the newly-setup PTE, to avoid having to fault again right
+ away on the same instruction. */
+ pte = lookup_pte (mm, address);
+ if (!pte) {
+ /* From empirical evidence, we can get here, due to
+ !pte_present(pte). (e.g. if a swap-in occurs, and the page
+ is swapped back out again before the process that wanted it
+ gets rescheduled?) */
+ goto no_pte;
+ }
+
+ __do_tlb_refill(address, textaccess, pte);
+
+no_pte:
+
+ up_read(&mm->mmap_sem);
+ return;
+
+/*
+ * Something tried to access memory that isn't in our memory map..
+ * Fix it, but check if it's kernel or user first..
+ */
+bad_area:
+#ifdef DEBUG_FAULT
+ printk("fault:bad area\n");
+#endif
+ up_read(&mm->mmap_sem);
+
+ if (user_mode(regs)) {
+ static int count=0;
+ siginfo_t info;
+ if (count < 4) {
+ /* This is really to help debug faults when starting
+ * usermode, so only need a few */
+ count++;
+ printk("user mode bad_area address=%08lx pid=%d (%s) pc=%08lx\n",
+ address, task_pid_nr(current), current->comm,
+ (unsigned long) regs->pc);
+#if 0
+ show_regs(regs);
+#endif
+ }
+ if (is_global_init(tsk)) {
+ panic("INIT had user mode bad_area\n");
+ }
+ tsk->thread.address = address;
+ tsk->thread.error_code = writeaccess;
+ info.si_signo = SIGSEGV;
+ info.si_errno = 0;
+ info.si_addr = (void *) address;
+ force_sig_info(SIGSEGV, &info, tsk);
+ return;
+ }
+
+no_context:
+#ifdef DEBUG_FAULT
+ printk("fault:No context\n");
+#endif
+ /* Are we prepared to handle this kernel fault? */
+ fixup = search_exception_tables(regs->pc);
+ if (fixup) {
+ regs->pc = fixup->fixup;
+ return;
+ }
+
+/*
+ * Oops. The kernel tried to access some bad page. We'll have to
+ * terminate things with extreme prejudice.
+ *
+ */
+ if (address < PAGE_SIZE)
+ printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
+ else
+ printk(KERN_ALERT "Unable to handle kernel paging request");
+ printk(" at virtual address %08lx\n", address);
+ printk(KERN_ALERT "pc = %08Lx%08Lx\n", regs->pc >> 32, regs->pc & 0xffffffff);
+ die("Oops", regs, writeaccess);
+ do_exit(SIGKILL);
+
+/*
+ * We ran out of memory, or some other thing happened to us that made
+ * us unable to handle the page fault gracefully.
+ */
+out_of_memory:
+ if (is_global_init(current)) {
+ panic("INIT out of memory\n");
+ yield();
+ goto survive;
+ }
+ printk("fault:Out of memory\n");
+ up_read(&mm->mmap_sem);
+ if (is_global_init(current)) {
+ yield();
+ down_read(&mm->mmap_sem);
+ goto survive;
+ }
+ printk("VM: killing process %s\n", tsk->comm);
+ if (user_mode(regs))
+ do_group_exit(SIGKILL);
+ goto no_context;
+
+do_sigbus:
+ printk("fault:Do sigbus\n");
+ up_read(&mm->mmap_sem);
+
+ /*
+ * Send a sigbus, regardless of whether we were in kernel
+ * or user mode.
+ */
+ tsk->thread.address = address;
+ tsk->thread.error_code = writeaccess;
+ tsk->thread.trap_no = 14;
+ force_sig(SIGBUS, tsk);
+
+ /* Kernel mode? Handle exceptions or die */
+ if (!user_mode(regs))
+ goto no_context;
+}
+
+void update_mmu_cache(struct vm_area_struct * vma,
+ unsigned long address, pte_t pte)
+{
+ /*
+ * This appears to get called once for every pte entry that gets
+ * established => I don't think it's efficient to try refilling the
+ * TLBs with the pages - some may not get accessed even. Also, for
+ * executable pages, it is impossible to determine reliably here which
+ * TLB they should be mapped into (or both even).
+ *
+ * So, just do nothing here and handle faults on demand. In the
+ * TLBMISS handling case, the refill is now done anyway after the pte
+ * has been fixed up, so that deals with most useful cases.
+ */
+}
+
+void local_flush_tlb_one(unsigned long asid, unsigned long page)
+{
+ unsigned long long match, pteh=0, lpage;
+ unsigned long tlb;
+
+ /*
+ * Sign-extend based on neff.
+ */
+ lpage = (page & NEFF_SIGN) ? (page | NEFF_MASK) : page;
+ match = (asid << PTEH_ASID_SHIFT) | PTEH_VALID;
+ match |= lpage;
+
+ for_each_itlb_entry(tlb) {
+ asm volatile ("getcfg %1, 0, %0"
+ : "=r" (pteh)
+ : "r" (tlb) );
+
+ if (pteh == match) {
+ __flush_tlb_slot(tlb);
+ break;
+ }
+ }
+
+ for_each_dtlb_entry(tlb) {
+ asm volatile ("getcfg %1, 0, %0"
+ : "=r" (pteh)
+ : "r" (tlb) );
+
+ if (pteh == match) {
+ __flush_tlb_slot(tlb);
+ break;
+ }
+
+ }
+}
+
+void local_flush_tlb_page(struct vm_area_struct *vma, unsigned long page)
+{
+ unsigned long flags;
+
+ if (vma->vm_mm) {
+ page &= PAGE_MASK;
+ local_irq_save(flags);
+ local_flush_tlb_one(get_asid(), page);
+ local_irq_restore(flags);
+ }
+}
+
+void local_flush_tlb_range(struct vm_area_struct *vma, unsigned long start,
+ unsigned long end)
+{
+ unsigned long flags;
+ unsigned long long match, pteh=0, pteh_epn, pteh_low;
+ unsigned long tlb;
+ unsigned int cpu = smp_processor_id();
+ struct mm_struct *mm;
+
+ mm = vma->vm_mm;
+ if (cpu_context(cpu, mm) == NO_CONTEXT)
+ return;
+
+ local_irq_save(flags);
+
+ start &= PAGE_MASK;
+ end &= PAGE_MASK;
+
+ match = (cpu_asid(cpu, mm) << PTEH_ASID_SHIFT) | PTEH_VALID;
+
+ /* Flush ITLB */
+ for_each_itlb_entry(tlb) {
+ asm volatile ("getcfg %1, 0, %0"
+ : "=r" (pteh)
+ : "r" (tlb) );
+
+ pteh_epn = pteh & PAGE_MASK;
+ pteh_low = pteh & ~PAGE_MASK;
+
+ if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
+ __flush_tlb_slot(tlb);
+ }
+
+ /* Flush DTLB */
+ for_each_dtlb_entry(tlb) {
+ asm volatile ("getcfg %1, 0, %0"
+ : "=r" (pteh)
+ : "r" (tlb) );
+
+ pteh_epn = pteh & PAGE_MASK;
+ pteh_low = pteh & ~PAGE_MASK;
+
+ if (pteh_low == match && pteh_epn >= start && pteh_epn <= end)
+ __flush_tlb_slot(tlb);
+ }
+
+ local_irq_restore(flags);
+}
+
+void local_flush_tlb_mm(struct mm_struct *mm)
+{
+ unsigned long flags;
+ unsigned int cpu = smp_processor_id();
+
+ if (cpu_context(cpu, mm) == NO_CONTEXT)
+ return;
+
+ local_irq_save(flags);
+
+ cpu_context(cpu, mm) = NO_CONTEXT;
+ if (mm == current->mm)
+ activate_context(mm, cpu);
+
+ local_irq_restore(flags);
+}
+
+void local_flush_tlb_all(void)
+{
+ /* Invalidate all, including shared pages, excluding fixed TLBs */
+ unsigned long flags, tlb;
+
+ local_irq_save(flags);
+
+ /* Flush each ITLB entry */
+ for_each_itlb_entry(tlb)
+ __flush_tlb_slot(tlb);
+
+ /* Flush each DTLB entry */
+ for_each_dtlb_entry(tlb)
+ __flush_tlb_slot(tlb);
+
+ local_irq_restore(flags);
+}
+
+void local_flush_tlb_kernel_range(unsigned long start, unsigned long end)
+{
+ /* FIXME: Optimize this later.. */
+ flush_tlb_all();
+}
OpenPOWER on IntegriCloud