summaryrefslogtreecommitdiffstats
path: root/arch/arm/mm
diff options
context:
space:
mode:
Diffstat (limited to 'arch/arm/mm')
-rw-r--r--arch/arm/mm/Kconfig30
-rw-r--r--arch/arm/mm/Makefile2
-rw-r--r--arch/arm/mm/abort-ev6.S5
-rw-r--r--arch/arm/mm/alignment.c55
-rw-r--r--arch/arm/mm/blockops.c185
-rw-r--r--arch/arm/mm/cache-v6.S9
-rw-r--r--arch/arm/mm/consistent.c27
-rw-r--r--arch/arm/mm/copypage-v6.c16
-rw-r--r--arch/arm/mm/fault-armv.c7
-rw-r--r--arch/arm/mm/flush.c43
-rw-r--r--arch/arm/mm/init.c497
-rw-r--r--arch/arm/mm/ioremap.c8
-rw-r--r--arch/arm/mm/mm-armv.c235
-rw-r--r--arch/arm/mm/proc-v6.S35
14 files changed, 495 insertions, 659 deletions
diff --git a/arch/arm/mm/Kconfig b/arch/arm/mm/Kconfig
index db5e47d..e84fdde 100644
--- a/arch/arm/mm/Kconfig
+++ b/arch/arm/mm/Kconfig
@@ -102,8 +102,8 @@ config CPU_ARM922T
# ARM925T
config CPU_ARM925T
bool "Support ARM925T processor" if ARCH_OMAP1
- depends on ARCH_OMAP1510
- default y if ARCH_OMAP1510
+ depends on ARCH_OMAP15XX
+ default y if ARCH_OMAP15XX
select CPU_32v4
select CPU_ABRT_EV4T
select CPU_CACHE_V4WT
@@ -120,8 +120,8 @@ config CPU_ARM925T
# ARM926T
config CPU_ARM926T
- bool "Support ARM926T processor" if ARCH_INTEGRATOR
- depends on ARCH_INTEGRATOR || ARCH_VERSATILE_PB || MACH_VERSATILE_AB || ARCH_OMAP730 || ARCH_OMAP16XX
+ bool "Support ARM926T processor"
+ depends on ARCH_INTEGRATOR || ARCH_VERSATILE_PB || MACH_VERSATILE_AB || ARCH_OMAP730 || ARCH_OMAP16XX || MACH_REALVIEW_EB
default y if ARCH_VERSATILE_PB || MACH_VERSATILE_AB || ARCH_OMAP730 || ARCH_OMAP16XX
select CPU_32v5
select CPU_ABRT_EV5TJ
@@ -242,7 +242,7 @@ config CPU_XSCALE
# ARMv6
config CPU_V6
bool "Support ARM V6 processor"
- depends on ARCH_INTEGRATOR
+ depends on ARCH_INTEGRATOR || MACH_REALVIEW_EB || ARCH_OMAP2
select CPU_32v6
select CPU_ABRT_EV6
select CPU_CACHE_V6
@@ -250,6 +250,18 @@ config CPU_V6
select CPU_COPY_V6
select CPU_TLB_V6
+# ARMv6k
+config CPU_32v6K
+ bool "Support ARM V6K processor extensions" if !SMP
+ depends on CPU_V6
+ default y if SMP
+ help
+ Say Y here if your ARMv6 processor supports the 'K' extension.
+ This enables the kernel to use some instructions not present
+ on previous processors, and as such a kernel build with this
+ enabled will not boot on processors with do not support these
+ instructions.
+
# Figure out what processor architecture version we should be using.
# This defines the compiler instruction set which depends on the machine type.
config CPU_32v3
@@ -370,21 +382,21 @@ config CPU_BIG_ENDIAN
config CPU_ICACHE_DISABLE
bool "Disable I-Cache"
- depends on CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM1020
+ depends on CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM1020 || CPU_V6
help
Say Y here to disable the processor instruction cache. Unless
you have a reason not to or are unsure, say N.
config CPU_DCACHE_DISABLE
bool "Disable D-Cache"
- depends on CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM1020
+ depends on CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM1020 || CPU_V6
help
Say Y here to disable the processor data cache. Unless
you have a reason not to or are unsure, say N.
config CPU_DCACHE_WRITETHROUGH
bool "Force write through D-cache"
- depends on (CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM1020) && !CPU_DCACHE_DISABLE
+ depends on (CPU_ARM920T || CPU_ARM922T || CPU_ARM925T || CPU_ARM926T || CPU_ARM1020 || CPU_V6) && !CPU_DCACHE_DISABLE
default y if CPU_ARM925T
help
Say Y here to use the data cache in writethrough mode. Unless you
@@ -399,7 +411,7 @@ config CPU_CACHE_ROUND_ROBIN
config CPU_BPREDICT_DISABLE
bool "Disable branch prediction"
- depends on CPU_ARM1020
+ depends on CPU_ARM1020 || CPU_V6
help
Say Y here to disable branch prediction. If unsure, say N.
diff --git a/arch/arm/mm/Makefile b/arch/arm/mm/Makefile
index 59f47d4..ffe73ba 100644
--- a/arch/arm/mm/Makefile
+++ b/arch/arm/mm/Makefile
@@ -51,4 +51,4 @@ obj-$(CONFIG_CPU_ARM1026) += proc-arm1026.o
obj-$(CONFIG_CPU_SA110) += proc-sa110.o
obj-$(CONFIG_CPU_SA1100) += proc-sa1100.o
obj-$(CONFIG_CPU_XSCALE) += proc-xscale.o
-obj-$(CONFIG_CPU_V6) += proc-v6.o blockops.o
+obj-$(CONFIG_CPU_V6) += proc-v6.o
diff --git a/arch/arm/mm/abort-ev6.S b/arch/arm/mm/abort-ev6.S
index 8f76f3d..dbd3460 100644
--- a/arch/arm/mm/abort-ev6.S
+++ b/arch/arm/mm/abort-ev6.S
@@ -20,6 +20,11 @@
*/
.align 5
ENTRY(v6_early_abort)
+#ifdef CONFIG_CPU_MPCORE
+ clrex
+#else
+ strex r0, r1, [sp] @ Clear the exclusive monitor
+#endif
mrc p15, 0, r1, c5, c0, 0 @ get FSR
mrc p15, 0, r0, c6, c0, 0 @ get FAR
/*
diff --git a/arch/arm/mm/alignment.c b/arch/arm/mm/alignment.c
index 4b39d86..705c989 100644
--- a/arch/arm/mm/alignment.c
+++ b/arch/arm/mm/alignment.c
@@ -111,7 +111,7 @@ proc_alignment_read(char *page, char **start, off_t off, int count, int *eof,
}
static int proc_alignment_write(struct file *file, const char __user *buffer,
- unsigned long count, void *data)
+ unsigned long count, void *data)
{
char mode;
@@ -119,7 +119,7 @@ static int proc_alignment_write(struct file *file, const char __user *buffer,
if (get_user(mode, buffer))
return -EFAULT;
if (mode >= '0' && mode <= '5')
- ai_usermode = mode - '0';
+ ai_usermode = mode - '0';
}
return count;
}
@@ -262,7 +262,7 @@ union offset_union {
goto fault; \
} while (0)
-#define put32_unaligned_check(val,addr) \
+#define put32_unaligned_check(val,addr) \
__put32_unaligned_check("strb", val, addr)
#define put32t_unaligned_check(val,addr) \
@@ -306,19 +306,19 @@ do_alignment_ldrhstrh(unsigned long addr, unsigned long instr, struct pt_regs *r
return TYPE_LDST;
user:
- if (LDST_L_BIT(instr)) {
- unsigned long val;
- get16t_unaligned_check(val, addr);
+ if (LDST_L_BIT(instr)) {
+ unsigned long val;
+ get16t_unaligned_check(val, addr);
- /* signed half-word? */
- if (instr & 0x40)
- val = (signed long)((signed short) val);
+ /* signed half-word? */
+ if (instr & 0x40)
+ val = (signed long)((signed short) val);
- regs->uregs[rd] = val;
- } else
- put16t_unaligned_check(regs->uregs[rd], addr);
+ regs->uregs[rd] = val;
+ } else
+ put16t_unaligned_check(regs->uregs[rd], addr);
- return TYPE_LDST;
+ return TYPE_LDST;
fault:
return TYPE_FAULT;
@@ -330,6 +330,9 @@ do_alignment_ldrdstrd(unsigned long addr, unsigned long instr,
{
unsigned int rd = RD_BITS(instr);
+ if (((rd & 1) == 1) || (rd == 14))
+ goto bad;
+
ai_dword += 1;
if (user_mode(regs))
@@ -339,11 +342,11 @@ do_alignment_ldrdstrd(unsigned long addr, unsigned long instr,
unsigned long val;
get32_unaligned_check(val, addr);
regs->uregs[rd] = val;
- get32_unaligned_check(val, addr+4);
- regs->uregs[rd+1] = val;
+ get32_unaligned_check(val, addr + 4);
+ regs->uregs[rd + 1] = val;
} else {
put32_unaligned_check(regs->uregs[rd], addr);
- put32_unaligned_check(regs->uregs[rd+1], addr+4);
+ put32_unaligned_check(regs->uregs[rd + 1], addr + 4);
}
return TYPE_LDST;
@@ -353,15 +356,16 @@ do_alignment_ldrdstrd(unsigned long addr, unsigned long instr,
unsigned long val;
get32t_unaligned_check(val, addr);
regs->uregs[rd] = val;
- get32t_unaligned_check(val, addr+4);
- regs->uregs[rd+1] = val;
+ get32t_unaligned_check(val, addr + 4);
+ regs->uregs[rd + 1] = val;
} else {
put32t_unaligned_check(regs->uregs[rd], addr);
- put32t_unaligned_check(regs->uregs[rd+1], addr+4);
+ put32t_unaligned_check(regs->uregs[rd + 1], addr + 4);
}
return TYPE_LDST;
-
+ bad:
+ return TYPE_ERROR;
fault:
return TYPE_FAULT;
}
@@ -439,7 +443,7 @@ do_alignment_ldmstm(unsigned long addr, unsigned long instr, struct pt_regs *reg
if (LDST_P_EQ_U(instr)) /* U = P */
eaddr += 4;
- /*
+ /*
* For alignment faults on the ARM922T/ARM920T the MMU makes
* the FSR (and hence addr) equal to the updated base address
* of the multiple access rather than the restored value.
@@ -566,7 +570,7 @@ thumb2arm(u16 tinstr)
/* 6.5.1 Format 3: */
case 0x4800 >> 11: /* 7.1.28 LDR(3) */
/* NOTE: This case is not technically possible. We're
- * loading 32-bit memory data via PC relative
+ * loading 32-bit memory data via PC relative
* addressing mode. So we can and should eliminate
* this case. But I'll leave it here for now.
*/
@@ -638,7 +642,7 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
if (fault) {
type = TYPE_FAULT;
- goto bad_or_fault;
+ goto bad_or_fault;
}
if (user_mode(regs))
@@ -663,6 +667,8 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
else if ((instr & 0x001000f0) == 0x000000d0 || /* LDRD */
(instr & 0x001000f0) == 0x000000f0) /* STRD */
handler = do_alignment_ldrdstrd;
+ else if ((instr & 0x01f00ff0) == 0x01000090) /* SWP */
+ goto swp;
else
goto bad;
break;
@@ -733,6 +739,9 @@ do_alignment(unsigned long addr, unsigned int fsr, struct pt_regs *regs)
do_bad_area(current, current->mm, addr, fsr, regs);
return 0;
+ swp:
+ printk(KERN_ERR "Alignment trap: not handling swp instruction\n");
+
bad:
/*
* Oops, we didn't handle the instruction.
diff --git a/arch/arm/mm/blockops.c b/arch/arm/mm/blockops.c
deleted file mode 100644
index 4f5ee2d..0000000
--- a/arch/arm/mm/blockops.c
+++ /dev/null
@@ -1,185 +0,0 @@
-#include <linux/kernel.h>
-#include <linux/init.h>
-#include <linux/errno.h>
-#include <linux/mm.h>
-
-#include <asm/memory.h>
-#include <asm/ptrace.h>
-#include <asm/cacheflush.h>
-#include <asm/traps.h>
-
-extern struct cpu_cache_fns blk_cache_fns;
-
-#define HARVARD_CACHE
-
-/*
- * blk_flush_kern_dcache_page(kaddr)
- *
- * Ensure that the data held in the page kaddr is written back
- * to the page in question.
- *
- * - kaddr - kernel address (guaranteed to be page aligned)
- */
-static void __attribute__((naked))
-blk_flush_kern_dcache_page(void *kaddr)
-{
- asm(
- "add r1, r0, %0 \n\
- sub r1, r1, %1 \n\
-1: .word 0xec401f0e @ mcrr p15, 0, r0, r1, c14, 0 @ blocking \n\
- mov r0, #0 \n\
- mcr p15, 0, r0, c7, c5, 0 \n\
- mcr p15, 0, r0, c7, c10, 4 \n\
- mov pc, lr"
- :
- : "I" (PAGE_SIZE), "I" (L1_CACHE_BYTES));
-}
-
-/*
- * blk_dma_inv_range(start,end)
- *
- * Invalidate the data cache within the specified region; we will
- * be performing a DMA operation in this region and we want to
- * purge old data in the cache.
- *
- * - start - virtual start address of region
- * - end - virtual end address of region
- */
-static void __attribute__((naked))
-blk_dma_inv_range_unified(unsigned long start, unsigned long end)
-{
- asm(
- "tst r0, %0 \n\
- mcrne p15, 0, r0, c7, c11, 1 @ clean unified line \n\
- tst r1, %0 \n\
- mcrne p15, 0, r1, c7, c15, 1 @ clean & invalidate unified line\n\
- .word 0xec401f06 @ mcrr p15, 0, r1, r0, c6, 0 @ blocking \n\
- mov r0, #0 \n\
- mcr p15, 0, r0, c7, c10, 4 @ drain write buffer \n\
- mov pc, lr"
- :
- : "I" (L1_CACHE_BYTES - 1));
-}
-
-static void __attribute__((naked))
-blk_dma_inv_range_harvard(unsigned long start, unsigned long end)
-{
- asm(
- "tst r0, %0 \n\
- mcrne p15, 0, r0, c7, c10, 1 @ clean D line \n\
- tst r1, %0 \n\
- mcrne p15, 0, r1, c7, c14, 1 @ clean & invalidate D line \n\
- .word 0xec401f06 @ mcrr p15, 0, r1, r0, c6, 0 @ blocking \n\
- mov r0, #0 \n\
- mcr p15, 0, r0, c7, c10, 4 @ drain write buffer \n\
- mov pc, lr"
- :
- : "I" (L1_CACHE_BYTES - 1));
-}
-
-/*
- * blk_dma_clean_range(start,end)
- * - start - virtual start address of region
- * - end - virtual end address of region
- */
-static void __attribute__((naked))
-blk_dma_clean_range(unsigned long start, unsigned long end)
-{
- asm(
- ".word 0xec401f0c @ mcrr p15, 0, r1, r0, c12, 0 @ blocking \n\
- mov r0, #0 \n\
- mcr p15, 0, r0, c7, c10, 4 @ drain write buffer \n\
- mov pc, lr");
-}
-
-/*
- * blk_dma_flush_range(start,end)
- * - start - virtual start address of region
- * - end - virtual end address of region
- */
-static void __attribute__((naked))
-blk_dma_flush_range(unsigned long start, unsigned long end)
-{
- asm(
- ".word 0xec401f0e @ mcrr p15, 0, r1, r0, c14, 0 @ blocking \n\
- mov pc, lr");
-}
-
-static int blockops_trap(struct pt_regs *regs, unsigned int instr)
-{
- regs->ARM_r4 |= regs->ARM_r2;
- regs->ARM_pc += 4;
- return 0;
-}
-
-static char *func[] = {
- "Prefetch data range",
- "Clean+Invalidate data range",
- "Clean data range",
- "Invalidate data range",
- "Invalidate instr range"
-};
-
-static struct undef_hook blockops_hook __initdata = {
- .instr_mask = 0x0fffffd0,
- .instr_val = 0x0c401f00,
- .cpsr_mask = PSR_T_BIT,
- .cpsr_val = 0,
- .fn = blockops_trap,
-};
-
-static int __init blockops_check(void)
-{
- register unsigned int err asm("r4") = 0;
- unsigned int err_pos = 1;
- unsigned int cache_type;
- int i;
-
- asm("mrc p15, 0, %0, c0, c0, 1" : "=r" (cache_type));
-
- printk("Checking V6 block cache operations:\n");
- register_undef_hook(&blockops_hook);
-
- __asm__ ("mov r0, %0\n\t"
- "mov r1, %1\n\t"
- "mov r2, #1\n\t"
- ".word 0xec401f2c @ mcrr p15, 0, r1, r0, c12, 2\n\t"
- "mov r2, #2\n\t"
- ".word 0xec401f0e @ mcrr p15, 0, r1, r0, c14, 0\n\t"
- "mov r2, #4\n\t"
- ".word 0xec401f0c @ mcrr p15, 0, r1, r0, c12, 0\n\t"
- "mov r2, #8\n\t"
- ".word 0xec401f06 @ mcrr p15, 0, r1, r0, c6, 0\n\t"
- "mov r2, #16\n\t"
- ".word 0xec401f05 @ mcrr p15, 0, r1, r0, c5, 0\n\t"
- :
- : "r" (PAGE_OFFSET), "r" (PAGE_OFFSET + 128)
- : "r0", "r1", "r2");
-
- unregister_undef_hook(&blockops_hook);
-
- for (i = 0; i < ARRAY_SIZE(func); i++, err_pos <<= 1)
- printk("%30s: %ssupported\n", func[i], err & err_pos ? "not " : "");
-
- if ((err & 8) == 0) {
- printk(" --> Using %s block cache invalidate\n",
- cache_type & (1 << 24) ? "harvard" : "unified");
- if (cache_type & (1 << 24))
- cpu_cache.dma_inv_range = blk_dma_inv_range_harvard;
- else
- cpu_cache.dma_inv_range = blk_dma_inv_range_unified;
- }
- if ((err & 4) == 0) {
- printk(" --> Using block cache clean\n");
- cpu_cache.dma_clean_range = blk_dma_clean_range;
- }
- if ((err & 2) == 0) {
- printk(" --> Using block cache clean+invalidate\n");
- cpu_cache.dma_flush_range = blk_dma_flush_range;
- cpu_cache.flush_kern_dcache_page = blk_flush_kern_dcache_page;
- }
-
- return 0;
-}
-
-__initcall(blockops_check);
diff --git a/arch/arm/mm/cache-v6.S b/arch/arm/mm/cache-v6.S
index 85c10a7..72966d9 100644
--- a/arch/arm/mm/cache-v6.S
+++ b/arch/arm/mm/cache-v6.S
@@ -18,6 +18,7 @@
#define HARVARD_CACHE
#define CACHE_LINE_SIZE 32
#define D_CACHE_LINE_SIZE 32
+#define BTB_FLUSH_SIZE 8
/*
* v6_flush_cache_all()
@@ -98,7 +99,13 @@ ENTRY(v6_coherent_user_range)
mcr p15, 0, r0, c7, c5, 1 @ invalidate I line
#endif
mcr p15, 0, r0, c7, c5, 7 @ invalidate BTB entry
- add r0, r0, #CACHE_LINE_SIZE
+ add r0, r0, #BTB_FLUSH_SIZE
+ mcr p15, 0, r0, c7, c5, 7 @ invalidate BTB entry
+ add r0, r0, #BTB_FLUSH_SIZE
+ mcr p15, 0, r0, c7, c5, 7 @ invalidate BTB entry
+ add r0, r0, #BTB_FLUSH_SIZE
+ mcr p15, 0, r0, c7, c5, 7 @ invalidate BTB entry
+ add r0, r0, #BTB_FLUSH_SIZE
cmp r0, r1
blo 1b
#ifdef HARVARD_CACHE
diff --git a/arch/arm/mm/consistent.c b/arch/arm/mm/consistent.c
index 26356ce..dbfe9e8 100644
--- a/arch/arm/mm/consistent.c
+++ b/arch/arm/mm/consistent.c
@@ -66,6 +66,7 @@ struct vm_region {
unsigned long vm_start;
unsigned long vm_end;
struct page *vm_pages;
+ int vm_active;
};
static struct vm_region consistent_head = {
@@ -75,7 +76,7 @@ static struct vm_region consistent_head = {
};
static struct vm_region *
-vm_region_alloc(struct vm_region *head, size_t size, int gfp)
+vm_region_alloc(struct vm_region *head, size_t size, gfp_t gfp)
{
unsigned long addr = head->vm_start, end = head->vm_end - size;
unsigned long flags;
@@ -104,6 +105,7 @@ vm_region_alloc(struct vm_region *head, size_t size, int gfp)
list_add_tail(&new->vm_list, &c->vm_list);
new->vm_start = addr;
new->vm_end = addr + size;
+ new->vm_active = 1;
spin_unlock_irqrestore(&consistent_lock, flags);
return new;
@@ -120,7 +122,7 @@ static struct vm_region *vm_region_find(struct vm_region *head, unsigned long ad
struct vm_region *c;
list_for_each_entry(c, &head->vm_list, vm_list) {
- if (c->vm_start == addr)
+ if (c->vm_active && c->vm_start == addr)
goto out;
}
c = NULL;
@@ -133,7 +135,7 @@ static struct vm_region *vm_region_find(struct vm_region *head, unsigned long ad
#endif
static void *
-__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, int gfp,
+__dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp,
pgprot_t prot)
{
struct page *page;
@@ -251,7 +253,7 @@ __dma_alloc(struct device *dev, size_t size, dma_addr_t *handle, int gfp,
* virtual and bus address for that space.
*/
void *
-dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, int gfp)
+dma_alloc_coherent(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
{
return __dma_alloc(dev, size, handle, gfp,
pgprot_noncached(pgprot_kernel));
@@ -263,7 +265,7 @@ EXPORT_SYMBOL(dma_alloc_coherent);
* dma_alloc_coherent above.
*/
void *
-dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, int gfp)
+dma_alloc_writecombine(struct device *dev, size_t size, dma_addr_t *handle, gfp_t gfp)
{
return __dma_alloc(dev, size, handle, gfp,
pgprot_writecombine(pgprot_kernel));
@@ -319,6 +321,7 @@ EXPORT_SYMBOL(dma_mmap_writecombine);
/*
* free a page as defined by the above mapping.
+ * Must not be called with IRQs disabled.
*/
void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr_t handle)
{
@@ -326,14 +329,18 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr
unsigned long flags, addr;
pte_t *ptep;
+ WARN_ON(irqs_disabled());
+
size = PAGE_ALIGN(size);
spin_lock_irqsave(&consistent_lock, flags);
-
c = vm_region_find(&consistent_head, (unsigned long)cpu_addr);
if (!c)
goto no_area;
+ c->vm_active = 0;
+ spin_unlock_irqrestore(&consistent_lock, flags);
+
if ((c->vm_end - c->vm_start) != size) {
printk(KERN_ERR "%s: freeing wrong coherent size (%ld != %d)\n",
__func__, c->vm_end - c->vm_start, size);
@@ -372,8 +379,8 @@ void dma_free_coherent(struct device *dev, size_t size, void *cpu_addr, dma_addr
flush_tlb_kernel_range(c->vm_start, c->vm_end);
+ spin_lock_irqsave(&consistent_lock, flags);
list_del(&c->vm_list);
-
spin_unlock_irqrestore(&consistent_lock, flags);
kfree(c);
@@ -397,8 +404,6 @@ static int __init consistent_init(void)
pte_t *pte;
int ret = 0;
- spin_lock(&init_mm.page_table_lock);
-
do {
pgd = pgd_offset(&init_mm, CONSISTENT_BASE);
pmd = pmd_alloc(&init_mm, pgd, CONSISTENT_BASE);
@@ -409,7 +414,7 @@ static int __init consistent_init(void)
}
WARN_ON(!pmd_none(*pmd));
- pte = pte_alloc_kernel(&init_mm, pmd, CONSISTENT_BASE);
+ pte = pte_alloc_kernel(pmd, CONSISTENT_BASE);
if (!pte) {
printk(KERN_ERR "%s: no pte tables\n", __func__);
ret = -ENOMEM;
@@ -419,8 +424,6 @@ static int __init consistent_init(void)
consistent_pte = pte;
} while (0);
- spin_unlock(&init_mm.page_table_lock);
-
return ret;
}
diff --git a/arch/arm/mm/copypage-v6.c b/arch/arm/mm/copypage-v6.c
index 27d0415..269ce69 100644
--- a/arch/arm/mm/copypage-v6.c
+++ b/arch/arm/mm/copypage-v6.c
@@ -22,9 +22,7 @@
#endif
#define from_address (0xffff8000)
-#define from_pgprot PAGE_KERNEL
#define to_address (0xffffc000)
-#define to_pgprot PAGE_KERNEL
#define TOP_PTE(x) pte_offset_kernel(top_pmd, x)
@@ -34,7 +32,7 @@ static DEFINE_SPINLOCK(v6_lock);
* Copy the user page. No aliasing to deal with so we can just
* attack the kernel's existing mapping of these pages.
*/
-void v6_copy_user_page_nonaliasing(void *kto, const void *kfrom, unsigned long vaddr)
+static void v6_copy_user_page_nonaliasing(void *kto, const void *kfrom, unsigned long vaddr)
{
copy_page(kto, kfrom);
}
@@ -43,7 +41,7 @@ void v6_copy_user_page_nonaliasing(void *kto, const void *kfrom, unsigned long v
* Clear the user page. No aliasing to deal with so we can just
* attack the kernel's existing mapping of this page.
*/
-void v6_clear_user_page_nonaliasing(void *kaddr, unsigned long vaddr)
+static void v6_clear_user_page_nonaliasing(void *kaddr, unsigned long vaddr)
{
clear_page(kaddr);
}
@@ -51,7 +49,7 @@ void v6_clear_user_page_nonaliasing(void *kaddr, unsigned long vaddr)
/*
* Copy the page, taking account of the cache colour.
*/
-void v6_copy_user_page_aliasing(void *kto, const void *kfrom, unsigned long vaddr)
+static void v6_copy_user_page_aliasing(void *kto, const void *kfrom, unsigned long vaddr)
{
unsigned int offset = CACHE_COLOUR(vaddr);
unsigned long from, to;
@@ -72,8 +70,8 @@ void v6_copy_user_page_aliasing(void *kto, const void *kfrom, unsigned long vadd
*/
spin_lock(&v6_lock);
- set_pte(TOP_PTE(from_address) + offset, pfn_pte(__pa(kfrom) >> PAGE_SHIFT, from_pgprot));
- set_pte(TOP_PTE(to_address) + offset, pfn_pte(__pa(kto) >> PAGE_SHIFT, to_pgprot));
+ set_pte(TOP_PTE(from_address) + offset, pfn_pte(__pa(kfrom) >> PAGE_SHIFT, PAGE_KERNEL));
+ set_pte(TOP_PTE(to_address) + offset, pfn_pte(__pa(kto) >> PAGE_SHIFT, PAGE_KERNEL));
from = from_address + (offset << PAGE_SHIFT);
to = to_address + (offset << PAGE_SHIFT);
@@ -91,7 +89,7 @@ void v6_copy_user_page_aliasing(void *kto, const void *kfrom, unsigned long vadd
* so remap the kernel page into the same cache colour as the user
* page.
*/
-void v6_clear_user_page_aliasing(void *kaddr, unsigned long vaddr)
+static void v6_clear_user_page_aliasing(void *kaddr, unsigned long vaddr)
{
unsigned int offset = CACHE_COLOUR(vaddr);
unsigned long to = to_address + (offset << PAGE_SHIFT);
@@ -112,7 +110,7 @@ void v6_clear_user_page_aliasing(void *kaddr, unsigned long vaddr)
*/
spin_lock(&v6_lock);
- set_pte(TOP_PTE(to_address) + offset, pfn_pte(__pa(kaddr) >> PAGE_SHIFT, to_pgprot));
+ set_pte(TOP_PTE(to_address) + offset, pfn_pte(__pa(kaddr) >> PAGE_SHIFT, PAGE_KERNEL));
flush_tlb_kernel_page(to);
clear_page((void *)to);
diff --git a/arch/arm/mm/fault-armv.c b/arch/arm/mm/fault-armv.c
index be4ab3d..7fc1b35 100644
--- a/arch/arm/mm/fault-armv.c
+++ b/arch/arm/mm/fault-armv.c
@@ -26,6 +26,11 @@ static unsigned long shared_pte_mask = L_PTE_CACHEABLE;
/*
* We take the easy way out of this problem - we make the
* PTE uncacheable. However, we leave the write buffer on.
+ *
+ * Note that the pte lock held when calling update_mmu_cache must also
+ * guard the pte (somewhere else in the same mm) that we modify here.
+ * Therefore those configurations which might call adjust_pte (those
+ * without CONFIG_CPU_CACHE_VIPT) cannot support split page_table_lock.
*/
static int adjust_pte(struct vm_area_struct *vma, unsigned long address)
{
@@ -127,7 +132,7 @@ void __flush_dcache_page(struct address_space *mapping, struct page *page);
* 2. If we have multiple shared mappings of the same space in
* an object, we need to deal with the cache aliasing issues.
*
- * Note that the page_table_lock will be held.
+ * Note that the pte lock will be held.
*/
void update_mmu_cache(struct vm_area_struct *vma, unsigned long addr, pte_t pte)
{
diff --git a/arch/arm/mm/flush.c b/arch/arm/mm/flush.c
index b0208c9..330695b 100644
--- a/arch/arm/mm/flush.c
+++ b/arch/arm/mm/flush.c
@@ -17,6 +17,24 @@
#ifdef CONFIG_CPU_CACHE_VIPT
+#define ALIAS_FLUSH_START 0xffff4000
+
+#define TOP_PTE(x) pte_offset_kernel(top_pmd, x)
+
+static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
+{
+ unsigned long to = ALIAS_FLUSH_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
+
+ set_pte(TOP_PTE(to), pfn_pte(pfn, PAGE_KERNEL));
+ flush_tlb_kernel_page(to);
+
+ asm( "mcrr p15, 0, %1, %0, c14\n"
+ " mcrr p15, 0, %1, %0, c5\n"
+ :
+ : "r" (to), "r" (to + PAGE_SIZE - L1_CACHE_BYTES)
+ : "cc");
+}
+
void flush_cache_mm(struct mm_struct *mm)
{
if (cache_is_vivt()) {
@@ -67,24 +85,6 @@ void flush_cache_page(struct vm_area_struct *vma, unsigned long user_addr, unsig
if (cache_is_vipt_aliasing())
flush_pfn_alias(pfn, user_addr);
}
-
-#define ALIAS_FLUSH_START 0xffff4000
-
-#define TOP_PTE(x) pte_offset_kernel(top_pmd, x)
-
-static void flush_pfn_alias(unsigned long pfn, unsigned long vaddr)
-{
- unsigned long to = ALIAS_FLUSH_START + (CACHE_COLOUR(vaddr) << PAGE_SHIFT);
-
- set_pte(TOP_PTE(to), pfn_pte(pfn, PAGE_KERNEL));
- flush_tlb_kernel_page(to);
-
- asm( "mcrr p15, 0, %1, %0, c14\n"
- " mcrr p15, 0, %1, %0, c5\n"
- :
- : "r" (to), "r" (to + PAGE_SIZE - L1_CACHE_BYTES)
- : "cc");
-}
#else
#define flush_pfn_alias(pfn,vaddr) do { } while (0)
#endif
@@ -155,14 +155,19 @@ static void __flush_dcache_aliases(struct address_space *mapping, struct page *p
* space mappings, we can be lazy and remember that we may have dirty
* kernel cache lines for later. Otherwise, we assume we have
* aliasing mappings.
+ *
+ * Note that we disable the lazy flush for SMP.
*/
void flush_dcache_page(struct page *page)
{
struct address_space *mapping = page_mapping(page);
+#ifndef CONFIG_SMP
if (mapping && !mapping_mapped(mapping))
set_bit(PG_dcache_dirty, &page->flags);
- else {
+ else
+#endif
+ {
__flush_dcache_page(mapping, page);
if (mapping && cache_is_vivt())
__flush_dcache_aliases(mapping, page);
diff --git a/arch/arm/mm/init.c b/arch/arm/mm/init.c
index edffa47..8b276ee 100644
--- a/arch/arm/mm/init.c
+++ b/arch/arm/mm/init.c
@@ -1,7 +1,7 @@
/*
* linux/arch/arm/mm/init.c
*
- * Copyright (C) 1995-2002 Russell King
+ * Copyright (C) 1995-2005 Russell King
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
@@ -86,14 +86,19 @@ void show_mem(void)
printk("%d pages swap cached\n", cached);
}
-struct node_info {
- unsigned int start;
- unsigned int end;
- int bootmap_pages;
-};
+static inline pmd_t *pmd_off(pgd_t *pgd, unsigned long virt)
+{
+ return pmd_offset(pgd, virt);
+}
+
+static inline pmd_t *pmd_off_k(unsigned long virt)
+{
+ return pmd_off(pgd_offset_k(virt), virt);
+}
-#define O_PFN_DOWN(x) ((x) >> PAGE_SHIFT)
-#define O_PFN_UP(x) (PAGE_ALIGN(x) >> PAGE_SHIFT)
+#define for_each_nodebank(iter,mi,no) \
+ for (iter = 0; iter < mi->nr_banks; iter++) \
+ if (mi->bank[iter].node == no)
/*
* FIXME: We really want to avoid allocating the bootmap bitmap
@@ -106,15 +111,12 @@ find_bootmap_pfn(int node, struct meminfo *mi, unsigned int bootmap_pages)
{
unsigned int start_pfn, bank, bootmap_pfn;
- start_pfn = O_PFN_UP(__pa(&_end));
+ start_pfn = PAGE_ALIGN(__pa(&_end)) >> PAGE_SHIFT;
bootmap_pfn = 0;
- for (bank = 0; bank < mi->nr_banks; bank ++) {
+ for_each_nodebank(bank, mi, node) {
unsigned int start, end;
- if (mi->bank[bank].node != node)
- continue;
-
start = mi->bank[bank].start >> PAGE_SHIFT;
end = (mi->bank[bank].size +
mi->bank[bank].start) >> PAGE_SHIFT;
@@ -140,92 +142,6 @@ find_bootmap_pfn(int node, struct meminfo *mi, unsigned int bootmap_pages)
return bootmap_pfn;
}
-/*
- * Scan the memory info structure and pull out:
- * - the end of memory
- * - the number of nodes
- * - the pfn range of each node
- * - the number of bootmem bitmap pages
- */
-static unsigned int __init
-find_memend_and_nodes(struct meminfo *mi, struct node_info *np)
-{
- unsigned int i, bootmem_pages = 0, memend_pfn = 0;
-
- for (i = 0; i < MAX_NUMNODES; i++) {
- np[i].start = -1U;
- np[i].end = 0;
- np[i].bootmap_pages = 0;
- }
-
- for (i = 0; i < mi->nr_banks; i++) {
- unsigned long start, end;
- int node;
-
- if (mi->bank[i].size == 0) {
- /*
- * Mark this bank with an invalid node number
- */
- mi->bank[i].node = -1;
- continue;
- }
-
- node = mi->bank[i].node;
-
- /*
- * Make sure we haven't exceeded the maximum number of nodes
- * that we have in this configuration. If we have, we're in
- * trouble. (maybe we ought to limit, instead of bugging?)
- */
- if (node >= MAX_NUMNODES)
- BUG();
- node_set_online(node);
-
- /*
- * Get the start and end pfns for this bank
- */
- start = mi->bank[i].start >> PAGE_SHIFT;
- end = (mi->bank[i].start + mi->bank[i].size) >> PAGE_SHIFT;
-
- if (np[node].start > start)
- np[node].start = start;
-
- if (np[node].end < end)
- np[node].end = end;
-
- if (memend_pfn < end)
- memend_pfn = end;
- }
-
- /*
- * Calculate the number of pages we require to
- * store the bootmem bitmaps.
- */
- for_each_online_node(i) {
- if (np[i].end == 0)
- continue;
-
- np[i].bootmap_pages = bootmem_bootmap_pages(np[i].end -
- np[i].start);
- bootmem_pages += np[i].bootmap_pages;
- }
-
- high_memory = __va(memend_pfn << PAGE_SHIFT);
-
- /*
- * This doesn't seem to be used by the Linux memory
- * manager any more. If we can get rid of it, we
- * also get rid of some of the stuff above as well.
- *
- * Note: max_low_pfn and max_pfn reflect the number
- * of _pages_ in the system, not the maximum PFN.
- */
- max_low_pfn = memend_pfn - O_PFN_DOWN(PHYS_OFFSET);
- max_pfn = memend_pfn - O_PFN_DOWN(PHYS_OFFSET);
-
- return bootmem_pages;
-}
-
static int __init check_initrd(struct meminfo *mi)
{
int initrd_node = -2;
@@ -266,9 +182,8 @@ static int __init check_initrd(struct meminfo *mi)
/*
* Reserve the various regions of node 0
*/
-static __init void reserve_node_zero(unsigned int bootmap_pfn, unsigned int bootmap_pages)
+static __init void reserve_node_zero(pg_data_t *pgdat)
{
- pg_data_t *pgdat = NODE_DATA(0);
unsigned long res_size = 0;
/*
@@ -289,13 +204,6 @@ static __init void reserve_node_zero(unsigned int bootmap_pfn, unsigned int boot
PTRS_PER_PGD * sizeof(pgd_t));
/*
- * And don't forget to reserve the allocator bitmap,
- * which will be freed later.
- */
- reserve_bootmem_node(pgdat, bootmap_pfn << PAGE_SHIFT,
- bootmap_pages << PAGE_SHIFT);
-
- /*
* Hmm... This should go elsewhere, but we really really need to
* stop things allocating the low memory; ideally we need a better
* implementation of GFP_DMA which does not assume that DMA-able
@@ -324,183 +232,293 @@ static __init void reserve_node_zero(unsigned int bootmap_pfn, unsigned int boot
reserve_bootmem_node(pgdat, PHYS_OFFSET, res_size);
}
-/*
- * Register all available RAM in this node with the bootmem allocator.
- */
-static inline void free_bootmem_node_bank(int node, struct meminfo *mi)
+void __init build_mem_type_table(void);
+void __init create_mapping(struct map_desc *md);
+
+static unsigned long __init
+bootmem_init_node(int node, int initrd_node, struct meminfo *mi)
{
- pg_data_t *pgdat = NODE_DATA(node);
- int bank;
+ unsigned long zone_size[MAX_NR_ZONES], zhole_size[MAX_NR_ZONES];
+ unsigned long start_pfn, end_pfn, boot_pfn;
+ unsigned int boot_pages;
+ pg_data_t *pgdat;
+ int i;
- for (bank = 0; bank < mi->nr_banks; bank++)
- if (mi->bank[bank].node == node)
- free_bootmem_node(pgdat, mi->bank[bank].start,
- mi->bank[bank].size);
-}
+ start_pfn = -1UL;
+ end_pfn = 0;
-/*
- * Initialise the bootmem allocator for all nodes. This is called
- * early during the architecture specific initialisation.
- */
-static void __init bootmem_init(struct meminfo *mi)
-{
- struct node_info node_info[MAX_NUMNODES], *np = node_info;
- unsigned int bootmap_pages, bootmap_pfn, map_pg;
- int node, initrd_node;
+ /*
+ * Calculate the pfn range, and map the memory banks for this node.
+ */
+ for_each_nodebank(i, mi, node) {
+ unsigned long start, end;
+ struct map_desc map;
+
+ start = mi->bank[i].start >> PAGE_SHIFT;
+ end = (mi->bank[i].start + mi->bank[i].size) >> PAGE_SHIFT;
+
+ if (start_pfn > start)
+ start_pfn = start;
+ if (end_pfn < end)
+ end_pfn = end;
- bootmap_pages = find_memend_and_nodes(mi, np);
- bootmap_pfn = find_bootmap_pfn(0, mi, bootmap_pages);
- initrd_node = check_initrd(mi);
+ map.pfn = __phys_to_pfn(mi->bank[i].start);
+ map.virtual = __phys_to_virt(mi->bank[i].start);
+ map.length = mi->bank[i].size;
+ map.type = MT_MEMORY;
- map_pg = bootmap_pfn;
+ create_mapping(&map);
+ }
/*
- * Initialise the bootmem nodes.
- *
- * What we really want to do is:
- *
- * unmap_all_regions_except_kernel();
- * for_each_node_in_reverse_order(node) {
- * map_node(node);
- * allocate_bootmem_map(node);
- * init_bootmem_node(node);
- * free_bootmem_node(node);
- * }
- *
- * but this is a 2.5-type change. For now, we just set
- * the nodes up in reverse order.
- *
- * (we could also do with rolling bootmem_init and paging_init
- * into one generic "memory_init" type function).
+ * If there is no memory in this node, ignore it.
*/
- np += num_online_nodes() - 1;
- for (node = num_online_nodes() - 1; node >= 0; node--, np--) {
- /*
- * If there are no pages in this node, ignore it.
- * Note that node 0 must always have some pages.
- */
- if (np->end == 0 || !node_online(node)) {
- if (node == 0)
- BUG();
- continue;
- }
+ if (end_pfn == 0)
+ return end_pfn;
- /*
- * Initialise the bootmem allocator.
- */
- init_bootmem_node(NODE_DATA(node), map_pg, np->start, np->end);
- free_bootmem_node_bank(node, mi);
- map_pg += np->bootmap_pages;
+ /*
+ * Allocate the bootmem bitmap page.
+ */
+ boot_pages = bootmem_bootmap_pages(end_pfn - start_pfn);
+ boot_pfn = find_bootmap_pfn(node, mi, boot_pages);
- /*
- * If this is node 0, we need to reserve some areas ASAP -
- * we may use bootmem on node 0 to setup the other nodes.
- */
- if (node == 0)
- reserve_node_zero(bootmap_pfn, bootmap_pages);
- }
+ /*
+ * Initialise the bootmem allocator for this node, handing the
+ * memory banks over to bootmem.
+ */
+ node_set_online(node);
+ pgdat = NODE_DATA(node);
+ init_bootmem_node(pgdat, boot_pfn, start_pfn, end_pfn);
+ for_each_nodebank(i, mi, node)
+ free_bootmem_node(pgdat, mi->bank[i].start, mi->bank[i].size);
+
+ /*
+ * Reserve the bootmem bitmap for this node.
+ */
+ reserve_bootmem_node(pgdat, boot_pfn << PAGE_SHIFT,
+ boot_pages << PAGE_SHIFT);
#ifdef CONFIG_BLK_DEV_INITRD
- if (phys_initrd_size && initrd_node >= 0) {
- reserve_bootmem_node(NODE_DATA(initrd_node), phys_initrd_start,
+ /*
+ * If the initrd is in this node, reserve its memory.
+ */
+ if (node == initrd_node) {
+ reserve_bootmem_node(pgdat, phys_initrd_start,
phys_initrd_size);
initrd_start = __phys_to_virt(phys_initrd_start);
initrd_end = initrd_start + phys_initrd_size;
}
#endif
- BUG_ON(map_pg != bootmap_pfn + bootmap_pages);
+ /*
+ * Finally, reserve any node zero regions.
+ */
+ if (node == 0)
+ reserve_node_zero(pgdat);
+
+ /*
+ * initialise the zones within this node.
+ */
+ memset(zone_size, 0, sizeof(zone_size));
+ memset(zhole_size, 0, sizeof(zhole_size));
+
+ /*
+ * The size of this node has already been determined. If we need
+ * to do anything fancy with the allocation of this memory to the
+ * zones, now is the time to do it.
+ */
+ zone_size[0] = end_pfn - start_pfn;
+
+ /*
+ * For each bank in this node, calculate the size of the holes.
+ * holes = node_size - sum(bank_sizes_in_node)
+ */
+ zhole_size[0] = zone_size[0];
+ for_each_nodebank(i, mi, node)
+ zhole_size[0] -= mi->bank[i].size >> PAGE_SHIFT;
+
+ /*
+ * Adjust the sizes according to any special requirements for
+ * this machine type.
+ */
+ arch_adjust_zones(node, zone_size, zhole_size);
+
+ free_area_init_node(node, pgdat, zone_size, start_pfn, zhole_size);
+
+ return end_pfn;
}
-/*
- * paging_init() sets up the page tables, initialises the zone memory
- * maps, and sets up the zero page, bad page and bad page tables.
- */
-void __init paging_init(struct meminfo *mi, struct machine_desc *mdesc)
+static void __init bootmem_init(struct meminfo *mi)
{
- void *zero_page;
- int node;
+ unsigned long addr, memend_pfn = 0;
+ int node, initrd_node, i;
- bootmem_init(mi);
+ /*
+ * Invalidate the node number for empty or invalid memory banks
+ */
+ for (i = 0; i < mi->nr_banks; i++)
+ if (mi->bank[i].size == 0 || mi->bank[i].node >= MAX_NUMNODES)
+ mi->bank[i].node = -1;
memcpy(&meminfo, mi, sizeof(meminfo));
/*
- * allocate the zero page. Note that we count on this going ok.
+ * Clear out all the mappings below the kernel image.
*/
- zero_page = alloc_bootmem_low_pages(PAGE_SIZE);
+ for (addr = 0; addr < MODULE_START; addr += PGDIR_SIZE)
+ pmd_clear(pmd_off_k(addr));
+#ifdef CONFIG_XIP_KERNEL
+ /* The XIP kernel is mapped in the module area -- skip over it */
+ addr = ((unsigned long)&_etext + PGDIR_SIZE - 1) & PGDIR_MASK;
+#endif
+ for ( ; addr < PAGE_OFFSET; addr += PGDIR_SIZE)
+ pmd_clear(pmd_off_k(addr));
/*
- * initialise the page tables.
+ * Clear out all the kernel space mappings, except for the first
+ * memory bank, up to the end of the vmalloc region.
*/
- memtable_init(mi);
- if (mdesc->map_io)
- mdesc->map_io();
- local_flush_tlb_all();
+ for (addr = __phys_to_virt(mi->bank[0].start + mi->bank[0].size);
+ addr < VMALLOC_END; addr += PGDIR_SIZE)
+ pmd_clear(pmd_off_k(addr));
/*
- * initialise the zones within each node
+ * Locate which node contains the ramdisk image, if any.
*/
- for_each_online_node(node) {
- unsigned long zone_size[MAX_NR_ZONES];
- unsigned long zhole_size[MAX_NR_ZONES];
- struct bootmem_data *bdata;
- pg_data_t *pgdat;
- int i;
+ initrd_node = check_initrd(mi);
- /*
- * Initialise the zone size information.
- */
- for (i = 0; i < MAX_NR_ZONES; i++) {
- zone_size[i] = 0;
- zhole_size[i] = 0;
- }
+ /*
+ * Run through each node initialising the bootmem allocator.
+ */
+ for_each_node(node) {
+ unsigned long end_pfn;
- pgdat = NODE_DATA(node);
- bdata = pgdat->bdata;
+ end_pfn = bootmem_init_node(node, initrd_node, mi);
/*
- * The size of this node has already been determined.
- * If we need to do anything fancy with the allocation
- * of this memory to the zones, now is the time to do
- * it.
+ * Remember the highest memory PFN.
*/
- zone_size[0] = bdata->node_low_pfn -
- (bdata->node_boot_start >> PAGE_SHIFT);
+ if (end_pfn > memend_pfn)
+ memend_pfn = end_pfn;
+ }
- /*
- * If this zone has zero size, skip it.
- */
- if (!zone_size[0])
- continue;
+ high_memory = __va(memend_pfn << PAGE_SHIFT);
- /*
- * For each bank in this node, calculate the size of the
- * holes. holes = node_size - sum(bank_sizes_in_node)
- */
- zhole_size[0] = zone_size[0];
- for (i = 0; i < mi->nr_banks; i++) {
- if (mi->bank[i].node != node)
- continue;
+ /*
+ * This doesn't seem to be used by the Linux memory manager any
+ * more, but is used by ll_rw_block. If we can get rid of it, we
+ * also get rid of some of the stuff above as well.
+ *
+ * Note: max_low_pfn and max_pfn reflect the number of _pages_ in
+ * the system, not the maximum PFN.
+ */
+ max_pfn = max_low_pfn = memend_pfn - PHYS_PFN_OFFSET;
+}
- zhole_size[0] -= mi->bank[i].size >> PAGE_SHIFT;
- }
+/*
+ * Set up device the mappings. Since we clear out the page tables for all
+ * mappings above VMALLOC_END, we will remove any debug device mappings.
+ * This means you have to be careful how you debug this function, or any
+ * called function. This means you can't use any function or debugging
+ * method which may touch any device, otherwise the kernel _will_ crash.
+ */
+static void __init devicemaps_init(struct machine_desc *mdesc)
+{
+ struct map_desc map;
+ unsigned long addr;
+ void *vectors;
- /*
- * Adjust the sizes according to any special
- * requirements for this machine type.
- */
- arch_adjust_zones(node, zone_size, zhole_size);
+ /*
+ * Allocate the vector page early.
+ */
+ vectors = alloc_bootmem_low_pages(PAGE_SIZE);
+ BUG_ON(!vectors);
+
+ for (addr = VMALLOC_END; addr; addr += PGDIR_SIZE)
+ pmd_clear(pmd_off_k(addr));
- free_area_init_node(node, pgdat, zone_size,
- bdata->node_boot_start >> PAGE_SHIFT, zhole_size);
+ /*
+ * Map the kernel if it is XIP.
+ * It is always first in the modulearea.
+ */
+#ifdef CONFIG_XIP_KERNEL
+ map.pfn = __phys_to_pfn(CONFIG_XIP_PHYS_ADDR & PGDIR_MASK);
+ map.virtual = MODULE_START;
+ map.length = ((unsigned long)&_etext - map.virtual + ~PGDIR_MASK) & PGDIR_MASK;
+ map.type = MT_ROM;
+ create_mapping(&map);
+#endif
+
+ /*
+ * Map the cache flushing regions.
+ */
+#ifdef FLUSH_BASE
+ map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS);
+ map.virtual = FLUSH_BASE;
+ map.length = PGDIR_SIZE;
+ map.type = MT_CACHECLEAN;
+ create_mapping(&map);
+#endif
+#ifdef FLUSH_BASE_MINICACHE
+ map.pfn = __phys_to_pfn(FLUSH_BASE_PHYS + PGDIR_SIZE);
+ map.virtual = FLUSH_BASE_MINICACHE;
+ map.length = PGDIR_SIZE;
+ map.type = MT_MINICLEAN;
+ create_mapping(&map);
+#endif
+
+ /*
+ * Create a mapping for the machine vectors at the high-vectors
+ * location (0xffff0000). If we aren't using high-vectors, also
+ * create a mapping at the low-vectors virtual address.
+ */
+ map.pfn = __phys_to_pfn(virt_to_phys(vectors));
+ map.virtual = 0xffff0000;
+ map.length = PAGE_SIZE;
+ map.type = MT_HIGH_VECTORS;
+ create_mapping(&map);
+
+ if (!vectors_high()) {
+ map.virtual = 0;
+ map.type = MT_LOW_VECTORS;
+ create_mapping(&map);
}
/*
- * finish off the bad pages once
- * the mem_map is initialised
+ * Ask the machine support to map in the statically mapped devices.
+ */
+ if (mdesc->map_io)
+ mdesc->map_io();
+
+ /*
+ * Finally flush the caches and tlb to ensure that we're in a
+ * consistent state wrt the writebuffer. This also ensures that
+ * any write-allocated cache lines in the vector page are written
+ * back. After this point, we can start to touch devices again.
*/
+ local_flush_tlb_all();
+ flush_cache_all();
+}
+
+/*
+ * paging_init() sets up the page tables, initialises the zone memory
+ * maps, and sets up the zero page, bad page and bad page tables.
+ */
+void __init paging_init(struct meminfo *mi, struct machine_desc *mdesc)
+{
+ void *zero_page;
+
+ build_mem_type_table();
+ bootmem_init(mi);
+ devicemaps_init(mdesc);
+
+ top_pmd = pmd_off_k(0xffff0000);
+
+ /*
+ * allocate the zero page. Note that we count on this going ok.
+ */
+ zero_page = alloc_bootmem_low_pages(PAGE_SIZE);
memzero(zero_page, PAGE_SIZE);
empty_zero_page = virt_to_page(zero_page);
flush_dcache_page(empty_zero_page);
@@ -562,10 +580,7 @@ static void __init free_unused_memmap_node(int node, struct meminfo *mi)
* may not be the case, especially if the user has provided the
* information on the command line.
*/
- for (i = 0; i < mi->nr_banks; i++) {
- if (mi->bank[i].size == 0 || mi->bank[i].node != node)
- continue;
-
+ for_each_nodebank(i, mi, node) {
bank_start = mi->bank[i].start >> PAGE_SHIFT;
if (bank_start < prev_bank_end) {
printk(KERN_ERR "MEM: unordered memory banks. "
diff --git a/arch/arm/mm/ioremap.c b/arch/arm/mm/ioremap.c
index 7110e54..1090139 100644
--- a/arch/arm/mm/ioremap.c
+++ b/arch/arm/mm/ioremap.c
@@ -26,6 +26,7 @@
#include <linux/vmalloc.h>
#include <asm/cacheflush.h>
+#include <asm/hardware.h>
#include <asm/io.h>
#include <asm/tlbflush.h>
@@ -74,7 +75,7 @@ remap_area_pmd(pmd_t * pmd, unsigned long address, unsigned long size,
pgprot = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG | L_PTE_DIRTY | L_PTE_WRITE | flags);
do {
- pte_t * pte = pte_alloc_kernel(&init_mm, pmd, address);
+ pte_t * pte = pte_alloc_kernel(pmd, address);
if (!pte)
return -ENOMEM;
remap_area_pte(pte, address, end - address, address + phys_addr, pgprot);
@@ -96,7 +97,6 @@ remap_area_pages(unsigned long start, unsigned long phys_addr,
phys_addr -= address;
dir = pgd_offset(&init_mm, address);
BUG_ON(address >= end);
- spin_lock(&init_mm.page_table_lock);
do {
pmd_t *pmd = pmd_alloc(&init_mm, dir, address);
if (!pmd) {
@@ -113,7 +113,6 @@ remap_area_pages(unsigned long start, unsigned long phys_addr,
dir++;
} while (address && (address < end));
- spin_unlock(&init_mm.page_table_lock);
flush_cache_vmap(start, end);
return err;
}
@@ -131,8 +130,7 @@ remap_area_pages(unsigned long start, unsigned long phys_addr,
* mapping. See include/asm-arm/proc-armv/pgtable.h for more information.
*/
void __iomem *
-__ioremap(unsigned long phys_addr, size_t size, unsigned long flags,
- unsigned long align)
+__ioremap(unsigned long phys_addr, size_t size, unsigned long flags)
{
void * addr;
struct vm_struct * area;
diff --git a/arch/arm/mm/mm-armv.c b/arch/arm/mm/mm-armv.c
index d125a3d..9e50127 100644
--- a/arch/arm/mm/mm-armv.c
+++ b/arch/arm/mm/mm-armv.c
@@ -1,7 +1,7 @@
/*
* linux/arch/arm/mm/mm-armv.c
*
- * Copyright (C) 1998-2002 Russell King
+ * Copyright (C) 1998-2005 Russell King
*
* This program is free software; you can redistribute it and/or modify
* it under the terms of the GNU General Public License version 2 as
@@ -180,11 +180,6 @@ pgd_t *get_pgd_slow(struct mm_struct *mm)
if (!vectors_high()) {
/*
- * This lock is here just to satisfy pmd_alloc and pte_lock
- */
- spin_lock(&mm->page_table_lock);
-
- /*
* On ARM, first page must always be allocated since it
* contains the machine vectors.
*/
@@ -201,23 +196,14 @@ pgd_t *get_pgd_slow(struct mm_struct *mm)
set_pte(new_pte, *init_pte);
pte_unmap_nested(init_pte);
pte_unmap(new_pte);
-
- spin_unlock(&mm->page_table_lock);
}
return new_pgd;
no_pte:
- spin_unlock(&mm->page_table_lock);
pmd_free(new_pmd);
- free_pages((unsigned long)new_pgd, 2);
- return NULL;
-
no_pmd:
- spin_unlock(&mm->page_table_lock);
free_pages((unsigned long)new_pgd, 2);
- return NULL;
-
no_pgd:
return NULL;
}
@@ -243,6 +229,7 @@ void free_pgd_slow(pgd_t *pgd)
pte = pmd_page(*pmd);
pmd_clear(pmd);
dec_page_state(nr_page_table_pages);
+ pte_lock_deinit(pte);
pte_free(pte);
pmd_free(pmd);
free:
@@ -305,16 +292,6 @@ alloc_init_page(unsigned long virt, unsigned long phys, unsigned int prot_l1, pg
set_pte(ptep, pfn_pte(phys >> PAGE_SHIFT, prot));
}
-/*
- * Clear any PGD mapping. On a two-level page table system,
- * the clearance is done by the middle-level functions (pmd)
- * rather than the top-level (pgd) functions.
- */
-static inline void clear_mapping(unsigned long virt)
-{
- pmd_clear(pmd_off_k(virt));
-}
-
struct mem_types {
unsigned int prot_pte;
unsigned int prot_l1;
@@ -373,11 +350,11 @@ static struct mem_types mem_types[] __initdata = {
/*
* Adjust the PMD section entries according to the CPU in use.
*/
-static void __init build_mem_type_table(void)
+void __init build_mem_type_table(void)
{
struct cachepolicy *cp;
unsigned int cr = get_cr();
- unsigned int user_pgprot;
+ unsigned int user_pgprot, kern_pgprot;
int cpu_arch = cpu_architecture();
int i;
@@ -404,7 +381,7 @@ static void __init build_mem_type_table(void)
}
cp = &cache_policies[cachepolicy];
- user_pgprot = cp->pte;
+ kern_pgprot = user_pgprot = cp->pte;
/*
* ARMv6 and above have extended page tables.
@@ -416,6 +393,7 @@ static void __init build_mem_type_table(void)
*/
mem_types[MT_MEMORY].prot_sect &= ~PMD_BIT4;
mem_types[MT_ROM].prot_sect &= ~PMD_BIT4;
+
/*
* Mark cache clean areas and XIP ROM read only
* from SVC mode and no access from userspace.
@@ -435,32 +413,47 @@ static void __init build_mem_type_table(void)
* (iow, non-global)
*/
user_pgprot |= L_PTE_ASID;
+
+#ifdef CONFIG_SMP
+ /*
+ * Mark memory with the "shared" attribute for SMP systems
+ */
+ user_pgprot |= L_PTE_SHARED;
+ kern_pgprot |= L_PTE_SHARED;
+ mem_types[MT_MEMORY].prot_sect |= PMD_SECT_S;
+#endif
}
+ for (i = 0; i < 16; i++) {
+ unsigned long v = pgprot_val(protection_map[i]);
+ v = (v & ~(L_PTE_BUFFERABLE|L_PTE_CACHEABLE)) | user_pgprot;
+ protection_map[i] = __pgprot(v);
+ }
+
+ mem_types[MT_LOW_VECTORS].prot_pte |= kern_pgprot;
+ mem_types[MT_HIGH_VECTORS].prot_pte |= kern_pgprot;
+
if (cpu_arch >= CPU_ARCH_ARMv5) {
- mem_types[MT_LOW_VECTORS].prot_pte |= cp->pte & PTE_CACHEABLE;
- mem_types[MT_HIGH_VECTORS].prot_pte |= cp->pte & PTE_CACHEABLE;
+#ifndef CONFIG_SMP
+ /*
+ * Only use write-through for non-SMP systems
+ */
+ mem_types[MT_LOW_VECTORS].prot_pte &= ~L_PTE_BUFFERABLE;
+ mem_types[MT_HIGH_VECTORS].prot_pte &= ~L_PTE_BUFFERABLE;
+#endif
} else {
- mem_types[MT_LOW_VECTORS].prot_pte |= cp->pte;
- mem_types[MT_HIGH_VECTORS].prot_pte |= cp->pte;
mem_types[MT_MINICLEAN].prot_sect &= ~PMD_SECT_TEX(1);
}
+ pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
+ L_PTE_DIRTY | L_PTE_WRITE |
+ L_PTE_EXEC | kern_pgprot);
+
mem_types[MT_LOW_VECTORS].prot_l1 |= ecc_mask;
mem_types[MT_HIGH_VECTORS].prot_l1 |= ecc_mask;
mem_types[MT_MEMORY].prot_sect |= ecc_mask | cp->pmd;
mem_types[MT_ROM].prot_sect |= cp->pmd;
- for (i = 0; i < 16; i++) {
- unsigned long v = pgprot_val(protection_map[i]);
- v = (v & ~(PTE_BUFFERABLE|PTE_CACHEABLE)) | user_pgprot;
- protection_map[i] = __pgprot(v);
- }
-
- pgprot_kernel = __pgprot(L_PTE_PRESENT | L_PTE_YOUNG |
- L_PTE_DIRTY | L_PTE_WRITE |
- L_PTE_EXEC | cp->pte);
-
switch (cp->pmd) {
case PMD_SECT_WT:
mem_types[MT_CACHECLEAN].prot_sect |= PMD_SECT_WT;
@@ -483,25 +476,25 @@ static void __init build_mem_type_table(void)
* offsets, and we take full advantage of sections and
* supersections.
*/
-static void __init create_mapping(struct map_desc *md)
+void __init create_mapping(struct map_desc *md)
{
unsigned long virt, length;
int prot_sect, prot_l1, domain;
pgprot_t prot_pte;
- long off;
+ unsigned long off = (u32)__pfn_to_phys(md->pfn);
if (md->virtual != vectors_base() && md->virtual < TASK_SIZE) {
printk(KERN_WARNING "BUG: not creating mapping for "
- "0x%08lx at 0x%08lx in user region\n",
- md->physical, md->virtual);
+ "0x%08llx at 0x%08lx in user region\n",
+ __pfn_to_phys((u64)md->pfn), md->virtual);
return;
}
if ((md->type == MT_DEVICE || md->type == MT_ROM) &&
md->virtual >= PAGE_OFFSET && md->virtual < VMALLOC_END) {
- printk(KERN_WARNING "BUG: mapping for 0x%08lx at 0x%08lx "
+ printk(KERN_WARNING "BUG: mapping for 0x%08llx at 0x%08lx "
"overlaps vmalloc space\n",
- md->physical, md->virtual);
+ __pfn_to_phys((u64)md->pfn), md->virtual);
}
domain = mem_types[md->type].domain;
@@ -509,15 +502,40 @@ static void __init create_mapping(struct map_desc *md)
prot_l1 = mem_types[md->type].prot_l1 | PMD_DOMAIN(domain);
prot_sect = mem_types[md->type].prot_sect | PMD_DOMAIN(domain);
+ /*
+ * Catch 36-bit addresses
+ */
+ if(md->pfn >= 0x100000) {
+ if(domain) {
+ printk(KERN_ERR "MM: invalid domain in supersection "
+ "mapping for 0x%08llx at 0x%08lx\n",
+ __pfn_to_phys((u64)md->pfn), md->virtual);
+ return;
+ }
+ if((md->virtual | md->length | __pfn_to_phys(md->pfn))
+ & ~SUPERSECTION_MASK) {
+ printk(KERN_ERR "MM: cannot create mapping for "
+ "0x%08llx at 0x%08lx invalid alignment\n",
+ __pfn_to_phys((u64)md->pfn), md->virtual);
+ return;
+ }
+
+ /*
+ * Shift bits [35:32] of address into bits [23:20] of PMD
+ * (See ARMv6 spec).
+ */
+ off |= (((md->pfn >> (32 - PAGE_SHIFT)) & 0xF) << 20);
+ }
+
virt = md->virtual;
- off = md->physical - virt;
+ off -= virt;
length = md->length;
if (mem_types[md->type].prot_l1 == 0 &&
(virt & 0xfffff || (virt + off) & 0xfffff || (virt + length) & 0xfffff)) {
printk(KERN_WARNING "BUG: map for 0x%08lx at 0x%08lx can not "
"be mapped using pages, ignoring.\n",
- md->physical, md->virtual);
+ __pfn_to_phys(md->pfn), md->virtual);
return;
}
@@ -535,13 +553,22 @@ static void __init create_mapping(struct map_desc *md)
* of the actual domain assignments in use.
*/
if (cpu_architecture() >= CPU_ARCH_ARMv6 && domain == 0) {
- /* Align to supersection boundary */
- while ((virt & ~SUPERSECTION_MASK || (virt + off) &
- ~SUPERSECTION_MASK) && length >= (PGDIR_SIZE / 2)) {
- alloc_init_section(virt, virt + off, prot_sect);
-
- virt += (PGDIR_SIZE / 2);
- length -= (PGDIR_SIZE / 2);
+ /*
+ * Align to supersection boundary if !high pages.
+ * High pages have already been checked for proper
+ * alignment above and they will fail the SUPSERSECTION_MASK
+ * check because of the way the address is encoded into
+ * offset.
+ */
+ if (md->pfn <= 0x100000) {
+ while ((virt & ~SUPERSECTION_MASK ||
+ (virt + off) & ~SUPERSECTION_MASK) &&
+ length >= (PGDIR_SIZE / 2)) {
+ alloc_init_section(virt, virt + off, prot_sect);
+
+ virt += (PGDIR_SIZE / 2);
+ length -= (PGDIR_SIZE / 2);
+ }
}
while (length >= SUPERSECTION_SIZE) {
@@ -601,100 +628,6 @@ void setup_mm_for_reboot(char mode)
}
}
-extern void _stext, _etext;
-
-/*
- * Setup initial mappings. We use the page we allocated for zero page to hold
- * the mappings, which will get overwritten by the vectors in traps_init().
- * The mappings must be in virtual address order.
- */
-void __init memtable_init(struct meminfo *mi)
-{
- struct map_desc *init_maps, *p, *q;
- unsigned long address = 0;
- int i;
-
- build_mem_type_table();
-
- init_maps = p = alloc_bootmem_low_pages(PAGE_SIZE);
-
-#ifdef CONFIG_XIP_KERNEL
- p->physical = CONFIG_XIP_PHYS_ADDR & PMD_MASK;
- p->virtual = (unsigned long)&_stext & PMD_MASK;
- p->length = ((unsigned long)&_etext - p->virtual + ~PMD_MASK) & PMD_MASK;
- p->type = MT_ROM;
- p ++;
-#endif
-
- for (i = 0; i < mi->nr_banks; i++) {
- if (mi->bank[i].size == 0)
- continue;
-
- p->physical = mi->bank[i].start;
- p->virtual = __phys_to_virt(p->physical);
- p->length = mi->bank[i].size;
- p->type = MT_MEMORY;
- p ++;
- }
-
-#ifdef FLUSH_BASE
- p->physical = FLUSH_BASE_PHYS;
- p->virtual = FLUSH_BASE;
- p->length = PGDIR_SIZE;
- p->type = MT_CACHECLEAN;
- p ++;
-#endif
-
-#ifdef FLUSH_BASE_MINICACHE
- p->physical = FLUSH_BASE_PHYS + PGDIR_SIZE;
- p->virtual = FLUSH_BASE_MINICACHE;
- p->length = PGDIR_SIZE;
- p->type = MT_MINICLEAN;
- p ++;
-#endif
-
- /*
- * Go through the initial mappings, but clear out any
- * pgdir entries that are not in the description.
- */
- q = init_maps;
- do {
- if (address < q->virtual || q == p) {
- clear_mapping(address);
- address += PGDIR_SIZE;
- } else {
- create_mapping(q);
-
- address = q->virtual + q->length;
- address = (address + PGDIR_SIZE - 1) & PGDIR_MASK;
-
- q ++;
- }
- } while (address != 0);
-
- /*
- * Create a mapping for the machine vectors at the high-vectors
- * location (0xffff0000). If we aren't using high-vectors, also
- * create a mapping at the low-vectors virtual address.
- */
- init_maps->physical = virt_to_phys(init_maps);
- init_maps->virtual = 0xffff0000;
- init_maps->length = PAGE_SIZE;
- init_maps->type = MT_HIGH_VECTORS;
- create_mapping(init_maps);
-
- if (!vectors_high()) {
- init_maps->virtual = 0;
- init_maps->type = MT_LOW_VECTORS;
- create_mapping(init_maps);
- }
-
- flush_cache_all();
- local_flush_tlb_all();
-
- top_pmd = pmd_off_k(0xffff0000);
-}
-
/*
* Create the architecture specific mappings
*/
diff --git a/arch/arm/mm/proc-v6.S b/arch/arm/mm/proc-v6.S
index caf3b19..92f3ca3 100644
--- a/arch/arm/mm/proc-v6.S
+++ b/arch/arm/mm/proc-v6.S
@@ -12,6 +12,7 @@
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/asm-offsets.h>
+#include <asm/hardware/arm_scu.h>
#include <asm/procinfo.h>
#include <asm/pgtable.h>
@@ -55,7 +56,14 @@ ENTRY(cpu_v6_proc_init)
mov pc, lr
ENTRY(cpu_v6_proc_fin)
- mov pc, lr
+ stmfd sp!, {lr}
+ cpsid if @ disable interrupts
+ bl v6_flush_kern_cache_all
+ mrc p15, 0, r0, c1, c0, 0 @ ctrl register
+ bic r0, r0, #0x1000 @ ...i............
+ bic r0, r0, #0x0006 @ .............ca.
+ mcr p15, 0, r0, c1, c0, 0 @ disable caches
+ ldmfd sp!, {pc}
/*
* cpu_v6_reset(loc)
@@ -105,6 +113,9 @@ ENTRY(cpu_v6_dcache_clean_area)
ENTRY(cpu_v6_switch_mm)
mov r2, #0
ldr r1, [r1, #MM_CONTEXT_ID] @ get mm->context.id
+#ifdef CONFIG_SMP
+ orr r0, r0, #2 @ set shared pgtable
+#endif
mcr p15, 0, r2, c7, c5, 6 @ flush BTAC/BTB
mcr p15, 0, r2, c7, c10, 4 @ drain write buffer
mcr p15, 0, r0, c2, c0, 0 @ set TTB 0
@@ -133,7 +144,7 @@ ENTRY(cpu_v6_switch_mm)
ENTRY(cpu_v6_set_pte)
str r1, [r0], #-2048 @ linux version
- bic r2, r1, #0x000007f0
+ bic r2, r1, #0x000003f0
bic r2, r2, #0x00000003
orr r2, r2, #PTE_EXT_AP0 | 2
@@ -184,6 +195,23 @@ cpu_v6_name:
* - cache type register is implemented
*/
__v6_setup:
+#ifdef CONFIG_SMP
+ /* Set up the SCU on core 0 only */
+ mrc p15, 0, r0, c0, c0, 5 @ CPU core number
+ ands r0, r0, #15
+ moveq r0, #0x10000000 @ SCU_BASE
+ orreq r0, r0, #0x00100000
+ ldreq r5, [r0, #SCU_CTRL]
+ orreq r5, r5, #1
+ streq r5, [r0, #SCU_CTRL]
+
+#ifndef CONFIG_CPU_DCACHE_DISABLE
+ mrc p15, 0, r0, c1, c0, 1 @ Enable SMP/nAMP mode
+ orr r0, r0, #0x20
+ mcr p15, 0, r0, c1, c0, 1
+#endif
+#endif
+
mov r0, #0
mcr p15, 0, r0, c7, c14, 0 @ clean+invalidate D cache
mcr p15, 0, r0, c7, c5, 0 @ invalidate I cache
@@ -191,6 +219,9 @@ __v6_setup:
mcr p15, 0, r0, c7, c10, 4 @ drain write buffer
mcr p15, 0, r0, c8, c7, 0 @ invalidate I + D TLBs
mcr p15, 0, r0, c2, c0, 2 @ TTB control register
+#ifdef CONFIG_SMP
+ orr r4, r4, #2 @ set shared pgtable
+#endif
mcr p15, 0, r4, c2, c0, 1 @ load TTB1
#ifdef CONFIG_VFP
mrc p15, 0, r0, c1, c0, 2
OpenPOWER on IntegriCloud