diff options
Diffstat (limited to 'arch/mips/mm')
| -rw-r--r-- | arch/mips/mm/c-r4k.c | 17 | ||||
| -rw-r--r-- | arch/mips/mm/dma-default.c | 2 | ||||
| -rw-r--r-- | arch/mips/mm/pg-r4k.c | 66 | ||||
| -rw-r--r-- | arch/mips/mm/tlbex.c | 252 |
4 files changed, 152 insertions, 185 deletions
diff --git a/arch/mips/mm/c-r4k.c b/arch/mips/mm/c-r4k.c index 9355f1c..02bd180 100644 --- a/arch/mips/mm/c-r4k.c +++ b/arch/mips/mm/c-r4k.c @@ -449,7 +449,7 @@ static inline void local_r4k_flush_cache_page(void *args) * If the page isn't marked valid, the page cannot possibly be * in the cache. */ - if (!(pte_val(*ptep) & _PAGE_PRESENT)) + if (!(pte_present(*ptep))) return; if ((mm == current->active_mm) && (pte_val(*ptep) & _PAGE_VALID)) @@ -468,8 +468,6 @@ static inline void local_r4k_flush_cache_page(void *args) if (cpu_has_dc_aliases || (exec && !cpu_has_ic_fills_f_dc)) { r4k_blast_dcache_page(addr); - if (exec && !cpu_icache_snoops_remote_store) - r4k_blast_scache_page(addr); } if (exec) { if (vaddr && cpu_has_vtag_icache && mm == current->active_mm) { @@ -533,13 +531,6 @@ static inline void local_r4k_flush_icache_range(void *args) R4600_HIT_CACHEOP_WAR_IMPL; protected_blast_dcache_range(start, end); } - - if (!cpu_icache_snoops_remote_store && scache_size) { - if (end - start > scache_size) - r4k_blast_scache(); - else - protected_blast_scache_range(start, end); - } } if (end - start > icache_size) @@ -598,7 +589,7 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) if (size >= scache_size) r4k_blast_scache(); else - blast_scache_range(addr, addr + size); + blast_inv_scache_range(addr, addr + size); return; } @@ -606,7 +597,7 @@ static void r4k_dma_cache_inv(unsigned long addr, unsigned long size) r4k_blast_dcache(); } else { R4600_HIT_CACHEOP_WAR_IMPL; - blast_dcache_range(addr, addr + size); + blast_inv_dcache_range(addr, addr + size); } bc_inv(addr, size); @@ -989,6 +980,8 @@ static void __init probe_pcache(void) case CPU_AU1100: case CPU_AU1550: case CPU_AU1200: + case CPU_AU1210: + case CPU_AU1250: c->icache.flags |= MIPS_CACHE_IC_F_DC; break; } diff --git a/arch/mips/mm/dma-default.c b/arch/mips/mm/dma-default.c index 810535d..ae39dd8 100644 --- a/arch/mips/mm/dma-default.c +++ b/arch/mips/mm/dma-default.c @@ -383,7 +383,7 @@ void dma_cache_sync(struct device *dev, void *vaddr, size_t size, BUG_ON(direction == DMA_NONE); if (!plat_device_is_coherent(dev)) - dma_cache_wback_inv((unsigned long)vaddr, size); + __dma_sync((unsigned long)vaddr, size, direction); } EXPORT_SYMBOL(dma_cache_sync); diff --git a/arch/mips/mm/pg-r4k.c b/arch/mips/mm/pg-r4k.c index 4f770ac..9185fbf 100644 --- a/arch/mips/mm/pg-r4k.c +++ b/arch/mips/mm/pg-r4k.c @@ -4,6 +4,7 @@ * for more details. * * Copyright (C) 2003, 04, 05 Ralf Baechle (ralf@linux-mips.org) + * Copyright (C) 2007 Maciej W. Rozycki */ #include <linux/init.h> #include <linux/kernel.h> @@ -12,6 +13,7 @@ #include <linux/module.h> #include <linux/proc_fs.h> +#include <asm/bugs.h> #include <asm/cacheops.h> #include <asm/inst.h> #include <asm/io.h> @@ -255,64 +257,58 @@ static inline void build_store_reg(int reg) __build_store_reg(reg); } -static inline void build_addiu_a2_a0(unsigned long offset) +static inline void build_addiu_rt_rs(unsigned int rt, unsigned int rs, + unsigned long offset) { union mips_instruction mi; BUG_ON(offset > 0x7fff); - mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op; - mi.i_format.rs = 4; /* $a0 */ - mi.i_format.rt = 6; /* $a2 */ - mi.i_format.simmediate = offset; + if (cpu_has_64bit_gp_regs && DADDI_WAR && r4k_daddiu_bug()) { + mi.i_format.opcode = addiu_op; + mi.i_format.rs = 0; /* $zero */ + mi.i_format.rt = 25; /* $t9 */ + mi.i_format.simmediate = offset; + emit_instruction(mi); + mi.r_format.opcode = spec_op; + mi.r_format.rs = rs; + mi.r_format.rt = 25; /* $t9 */ + mi.r_format.rd = rt; + mi.r_format.re = 0; + mi.r_format.func = daddu_op; + } else { + mi.i_format.opcode = cpu_has_64bit_gp_regs ? + daddiu_op : addiu_op; + mi.i_format.rs = rs; + mi.i_format.rt = rt; + mi.i_format.simmediate = offset; + } emit_instruction(mi); } -static inline void build_addiu_a2(unsigned long offset) +static inline void build_addiu_a2_a0(unsigned long offset) { - union mips_instruction mi; - - BUG_ON(offset > 0x7fff); - - mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op; - mi.i_format.rs = 6; /* $a2 */ - mi.i_format.rt = 6; /* $a2 */ - mi.i_format.simmediate = offset; + build_addiu_rt_rs(6, 4, offset); /* $a2, $a0, offset */ +} - emit_instruction(mi); +static inline void build_addiu_a2(unsigned long offset) +{ + build_addiu_rt_rs(6, 6, offset); /* $a2, $a2, offset */ } static inline void build_addiu_a1(unsigned long offset) { - union mips_instruction mi; - - BUG_ON(offset > 0x7fff); - - mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op; - mi.i_format.rs = 5; /* $a1 */ - mi.i_format.rt = 5; /* $a1 */ - mi.i_format.simmediate = offset; + build_addiu_rt_rs(5, 5, offset); /* $a1, $a1, offset */ load_offset -= offset; - - emit_instruction(mi); } static inline void build_addiu_a0(unsigned long offset) { - union mips_instruction mi; - - BUG_ON(offset > 0x7fff); - - mi.i_format.opcode = cpu_has_64bit_gp_regs ? daddiu_op : addiu_op; - mi.i_format.rs = 4; /* $a0 */ - mi.i_format.rt = 4; /* $a0 */ - mi.i_format.simmediate = offset; + build_addiu_rt_rs(4, 4, offset); /* $a0, $a0, offset */ store_offset -= offset; - - emit_instruction(mi); } static inline void build_bne(unsigned int *dest) diff --git a/arch/mips/mm/tlbex.c b/arch/mips/mm/tlbex.c index a61246d..d026302 100644 --- a/arch/mips/mm/tlbex.c +++ b/arch/mips/mm/tlbex.c @@ -6,7 +6,7 @@ * Synthesize TLB refill handlers at runtime. * * Copyright (C) 2004,2005,2006 by Thiemo Seufer - * Copyright (C) 2005 Maciej W. Rozycki + * Copyright (C) 2005, 2007 Maciej W. Rozycki * Copyright (C) 2006 Ralf Baechle (ralf@linux-mips.org) * * ... and the days got worse and worse and now you see @@ -19,20 +19,15 @@ * (Condolences to Napoleon XIV) */ -#include <stdarg.h> - -#include <linux/mm.h> #include <linux/kernel.h> #include <linux/types.h> #include <linux/string.h> #include <linux/init.h> -#include <asm/pgtable.h> -#include <asm/cacheflush.h> +#include <asm/bugs.h> #include <asm/mmu_context.h> #include <asm/inst.h> #include <asm/elf.h> -#include <asm/smp.h> #include <asm/war.h> static inline int r45k_bvahwbug(void) @@ -66,7 +61,7 @@ static inline int __maybe_unused r10000_llsc_war(void) * why; it's not an issue caused by the core RTL. * */ -static __init int __attribute__((unused)) m4kc_tlbp_war(void) +static int __init m4kc_tlbp_war(void) { return (current_cpu_data.processor_id & 0xffff00) == (PRID_COMP_MIPS | PRID_IMP_4KC); @@ -140,7 +135,7 @@ struct insn { | (e) << RE_SH \ | (f) << FUNC_SH) -static __initdata struct insn insn_table[] = { +static struct insn insn_table[] __initdata = { { insn_addiu, M(addiu_op, 0, 0, 0, 0, 0), RS | RT | SIMM }, { insn_addu, M(spec_op, 0, 0, 0, 0, addu_op), RS | RT | RD }, { insn_and, M(spec_op, 0, 0, 0, 0, and_op), RS | RT | RD }, @@ -193,7 +188,7 @@ static __initdata struct insn insn_table[] = { #undef M -static __init u32 build_rs(u32 arg) +static u32 __init build_rs(u32 arg) { if (arg & ~RS_MASK) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -201,7 +196,7 @@ static __init u32 build_rs(u32 arg) return (arg & RS_MASK) << RS_SH; } -static __init u32 build_rt(u32 arg) +static u32 __init build_rt(u32 arg) { if (arg & ~RT_MASK) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -209,7 +204,7 @@ static __init u32 build_rt(u32 arg) return (arg & RT_MASK) << RT_SH; } -static __init u32 build_rd(u32 arg) +static u32 __init build_rd(u32 arg) { if (arg & ~RD_MASK) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -217,7 +212,7 @@ static __init u32 build_rd(u32 arg) return (arg & RD_MASK) << RD_SH; } -static __init u32 build_re(u32 arg) +static u32 __init build_re(u32 arg) { if (arg & ~RE_MASK) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -225,7 +220,7 @@ static __init u32 build_re(u32 arg) return (arg & RE_MASK) << RE_SH; } -static __init u32 build_simm(s32 arg) +static u32 __init build_simm(s32 arg) { if (arg > 0x7fff || arg < -0x8000) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -233,7 +228,7 @@ static __init u32 build_simm(s32 arg) return arg & 0xffff; } -static __init u32 build_uimm(u32 arg) +static u32 __init build_uimm(u32 arg) { if (arg & ~IMM_MASK) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -241,7 +236,7 @@ static __init u32 build_uimm(u32 arg) return arg & IMM_MASK; } -static __init u32 build_bimm(s32 arg) +static u32 __init build_bimm(s32 arg) { if (arg > 0x1ffff || arg < -0x20000) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -252,7 +247,7 @@ static __init u32 build_bimm(s32 arg) return ((arg < 0) ? (1 << 15) : 0) | ((arg >> 2) & 0x7fff); } -static __init u32 build_jimm(u32 arg) +static u32 __init build_jimm(u32 arg) { if (arg & ~((JIMM_MASK) << 2)) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -260,7 +255,7 @@ static __init u32 build_jimm(u32 arg) return (arg >> 2) & JIMM_MASK; } -static __init u32 build_func(u32 arg) +static u32 __init build_func(u32 arg) { if (arg & ~FUNC_MASK) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -268,7 +263,7 @@ static __init u32 build_func(u32 arg) return arg & FUNC_MASK; } -static __init u32 build_set(u32 arg) +static u32 __init build_set(u32 arg) { if (arg & ~SET_MASK) printk(KERN_WARNING "TLB synthesizer field overflow\n"); @@ -293,7 +288,7 @@ static void __init build_insn(u32 **buf, enum opcode opc, ...) break; } - if (!ip) + if (!ip || (opc == insn_daddiu && r4k_daddiu_bug())) panic("Unsupported TLB synthesizer instruction %d", opc); op = ip->match; @@ -315,69 +310,69 @@ static void __init build_insn(u32 **buf, enum opcode opc, ...) } #define I_u1u2u3(op) \ - static inline void __init i##op(u32 **buf, unsigned int a, \ + static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \ unsigned int b, unsigned int c) \ { \ build_insn(buf, insn##op, a, b, c); \ } #define I_u2u1u3(op) \ - static inline void __init i##op(u32 **buf, unsigned int a, \ + static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \ unsigned int b, unsigned int c) \ { \ build_insn(buf, insn##op, b, a, c); \ } #define I_u3u1u2(op) \ - static inline void __init i##op(u32 **buf, unsigned int a, \ + static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \ unsigned int b, unsigned int c) \ { \ build_insn(buf, insn##op, b, c, a); \ } #define I_u1u2s3(op) \ - static inline void __init i##op(u32 **buf, unsigned int a, \ + static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \ unsigned int b, signed int c) \ { \ build_insn(buf, insn##op, a, b, c); \ } #define I_u2s3u1(op) \ - static inline void __init i##op(u32 **buf, unsigned int a, \ + static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \ signed int b, unsigned int c) \ { \ build_insn(buf, insn##op, c, a, b); \ } #define I_u2u1s3(op) \ - static inline void __init i##op(u32 **buf, unsigned int a, \ + static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \ unsigned int b, signed int c) \ { \ build_insn(buf, insn##op, b, a, c); \ } #define I_u1u2(op) \ - static inline void __init i##op(u32 **buf, unsigned int a, \ + static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \ unsigned int b) \ { \ build_insn(buf, insn##op, a, b); \ } #define I_u1s2(op) \ - static inline void __init i##op(u32 **buf, unsigned int a, \ + static void __init __maybe_unused i##op(u32 **buf, unsigned int a, \ signed int b) \ { \ build_insn(buf, insn##op, a, b); \ } #define I_u1(op) \ - static inline void __init i##op(u32 **buf, unsigned int a) \ + static void __init __maybe_unused i##op(u32 **buf, unsigned int a) \ { \ build_insn(buf, insn##op, a); \ } #define I_0(op) \ - static inline void __init i##op(u32 **buf) \ + static void __init __maybe_unused i##op(u32 **buf) \ { \ build_insn(buf, insn##op); \ } @@ -457,7 +452,7 @@ struct label { enum label_id lab; }; -static __init void build_label(struct label **lab, u32 *addr, +static void __init build_label(struct label **lab, u32 *addr, enum label_id l) { (*lab)->addr = addr; @@ -466,7 +461,7 @@ static __init void build_label(struct label **lab, u32 *addr, } #define L_LA(lb) \ - static inline void l##lb(struct label **lab, u32 *addr) \ + static inline void __init l##lb(struct label **lab, u32 *addr) \ { \ build_label(lab, addr, label##lb); \ } @@ -525,37 +520,46 @@ L_LA(_r3000_write_probe_fail) #define i_ssnop(buf) i_sll(buf, 0, 0, 1) #define i_ehb(buf) i_sll(buf, 0, 0, 3) -#ifdef CONFIG_64BIT -static __init int __maybe_unused in_compat_space_p(long addr) +static int __init __maybe_unused in_compat_space_p(long addr) { /* Is this address in 32bit compat space? */ +#ifdef CONFIG_64BIT return (((addr) & 0xffffffff00000000L) == 0xffffffff00000000L); +#else + return 1; +#endif } -static __init int __maybe_unused rel_highest(long val) +static int __init __maybe_unused rel_highest(long val) { +#ifdef CONFIG_64BIT return ((((val + 0x800080008000L) >> 48) & 0xffff) ^ 0x8000) - 0x8000; +#else + return 0; +#endif } -static __init int __maybe_unused rel_higher(long val) +static int __init __maybe_unused rel_higher(long val) { +#ifdef CONFIG_64BIT return ((((val + 0x80008000L) >> 32) & 0xffff) ^ 0x8000) - 0x8000; -} +#else + return 0; #endif +} -static __init int rel_hi(long val) +static int __init rel_hi(long val) { return ((((val + 0x8000L) >> 16) & 0xffff) ^ 0x8000) - 0x8000; } -static __init int rel_lo(long val) +static int __init rel_lo(long val) { return ((val & 0xffff) ^ 0x8000) - 0x8000; } -static __init void i_LA_mostly(u32 **buf, unsigned int rs, long addr) +static void __init i_LA_mostly(u32 **buf, unsigned int rs, long addr) { -#ifdef CONFIG_64BIT if (!in_compat_space_p(addr)) { i_lui(buf, rs, rel_highest(addr)); if (rel_higher(addr)) @@ -567,16 +571,18 @@ static __init void i_LA_mostly(u32 **buf, unsigned int rs, long addr) } else i_dsll32(buf, rs, rs, 0); } else -#endif i_lui(buf, rs, rel_hi(addr)); } -static __init void __maybe_unused i_LA(u32 **buf, unsigned int rs, - long addr) +static void __init __maybe_unused i_LA(u32 **buf, unsigned int rs, long addr) { i_LA_mostly(buf, rs, addr); - if (rel_lo(addr)) - i_ADDIU(buf, rs, rs, rel_lo(addr)); + if (rel_lo(addr)) { + if (!in_compat_space_p(addr)) + i_daddiu(buf, rs, rs, rel_lo(addr)); + else + i_addiu(buf, rs, rs, rel_lo(addr)); + } } /* @@ -589,7 +595,7 @@ struct reloc { enum label_id lab; }; -static __init void r_mips_pc16(struct reloc **rel, u32 *addr, +static void __init r_mips_pc16(struct reloc **rel, u32 *addr, enum label_id l) { (*rel)->addr = addr; @@ -614,7 +620,7 @@ static inline void __resolve_relocs(struct reloc *rel, struct label *lab) } } -static __init void resolve_relocs(struct reloc *rel, struct label *lab) +static void __init resolve_relocs(struct reloc *rel, struct label *lab) { struct label *l; @@ -624,7 +630,7 @@ static __init void resolve_relocs(struct reloc *rel, struct label *lab) __resolve_relocs(rel, l); } -static __init void move_relocs(struct reloc *rel, u32 *first, u32 *end, +static void __init move_relocs(struct reloc *rel, u32 *first, u32 *end, long off) { for (; rel->lab != label_invalid; rel++) @@ -632,7 +638,7 @@ static __init void move_relocs(struct reloc *rel, u32 *first, u32 *end, rel->addr += off; } -static __init void move_labels(struct label *lab, u32 *first, u32 *end, +static void __init move_labels(struct label *lab, u32 *first, u32 *end, long off) { for (; lab->lab != label_invalid; lab++) @@ -640,7 +646,7 @@ static __init void move_labels(struct label *lab, u32 *first, u32 *end, lab->addr += off; } -static __init void copy_handler(struct reloc *rel, struct label *lab, +static void __init copy_handler(struct reloc *rel, struct label *lab, u32 *first, u32 *end, u32 *target) { long off = (long)(target - first); @@ -651,7 +657,7 @@ static __init void copy_handler(struct reloc *rel, struct label *lab, move_labels(lab, first, end, off); } -static __init int __maybe_unused insn_has_bdelay(struct reloc *rel, +static int __init __maybe_unused insn_has_bdelay(struct reloc *rel, u32 *addr) { for (; rel->lab != label_invalid; rel++) { @@ -714,6 +720,22 @@ il_bgez(u32 **p, struct reloc **r, unsigned int reg, enum label_id l) i_bgez(p, reg, 0); } +/* + * For debug purposes. + */ +static inline void dump_handler(const u32 *handler, int count) +{ + int i; + + pr_debug("\t.set push\n"); + pr_debug("\t.set noreorder\n"); + + for (i = 0; i < count; i++) + pr_debug("\t%p\t.word 0x%08x\n", &handler[i], handler[i]); + + pr_debug("\t.set pop\n"); +} + /* The only general purpose registers allowed in TLB handlers. */ #define K0 26 #define K1 27 @@ -743,11 +765,11 @@ il_bgez(u32 **p, struct reloc **r, unsigned int reg, enum label_id l) * We deliberately chose a buffer size of 128, so we won't scribble * over anything important on overflow before we panic. */ -static __initdata u32 tlb_handler[128]; +static u32 tlb_handler[128] __initdata; /* simply assume worst case size for labels and relocs */ -static __initdata struct label labels[128]; -static __initdata struct reloc relocs[128]; +static struct label labels[128] __initdata; +static struct reloc relocs[128] __initdata; /* * The R3000 TLB handler is simple. @@ -756,7 +778,6 @@ static void __init build_r3000_tlb_refill_handler(void) { long pgdc = (long)pgd_current; u32 *p; - int i; memset(tlb_handler, 0, sizeof(tlb_handler)); p = tlb_handler; @@ -785,13 +806,9 @@ static void __init build_r3000_tlb_refill_handler(void) pr_info("Synthesized TLB refill handler (%u instructions).\n", (unsigned int)(p - tlb_handler)); - pr_debug("\t.set push\n"); - pr_debug("\t.set noreorder\n"); - for (i = 0; i < (p - tlb_handler); i++) - pr_debug("\t.word 0x%08x\n", tlb_handler[i]); - pr_debug("\t.set pop\n"); - memcpy((void *)ebase, tlb_handler, 0x80); + + dump_handler((u32 *)ebase, 32); } /* @@ -801,7 +818,7 @@ static void __init build_r3000_tlb_refill_handler(void) * other one.To keep things simple, we first assume linear space, * then we relocate it to the final handler layout as needed. */ -static __initdata u32 final_handler[64]; +static u32 final_handler[64] __initdata; /* * Hazards @@ -825,7 +842,7 @@ static __initdata u32 final_handler[64]; * * As if we MIPS hackers wouldn't know how to nop pipelines happy ... */ -static __init void __maybe_unused build_tlb_probe_entry(u32 **p) +static void __init __maybe_unused build_tlb_probe_entry(u32 **p) { switch (current_cpu_type()) { /* Found by experiment: R4600 v2.0 needs this, too. */ @@ -849,7 +866,7 @@ static __init void __maybe_unused build_tlb_probe_entry(u32 **p) */ enum tlb_write_entry { tlb_random, tlb_indexed }; -static __init void build_tlb_write_entry(u32 **p, struct label **l, +static void __init build_tlb_write_entry(u32 **p, struct label **l, struct reloc **r, enum tlb_write_entry wmode) { @@ -860,6 +877,12 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l, case tlb_indexed: tlbw = i_tlbwi; break; } + if (cpu_has_mips_r2) { + i_ehb(p); + tlbw(p); + return; + } + switch (current_cpu_type()) { case CPU_R4000PC: case CPU_R4000SC: @@ -894,6 +917,8 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l, case CPU_AU1500: case CPU_AU1550: case CPU_AU1200: + case CPU_AU1210: + case CPU_AU1250: case CPU_PR4450: i_nop(p); tlbw(p); @@ -935,14 +960,6 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l, tlbw(p); break; - case CPU_4KEC: - case CPU_24K: - case CPU_34K: - case CPU_74K: - i_ehb(p); - tlbw(p); - break; - case CPU_RM9000: /* * When the JTLB is updated by tlbwi or tlbwr, a subsequent @@ -993,7 +1010,7 @@ static __init void build_tlb_write_entry(u32 **p, struct label **l, * TMP and PTR are scratch. * TMP will be clobbered, PTR will hold the pmd entry. */ -static __init void +static void __init build_get_pmde64(u32 **p, struct label **l, struct reloc **r, unsigned int tmp, unsigned int ptr) { @@ -1054,7 +1071,7 @@ build_get_pmde64(u32 **p, struct label **l, struct reloc **r, * BVADDR is the faulting address, PTR is scratch. * PTR will hold the pgd for vmalloc. */ -static __init void +static void __init build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r, unsigned int bvaddr, unsigned int ptr) { @@ -1087,7 +1104,10 @@ build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r, } else { i_LA_mostly(p, ptr, modd); il_b(p, r, label_vmalloc_done); - i_daddiu(p, ptr, ptr, rel_lo(modd)); + if (in_compat_space_p(modd)) + i_addiu(p, ptr, ptr, rel_lo(modd)); + else + i_daddiu(p, ptr, ptr, rel_lo(modd)); } l_vmalloc(l, *p); @@ -1108,7 +1128,10 @@ build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r, } else { i_LA_mostly(p, ptr, swpd); il_b(p, r, label_vmalloc_done); - i_daddiu(p, ptr, ptr, rel_lo(swpd)); + if (in_compat_space_p(swpd)) + i_addiu(p, ptr, ptr, rel_lo(swpd)); + else + i_daddiu(p, ptr, ptr, rel_lo(swpd)); } } @@ -1118,7 +1141,7 @@ build_get_pgd_vmalloc64(u32 **p, struct label **l, struct reloc **r, * TMP and PTR are scratch. * TMP will be clobbered, PTR will hold the pgd entry. */ -static __init void __maybe_unused +static void __init __maybe_unused build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) { long pgdc = (long)pgd_current; @@ -1153,7 +1176,7 @@ build_get_pgde32(u32 **p, unsigned int tmp, unsigned int ptr) #endif /* !CONFIG_64BIT */ -static __init void build_adjust_context(u32 **p, unsigned int ctx) +static void __init build_adjust_context(u32 **p, unsigned int ctx) { unsigned int shift = 4 - (PTE_T_LOG2 + 1) + PAGE_SHIFT - 12; unsigned int mask = (PTRS_PER_PTE / 2 - 1) << (PTE_T_LOG2 + 1); @@ -1179,7 +1202,7 @@ static __init void build_adjust_context(u32 **p, unsigned int ctx) i_andi(p, ctx, ctx, mask); } -static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) +static void __init build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) { /* * Bug workaround for the Nevada. It seems as if under certain @@ -1204,7 +1227,7 @@ static __init void build_get_ptep(u32 **p, unsigned int tmp, unsigned int ptr) i_ADDU(p, ptr, ptr, tmp); /* add in offset */ } -static __init void build_update_entries(u32 **p, unsigned int tmp, +static void __init build_update_entries(u32 **p, unsigned int tmp, unsigned int ptep) { /* @@ -1254,7 +1277,6 @@ static void __init build_r4000_tlb_refill_handler(void) struct reloc *r = relocs; u32 *f; unsigned int final_len; - int i; memset(tlb_handler, 0, sizeof(tlb_handler)); memset(labels, 0, sizeof(labels)); @@ -1356,20 +1378,9 @@ static void __init build_r4000_tlb_refill_handler(void) pr_info("Synthesized TLB refill handler (%u instructions).\n", final_len); - f = final_handler; -#if defined(CONFIG_64BIT) && !defined(CONFIG_CPU_LOONGSON2) - if (final_len > 32) - final_len = 64; - else - f = final_handler + 32; -#endif /* CONFIG_64BIT */ - pr_debug("\t.set push\n"); - pr_debug("\t.set noreorder\n"); - for (i = 0; i < final_len; i++) - pr_debug("\t.word 0x%08x\n", f[i]); - pr_debug("\t.set pop\n"); - memcpy((void *)ebase, final_handler, 0x100); + + dump_handler((u32 *)ebase, 64); } /* @@ -1381,18 +1392,15 @@ static void __init build_r4000_tlb_refill_handler(void) extern void tlb_do_page_fault_0(void); extern void tlb_do_page_fault_1(void); -#define __tlb_handler_align \ - __attribute__((__aligned__(1 << CONFIG_MIPS_L1_CACHE_SHIFT))) - /* * 128 instructions for the fastpath handler is generous and should * never be exceeded. */ #define FASTPATH_SIZE 128 -u32 __tlb_handler_align handle_tlbl[FASTPATH_SIZE]; -u32 __tlb_handler_align handle_tlbs[FASTPATH_SIZE]; -u32 __tlb_handler_align handle_tlbm[FASTPATH_SIZE]; +u32 handle_tlbl[FASTPATH_SIZE] __cacheline_aligned; +u32 handle_tlbs[FASTPATH_SIZE] __cacheline_aligned; +u32 handle_tlbm[FASTPATH_SIZE] __cacheline_aligned; static void __init iPTE_LW(u32 **p, struct label **l, unsigned int pte, unsigned int ptr) @@ -1600,7 +1608,6 @@ static void __init build_r3000_tlb_load_handler(void) u32 *p = handle_tlbl; struct label *l = labels; struct reloc *r = relocs; - int i; memset(handle_tlbl, 0, sizeof(handle_tlbl)); memset(labels, 0, sizeof(labels)); @@ -1623,11 +1630,7 @@ static void __init build_r3000_tlb_load_handler(void) pr_info("Synthesized TLB load handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbl)); - pr_debug("\t.set push\n"); - pr_debug("\t.set noreorder\n"); - for (i = 0; i < (p - handle_tlbl); i++) - pr_debug("\t.word 0x%08x\n", handle_tlbl[i]); - pr_debug("\t.set pop\n"); + dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); } static void __init build_r3000_tlb_store_handler(void) @@ -1635,7 +1638,6 @@ static void __init build_r3000_tlb_store_handler(void) u32 *p = handle_tlbs; struct label *l = labels; struct reloc *r = relocs; - int i; memset(handle_tlbs, 0, sizeof(handle_tlbs)); memset(labels, 0, sizeof(labels)); @@ -1658,11 +1660,7 @@ static void __init build_r3000_tlb_store_handler(void) pr_info("Synthesized TLB store handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbs)); - pr_debug("\t.set push\n"); - pr_debug("\t.set noreorder\n"); - for (i = 0; i < (p - handle_tlbs); i++) - pr_debug("\t.word 0x%08x\n", handle_tlbs[i]); - pr_debug("\t.set pop\n"); + dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); } static void __init build_r3000_tlb_modify_handler(void) @@ -1670,7 +1668,6 @@ static void __init build_r3000_tlb_modify_handler(void) u32 *p = handle_tlbm; struct label *l = labels; struct reloc *r = relocs; - int i; memset(handle_tlbm, 0, sizeof(handle_tlbm)); memset(labels, 0, sizeof(labels)); @@ -1693,11 +1690,7 @@ static void __init build_r3000_tlb_modify_handler(void) pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbm)); - pr_debug("\t.set push\n"); - pr_debug("\t.set noreorder\n"); - for (i = 0; i < (p - handle_tlbm); i++) - pr_debug("\t.word 0x%08x\n", handle_tlbm[i]); - pr_debug("\t.set pop\n"); + dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); } /* @@ -1750,7 +1743,6 @@ static void __init build_r4000_tlb_load_handler(void) u32 *p = handle_tlbl; struct label *l = labels; struct reloc *r = relocs; - int i; memset(handle_tlbl, 0, sizeof(handle_tlbl)); memset(labels, 0, sizeof(labels)); @@ -1783,11 +1775,7 @@ static void __init build_r4000_tlb_load_handler(void) pr_info("Synthesized TLB load handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbl)); - pr_debug("\t.set push\n"); - pr_debug("\t.set noreorder\n"); - for (i = 0; i < (p - handle_tlbl); i++) - pr_debug("\t.word 0x%08x\n", handle_tlbl[i]); - pr_debug("\t.set pop\n"); + dump_handler(handle_tlbl, ARRAY_SIZE(handle_tlbl)); } static void __init build_r4000_tlb_store_handler(void) @@ -1795,7 +1783,6 @@ static void __init build_r4000_tlb_store_handler(void) u32 *p = handle_tlbs; struct label *l = labels; struct reloc *r = relocs; - int i; memset(handle_tlbs, 0, sizeof(handle_tlbs)); memset(labels, 0, sizeof(labels)); @@ -1819,11 +1806,7 @@ static void __init build_r4000_tlb_store_handler(void) pr_info("Synthesized TLB store handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbs)); - pr_debug("\t.set push\n"); - pr_debug("\t.set noreorder\n"); - for (i = 0; i < (p - handle_tlbs); i++) - pr_debug("\t.word 0x%08x\n", handle_tlbs[i]); - pr_debug("\t.set pop\n"); + dump_handler(handle_tlbs, ARRAY_SIZE(handle_tlbs)); } static void __init build_r4000_tlb_modify_handler(void) @@ -1831,7 +1814,6 @@ static void __init build_r4000_tlb_modify_handler(void) u32 *p = handle_tlbm; struct label *l = labels; struct reloc *r = relocs; - int i; memset(handle_tlbm, 0, sizeof(handle_tlbm)); memset(labels, 0, sizeof(labels)); @@ -1856,11 +1838,7 @@ static void __init build_r4000_tlb_modify_handler(void) pr_info("Synthesized TLB modify handler fastpath (%u instructions).\n", (unsigned int)(p - handle_tlbm)); - pr_debug("\t.set push\n"); - pr_debug("\t.set noreorder\n"); - for (i = 0; i < (p - handle_tlbm); i++) - pr_debug("\t.word 0x%08x\n", handle_tlbm[i]); - pr_debug("\t.set pop\n"); + dump_handler(handle_tlbm, ARRAY_SIZE(handle_tlbm)); } void __init build_tlb_refill_handler(void) |
