diff options
Diffstat (limited to 'sys')
-rw-r--r-- | sys/conf/files.powerpc | 1 | ||||
-rw-r--r-- | sys/powerpc/aim/clock.c | 40 | ||||
-rw-r--r-- | sys/powerpc/aim/locore.S | 10 | ||||
-rw-r--r-- | sys/powerpc/aim/machdep.c | 15 | ||||
-rw-r--r-- | sys/powerpc/aim/mmu_oea.c | 82 | ||||
-rw-r--r-- | sys/powerpc/aim/mp_cpudep.c | 231 | ||||
-rw-r--r-- | sys/powerpc/aim/swtch.S | 14 | ||||
-rw-r--r-- | sys/powerpc/aim/trap_subr.S | 71 | ||||
-rw-r--r-- | sys/powerpc/aim/vm_machdep.c | 9 | ||||
-rw-r--r-- | sys/powerpc/booke/trap_subr.S | 4 | ||||
-rw-r--r-- | sys/powerpc/conf/NOTES | 2 | ||||
-rw-r--r-- | sys/powerpc/include/cpufunc.h | 10 | ||||
-rw-r--r-- | sys/powerpc/include/pcpu.h | 15 | ||||
-rw-r--r-- | sys/powerpc/powerpc/intr_machdep.c | 29 | ||||
-rw-r--r-- | sys/powerpc/powerpc/mp_machdep.c | 297 | ||||
-rw-r--r-- | sys/powerpc/powerpc/openpic.c | 22 |
16 files changed, 715 insertions, 137 deletions
diff --git a/sys/conf/files.powerpc b/sys/conf/files.powerpc index 1508582..1cb0bb6 100644 --- a/sys/conf/files.powerpc +++ b/sys/conf/files.powerpc @@ -68,6 +68,7 @@ powerpc/aim/interrupt.c optional aim powerpc/aim/locore.S optional aim no-obj powerpc/aim/machdep.c optional aim powerpc/aim/mmu_oea.c optional aim +powerpc/aim/mp_cpudep.c optional aim smp powerpc/aim/nexus.c optional aim powerpc/aim/ofw_machdep.c optional aim powerpc/aim/ofwmagic.S optional aim diff --git a/sys/powerpc/aim/clock.c b/sys/powerpc/aim/clock.c index 1911220..34c00c8 100644 --- a/sys/powerpc/aim/clock.c +++ b/sys/powerpc/aim/clock.c @@ -61,10 +61,11 @@ __FBSDID("$FreeBSD$"); #include <sys/param.h> #include <sys/systm.h> #include <sys/kernel.h> -#include <sys/sysctl.h> #include <sys/bus.h> -#include <sys/timetc.h> #include <sys/interrupt.h> +#include <sys/pcpu.h> +#include <sys/sysctl.h> +#include <sys/timetc.h> #include <dev/ofw/openfirm.h> @@ -76,11 +77,9 @@ __FBSDID("$FreeBSD$"); /* * Initially we assume a processor with a bus frequency of 12.5 MHz. */ -u_int tickspending; u_long ns_per_tick = 80; static u_long ticks_per_sec = 12500000; static long ticks_per_intr; -static volatile u_long lasttb; static timecounter_get_t decr_get_timecount; @@ -95,7 +94,6 @@ static struct timecounter decr_timecounter = { void decr_intr(struct trapframe *frame) { - u_long tb; long tick; int nticks; @@ -109,36 +107,17 @@ decr_intr(struct trapframe *frame) * Based on the actual time delay since the last decrementer reload, * we arrange for earlier interrupt next time. */ - __asm ("mftb %0; mfdec %1" : "=r"(tb), "=r"(tick)); + __asm ("mfdec %0" : "=r"(tick)); for (nticks = 0; tick < 0; nticks++) tick += ticks_per_intr; mtdec(tick); - /* - * lasttb is used during microtime. Set it to the virtual - * start of this tick interval. - */ - lasttb = tb + tick - ticks_per_intr; - nticks += tickspending; - tickspending = 0; - - /* - * Reenable interrupts - */ -#if 0 - msr = mfmsr(); - mtmsr(msr | PSL_EE | PSL_RI); -#endif - /* - * Do standard timer interrupt stuff. - * Do softclock stuff only on the last iteration. - */ -#if 0 - while (--nticks > 0) { - hardclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); + while (nticks-- > 0) { + if (PCPU_GET(cpuid) == 0) + hardclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); + else + hardclock_cpu(TRAPF_USERMODE(frame)); } -#endif - hardclock(TRAPF_USERMODE(frame), TRAPF_PC(frame)); } void @@ -166,7 +145,6 @@ decr_init(void) ns_per_tick = 1000000000 / ticks_per_sec; ticks_per_intr = ticks_per_sec / hz; - __asm __volatile ("mftb %0" : "=r"(lasttb)); mtdec(ticks_per_intr); mtmsr(msr); diff --git a/sys/powerpc/aim/locore.S b/sys/powerpc/aim/locore.S index 235e13a..ed5c7b8 100644 --- a/sys/powerpc/aim/locore.S +++ b/sys/powerpc/aim/locore.S @@ -77,20 +77,21 @@ .globl kernbase .set kernbase, KERNBASE +#define TMPSTKSZ 8192 /* 8K temporary stack */ + /* * Globals */ .data .align 4 GLOBAL(tmpstk) - .space 8192 + .space TMPSTKSZ GLOBAL(esym) .long 0 /* end of symbol table */ GLOBAL(ofmsr) .long 0, 0, 0, 0, 0 /* msr/sprg0-3 used in Open Firmware */ -#define INTSTK 16384 /* 16K interrupt stack */ #define INTRCNT_COUNT 256 /* max(HROWPIC_IRQMAX,OPENPIC_IRQMAX) */ GLOBAL(intrnames) .space INTRCNT_COUNT * (MAXCOMLEN + 1) * 2 @@ -152,9 +153,8 @@ __start: stw 5,openfirmware_entry@l(8) /* save client interface handler */ mr 3,5 - lis 1,tmpstk@ha - addi 1,1,tmpstk@l - addi 1,1,8192-16 + lis 1,(tmpstk+TMPSTKSZ-16)@ha + addi 1,1,(tmpstk+TMPSTKSZ-16)@l mfmsr 0 lis 9,ofmsr@ha diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c index 15decb7..6a6f407 100644 --- a/sys/powerpc/aim/machdep.c +++ b/sys/powerpc/aim/machdep.c @@ -129,7 +129,8 @@ extern vm_offset_t ksym_start, ksym_end; int cold = 1; -static struct pcpu pcpu0; +struct pcpu __pcpu[MAXCPU]; + static struct trapframe frame0; char machine[] = "powerpc"; @@ -236,6 +237,9 @@ cpu_startup(void *dummy) extern char kernel_text[], _end[]; +#ifdef SMP +extern void *rstcode, *rstsize; +#endif extern void *trapcode, *trapsize; extern void *alitrap, *alisize; extern void *dsitrap, *dsisize; @@ -288,7 +292,7 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp) /* * Set up per-cpu data. */ - pc = &pcpu0; + pc = __pcpu; pcpu_init(pc, 0, sizeof(struct pcpu)); pc->pc_curthread = &thread0; pc->pc_cpuid = 0; @@ -320,7 +324,11 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp) */ mtmsr(mfmsr() & ~(PSL_IR | PSL_DR)); isync(); +#ifdef SMP + bcopy(&rstcode, (void *)EXC_RST, (size_t)&rstsize); +#else bcopy(&trapcode, (void *)EXC_RST, (size_t)&trapsize); +#endif bcopy(&trapcode, (void *)EXC_MCHK, (size_t)&trapsize); bcopy(&dsitrap, (void *)EXC_DSI, (size_t)&dsisize); bcopy(&trapcode, (void *)EXC_ISI, (size_t)&trapsize); @@ -337,8 +345,7 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp) bcopy(&trapcode, (void *)EXC_THRM, (size_t)&trapsize); bcopy(&trapcode, (void *)EXC_BPT, (size_t)&trapsize); #ifdef KDB - bcopy(&dblow, (void *)EXC_RST, (size_t)&dbsize); - bcopy(&dblow, (void *)EXC_MCHK, (size_t)&dbsize); + bcopy(&dblow, (void *)EXC_MCHK, (size_t)&dbsize); bcopy(&dblow, (void *)EXC_PGM, (size_t)&dbsize); bcopy(&dblow, (void *)EXC_TRC, (size_t)&dbsize); bcopy(&dblow, (void *)EXC_BPT, (size_t)&dbsize); diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c index d76fc23..d179b40 100644 --- a/sys/powerpc/aim/mmu_oea.c +++ b/sys/powerpc/aim/mmu_oea.c @@ -147,6 +147,7 @@ __FBSDID("$FreeBSD$"); #include <machine/md_var.h> #include <machine/psl.h> #include <machine/pte.h> +#include <machine/smp.h> #include <machine/sr.h> #include <machine/mmuvar.h> @@ -203,8 +204,6 @@ static struct ofw_map *translations; extern struct pmap ofw_pmap; - - /* * Lock for the pteg and pvo tables. */ @@ -605,6 +604,59 @@ om_cmp(const void *a, const void *b) } void +pmap_cpu_bootstrap(volatile uint32_t *trcp, int ap) +{ + u_int sdr; + int i; + + trcp[0] = 0x1000; + trcp[1] = (uint32_t)&pmap_cpu_bootstrap; + + if (ap) { + __asm __volatile("mtdbatu 0,%0" :: "r"(battable[0].batu)); + __asm __volatile("mtdbatl 0,%0" :: "r"(battable[0].batl)); + isync(); + __asm __volatile("mtibatu 0,%0" :: "r"(battable[0].batu)); + __asm __volatile("mtibatl 0,%0" :: "r"(battable[0].batl)); + isync(); + } + + trcp[0] = 0x1001; + + for (i = 1; i < 4; i++) { + __asm __volatile("mtdbatu %0,%1" :: "n"(i), "r"(0)); + __asm __volatile("mtibatu %0,%1" :: "n"(i), "r"(0)); + isync(); + } + + trcp[0] = 0x1002; + + __asm __volatile("mtdbatu 1,%0" :: "r"(battable[8].batu)); + __asm __volatile("mtdbatl 1,%0" :: "r"(battable[8].batl)); + isync(); + + trcp[0] = 0x1003; + + for (i = 0; i < 16; i++) + mtsrin(i << ADDR_SR_SHFT, EMPTY_SEGMENT); + + trcp[0] = 0x1004; + + __asm __volatile("mtsr %0,%1" :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT)); + __asm __volatile("mtsr %0,%1" :: "n"(KERNEL2_SR), "r"(KERNEL2_SEGMENT)); + __asm __volatile("sync"); + + trcp[0] = 0x1005; + + sdr = (u_int)moea_pteg_table | (moea_pteg_mask >> 10); + __asm __volatile("mtsdr1 %0" :: "r"(sdr)); + isync(); + + trcp[0] = 0x1006; + trcp[1] = sdr; +} + +void moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) { ihandle_t mmui; @@ -612,9 +664,9 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) int sz; int i, j; int ofw_mappings; + uint32_t trace[2]; vm_size_t size, physsz, hwphyssz; vm_offset_t pa, va, off; - u_int batl, batu; /* * Set up BAT0 to map the lowest 256 MB area @@ -647,18 +699,15 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) * Use an IBAT and a DBAT to map the bottom segment of memory * where we are. */ - batu = BATU(0x00000000, BAT_BL_256M, BAT_Vs); - batl = BATL(0x00000000, BAT_M, BAT_PP_RW); __asm (".balign 32; \n" "mtibatu 0,%0; mtibatl 0,%1; isync; \n" "mtdbatu 0,%0; mtdbatl 0,%1; isync" - :: "r"(batu), "r"(batl)); + :: "r"(battable[0].batu), "r"(battable[0].batl)); /* map pci space */ - batu = BATU(0x80000000, BAT_BL_256M, BAT_Vs); - batl = BATL(0x80000000, BAT_I|BAT_G, BAT_PP_RW); - __asm ("mtdbatu 1,%0; mtdbatl 1,%1; isync" - :: "r"(batu), "r"(batl)); + __asm __volatile("mtdbatu 1,%0" :: "r"(battable[8].batu)); + __asm __volatile("mtdbatl 1,%0" :: "r"(battable[8].batl)); + isync(); mem_regions(&pregions, &pregions_sz, ®ions, ®ions_sz); CTR0(KTR_PMAP, "moea_bootstrap: physical memory"); @@ -844,18 +893,7 @@ moea_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend) kernel_pmap->pm_sr[KERNEL2_SR] = KERNEL2_SEGMENT; kernel_pmap->pm_active = ~0; - /* - * Initialize hardware. - */ - for (i = 0; i < 16; i++) { - mtsrin(i << ADDR_SR_SHFT, EMPTY_SEGMENT); - } - __asm __volatile ("mtsr %0,%1" - :: "n"(KERNEL_SR), "r"(KERNEL_SEGMENT)); - __asm __volatile ("mtsr %0,%1" - :: "n"(KERNEL2_SR), "r"(KERNEL2_SEGMENT)); - __asm __volatile ("sync; mtsdr1 %0; isync" - :: "r"((u_int)moea_pteg_table | (moea_pteg_mask >> 10))); + pmap_cpu_bootstrap(trace, 0); tlbia(); pmap_bootstrapped++; diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c new file mode 100644 index 0000000..5b867dd --- /dev/null +++ b/sys/powerpc/aim/mp_cpudep.c @@ -0,0 +1,231 @@ +/*- + * Copyright (c) 2008 Marcel Moolenaar + * All rights reserved. + * + * Redistribution and use in source and binary forms, with or without + * modification, are permitted provided that the following conditions + * are met: + * + * 1. Redistributions of source code must retain the above copyright + * notice, this list of conditions and the following disclaimer. + * 2. Redistributions in binary form must reproduce the above copyright + * notice, this list of conditions and the following disclaimer in the + * documentation and/or other materials provided with the distribution. + * + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + */ + +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + +#include <sys/param.h> +#include <sys/systm.h> +#include <sys/kernel.h> +#include <sys/bus.h> +#include <sys/pcpu.h> +#include <sys/proc.h> +#include <sys/smp.h> + +#include <machine/bat.h> +#include <machine/bus.h> +#include <machine/cpu.h> +#include <machine/hid.h> +#include <machine/intr_machdep.h> +#include <machine/pcb.h> +#include <machine/psl.h> +#include <machine/smp.h> +#include <machine/spr.h> +#include <machine/trap_aim.h> + +#include <dev/ofw/openfirm.h> +#include <machine/ofw_machdep.h> + +extern void *rstcode; + +void *ap_pcpu; + +static int +powerpc_smp_fill_cpuref(struct cpuref *cpuref, phandle_t cpu) +{ + int cpuid, res; + + cpuref->cr_hwref = cpu; + res = OF_getprop(cpu, "reg", &cpuid, sizeof(cpuid)); + if (res < 0) + return (ENOENT); + + cpuref->cr_cpuid = cpuid & 0xff; + return (0); +} + +int +powerpc_smp_first_cpu(struct cpuref *cpuref) +{ + char buf[8]; + phandle_t cpu, dev, root; + int res; + + root = OF_peer(0); + + dev = OF_child(root); + while (dev != 0) { + res = OF_getprop(dev, "name", buf, sizeof(buf)); + if (res > 0 && strcmp(buf, "cpus") == 0) + break; + dev = OF_peer(dev); + } + if (dev == 0) + return (ENOENT); + + cpu = OF_child(dev); + while (cpu != 0) { + res = OF_getprop(cpu, "device_type", buf, sizeof(buf)); + if (res > 0 && strcmp(buf, "cpu") == 0) + break; + cpu = OF_peer(cpu); + } + if (cpu == 0) + return (ENOENT); + + return (powerpc_smp_fill_cpuref(cpuref, cpu)); +} + +int +powerpc_smp_next_cpu(struct cpuref *cpuref) +{ + char buf[8]; + phandle_t cpu; + int res; + + cpu = OF_peer(cpuref->cr_hwref); + while (cpu != 0) { + res = OF_getprop(cpu, "device_type", buf, sizeof(buf)); + if (res > 0 && strcmp(buf, "cpu") == 0) + break; + cpu = OF_peer(cpu); + } + if (cpu == 0) + return (ENOENT); + + return (powerpc_smp_fill_cpuref(cpuref, cpu)); +} + +int +powerpc_smp_get_bsp(struct cpuref *cpuref) +{ + ihandle_t inst; + phandle_t bsp, chosen; + int res; + + chosen = OF_finddevice("/chosen"); + if (chosen == 0) + return (ENXIO); + + res = OF_getprop(chosen, "cpu", &inst, sizeof(inst)); + if (res < 0) + return (ENXIO); + + bsp = OF_instance_to_package(inst); + return (powerpc_smp_fill_cpuref(cpuref, bsp)); +} + +uint32_t +cpudep_ap_bootstrap(volatile uint32_t *trcp) +{ + uint32_t hid, msr, sp; + + trcp[0] = 0x2000; + trcp[1] = (uint32_t)&cpudep_ap_bootstrap; + + __asm __volatile("mtsprg 0, %0" :: "r"(ap_pcpu)); + __asm __volatile("sync"); + + trcp[0] = 0x2001; + trcp[1] = (uint32_t)pcpup; + + hid = mfspr(SPR_HID0); + hid &= ~(HID0_ICE | HID0_DCE); + hid &= ~(HID0_DOZE | HID0_NAP | HID0_SLEEP); + mtspr(SPR_HID0, hid); + isync(); + + trcp[0] = 0x2002; + trcp[1] = hid; + + hid |= HID0_ICFI | HID0_DCFI; + hid |= HID0_ICE | HID0_DCE; + mtspr(SPR_HID0, hid); + isync(); + + trcp[0] = 0x2003; + trcp[1] = hid; + + msr = PSL_IR | PSL_DR | PSL_ME; + mtmsr(msr); + isync(); + + trcp[0] = 0x2004; + trcp[1] = msr; + + hid |= HID0_NAP | HID0_DPM; + mtspr(SPR_HID0, hid); + isync(); + + trcp[0] = 0x2005; + trcp[1] = hid; + + pcpup->pc_curthread = pcpup->pc_idlethread; + pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb; + sp = pcpup->pc_curpcb->pcb_sp; + + trcp[0] = 0x2006; + trcp[1] = sp; + + return (sp); +} + +int +powerpc_smp_start_cpu(struct pcpu *pc) +{ + phandle_t cpu; + volatile uint32_t *trcp; + volatile uint8_t *rstvec; + uint32_t trace; + int res, reset, timeout; + + cpu = pc->pc_hwref; + res = OF_getprop(cpu, "soft-reset", &reset, sizeof(reset)); + if (res < 0) + return (ENXIO); + + trcp = (uint32_t *)(EXC_RST + 4); + trace = *trcp; + + ap_pcpu = pc; + + rstvec = (uint8_t *)(0x80000000 + reset); + + *rstvec = 4; + __asm __volatile("sync"); + DELAY(1); + *rstvec = 0; + __asm __volatile("sync"); + + timeout = 1000; + while (!pc->pc_awake && timeout--) + DELAY(100); + + if (!pc->pc_awake) + printf("XXX: timeout (trace=%x; data=%x)\n", trcp[0], trcp[1]); + + return (0); +} diff --git a/sys/powerpc/aim/swtch.S b/sys/powerpc/aim/swtch.S index 70b5b4f..0d6875d 100644 --- a/sys/powerpc/aim/swtch.S +++ b/sys/powerpc/aim/swtch.S @@ -67,6 +67,13 @@ #include <machine/asm.h> /* + * void cpu_throw(struct thread *old, struct thread *new) + */ +ENTRY(cpu_throw) + mr %r15, %r4 + b cpu_switchin + +/* * void cpu_switch(struct thread *old, * struct thread *new, * struct mutex *mtx); @@ -94,7 +101,8 @@ ENTRY(cpu_switch) mr %r14,%r3 /* Copy the old thread ptr... */ mr %r15,%r4 /* and the new thread ptr in scratch */ - lwz %r6,PCB_FLAGS(%r5) /* Save FPU context if needed */ + lwz %r6,PCB_FLAGS(%r5) + /* Save FPU context if needed */ andi. %r6, %r6, PCB_FPU beq .L1 bl save_fpu @@ -102,6 +110,7 @@ ENTRY(cpu_switch) .L1: bl pmap_deactivate /* Deactivate the current pmap */ +cpu_switchin: mr %r3,%r15 /* Get new thread ptr */ bl pmap_activate /* Activate the new address space */ @@ -110,7 +119,8 @@ ENTRY(cpu_switch) lwz %r17,TD_PCB(%r15) /* Store new current PCB */ stw %r17,PC_CURPCB(%r7) - lwz %r6, PCB_FLAGS(%r17) /* Restore FPU context if needed */ + lwz %r6, PCB_FLAGS(%r17) + /* Restore FPU context if needed */ andi. %r6, %r6, PCB_FPU beq .L2 mr %r3,%r15 /* Pass curthread to enable_fpu */ diff --git a/sys/powerpc/aim/trap_subr.S b/sys/powerpc/aim/trap_subr.S index 03fb7ec..2c0e78f 100644 --- a/sys/powerpc/aim/trap_subr.S +++ b/sys/powerpc/aim/trap_subr.S @@ -228,20 +228,67 @@ mfsprg2 %r2; /* restore r2 & r3 */ \ mfsprg3 %r3 -#ifdef KDB +#ifdef SMP /* - * Define the kdb debugger stack + * Processor reset exception handler. These are typically + * the first instructions the processor executes after a + * software reset. */ - .data -GLOBAL(dbstk) - .space INTSTK+8 /* kdb stack */ + .globl CNAME(rstcode), CNAME(rstsize) +CNAME(rstcode): + bl 1f + + /* We use this space for tracing purposes. */ + .long 0 + .long 0 + +1: + mflr %r2 + mfmsr %r3 + stw %r2,0(%r2) /* trace: 0x104 - we're here. */ + stw %r3,4(%r2) /* trace data: MSR */ + sync + + lis %r1,(tmpstk+TMPSTKSZ-16)@ha + addi %r1,%r1,(tmpstk+TMPSTKSZ-16)@l + + addi %r3,%r2,4 + stw %r3,0(%r1) + sync + stw %r3,0(%r2) /* trace: 0x108 - stack is writable */ + stw %r1,4(%r2) /* trace data: SP */ + sync + + mr %r3,%r2 + lis %r4,1@l + bla CNAME(pmap_cpu_bootstrap) + + addi %r3,%r2,8 + stw %r3,0(%r2) /* trace 0x10c - back from 1st call */ + sync + + mr %r3,%r2 + bla CNAME(cpudep_ap_bootstrap) + mr %r1,%r3 + + addi %r3,%r2,12 + stw %r3,0(%r2) /* trace 0x110 - back from 2nd call */ + stw %r1,4(%r2) /* trace data: SP */ + + mr %r3,%r2 + bla CNAME(machdep_ap_bootstrap) + + /* Should not be reached */ +9: + b 9b +CNAME(rstsize) = . - CNAME(rstcode) #endif /* * This code gets copied to all the trap vectors * (except ISI/DSI, ALI, and the interrupts) */ - .text + .globl CNAME(trapcode),CNAME(trapsize) CNAME(trapcode): mtsprg1 %r1 /* save SP */ @@ -385,8 +432,8 @@ disitrap: stw %r30,(PC_DBSAVE +CPUSAVE_R30)(%r1) /* save r30 */ lwz %r31,(PC_DISISAVE+CPUSAVE_R31)(%r1) /* get r31 */ stw %r31,(PC_DBSAVE +CPUSAVE_R31)(%r1) /* save r31 */ - lis %r1,dbstk+INTSTK@ha /* get new SP */ - addi %r1,%r1,dbstk+INTSTK@l + lis %r1,(tmpstk+TMPSTKSZ-16)@ha /* get new SP */ + addi %r1,%r1,(tmpstk+TMPSTKSZ-16)@l b dbtrap #endif @@ -457,8 +504,8 @@ CNAME(vectrapsize) = .-CNAME(vectrap) /* * Deliberate entry to dbtrap */ - .globl CNAME(ppc_db_trap) -CNAME(ppc_db_trap): + .globl CNAME(breakpoint) +CNAME(breakpoint): mtsprg1 %r1 mfmsr %r3 mtsrr1 %r3 @@ -533,8 +580,8 @@ CNAME(dblow): stw %r30,(PC_DBSAVE+CPUSAVE_R30)(%r1) /* free r30 */ stw %r31,(PC_DBSAVE+CPUSAVE_R31)(%r1) /* free r31 */ mflr %r28 /* save LR */ - lis %r1,dbstk+INTSTK@ha /* get new SP */ - addi %r1,%r1,dbstk+INTSTK@l + lis %r1,(tmpstk+TMPSTKSZ-16)@ha /* get new SP */ + addi %r1,%r1,(tmpstk+TMPSTKSZ-16)@l bla dbtrap CNAME(dbsize) = .-CNAME(dblow) #endif /* KDB */ diff --git a/sys/powerpc/aim/vm_machdep.c b/sys/powerpc/aim/vm_machdep.c index 5bd1e2b..33b3382 100644 --- a/sys/powerpc/aim/vm_machdep.c +++ b/sys/powerpc/aim/vm_machdep.c @@ -192,15 +192,6 @@ cpu_exit(td) { } -/* Temporary helper */ -void -cpu_throw(struct thread *old, struct thread *new) -{ - - cpu_switch(old, new, old->td_lock); - panic("cpu_throw() didn't"); -} - /* * Reset back to firmware. */ diff --git a/sys/powerpc/booke/trap_subr.S b/sys/powerpc/booke/trap_subr.S index f9da569..f22a0ce 100644 --- a/sys/powerpc/booke/trap_subr.S +++ b/sys/powerpc/booke/trap_subr.S @@ -795,8 +795,8 @@ CNAME(asttrapexit): /* * Deliberate entry to dbtrap */ - .globl CNAME(ppc_db_trap) -CNAME(ppc_db_trap): + .globl CNAME(breakpoint) +CNAME(breakpoint): mtsprg1 %r1 mfmsr %r3 mtsrr1 %r3 diff --git a/sys/powerpc/conf/NOTES b/sys/powerpc/conf/NOTES index a399e9c..090f388 100644 --- a/sys/powerpc/conf/NOTES +++ b/sys/powerpc/conf/NOTES @@ -61,8 +61,6 @@ nodevice dcons_crom ##################################################################### # Options we don't want to deal with -nooption SMP -nooption ADAPTIVE_SX nooption PPC_DEBUG nooption PPC_PROBE_CHIPSET nooption SC_NO_MODE_CHANGE diff --git a/sys/powerpc/include/cpufunc.h b/sys/powerpc/include/cpufunc.h index a58d4c5..6928fc3 100644 --- a/sys/powerpc/include/cpufunc.h +++ b/sys/powerpc/include/cpufunc.h @@ -49,17 +49,9 @@ powerpc_mb(void) struct thread; #ifdef KDB -void ppc_db_trap(void); +void breakpoint(void); #endif -static __inline void -breakpoint(void) -{ -#ifdef KDB - ppc_db_trap(); -#endif -} - /* CPU register mangling inlines */ static __inline void diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h index 3970ad5..af9e54b 100644 --- a/sys/powerpc/include/pcpu.h +++ b/sys/powerpc/include/pcpu.h @@ -40,6 +40,11 @@ struct pmap; int pc_inside_intr; \ struct pmap *pc_curpmap; /* current pmap */ \ struct thread *pc_fputhread; /* current fpu user */ \ + uintptr_t pc_hwref; \ + uint32_t pc_pir; \ + int pc_bsp:1; \ + int pc_awake:1; \ + uint32_t pc_ipimask; \ register_t pc_tempsave[CPUSAVE_LEN]; \ register_t pc_disisave[CPUSAVE_LEN]; \ register_t pc_dbsave[CPUSAVE_LEN]; @@ -112,18 +117,18 @@ struct pmap; int pc_md_placeholder #endif -#define PCPUP ((struct pcpu *) powerpc_get_pcpup()) +#define pcpup ((struct pcpu *) powerpc_get_pcpup()) -#define PCPU_GET(member) (PCPUP->pc_ ## member) +#define PCPU_GET(member) (pcpup->pc_ ## member) /* * XXX The implementation of this operation should be made atomic * with respect to preemption. */ -#define PCPU_ADD(member, value) (PCPUP->pc_ ## member += (value)) +#define PCPU_ADD(member, value) (pcpup->pc_ ## member += (value)) #define PCPU_INC(member) PCPU_ADD(member, 1) -#define PCPU_PTR(member) (&PCPUP->pc_ ## member) -#define PCPU_SET(member,value) (PCPUP->pc_ ## member = (value)) +#define PCPU_PTR(member) (&pcpup->pc_ ## member) +#define PCPU_SET(member,value) (pcpup->pc_ ## member = (value)) #endif /* _KERNEL */ diff --git a/sys/powerpc/powerpc/intr_machdep.c b/sys/powerpc/powerpc/intr_machdep.c index d9d7591..dbf3cfc 100644 --- a/sys/powerpc/powerpc/intr_machdep.c +++ b/sys/powerpc/powerpc/intr_machdep.c @@ -78,6 +78,7 @@ #include <machine/frame.h> #include <machine/intr_machdep.h> #include <machine/md_var.h> +#include <machine/smp.h> #include <machine/trap.h> #include "pic_if.h" @@ -99,6 +100,12 @@ static struct powerpc_intr *powerpc_intrs[INTR_VECTORS]; static u_int nvectors; /* Allocated vectors */ static u_int stray_count; +#ifdef SMP +static void *ipi_cookie; +#endif + +static u_int ipi_irq; + device_t pic; static void @@ -190,14 +197,31 @@ powerpc_register_pic(device_t dev, u_int ipi) { pic = dev; + ipi_irq = ipi; } int powerpc_enable_intr(void) { struct powerpc_intr *i; +#ifdef SMP + int error; +#endif int vector; + if (pic == NULL) + panic("no PIC detected\n"); + +#ifdef SMP + /* Install an IPI handler. */ + error = powerpc_setup_intr("IPI", ipi_irq, powerpc_ipi_handler, + NULL, NULL, INTR_TYPE_MISC | INTR_EXCL | INTR_FAST, &ipi_cookie); + if (error) { + printf("unable to setup IPI handler\n"); + return (error); + } +#endif + for (vector = 0; vector < nvectors; vector++) { i = powerpc_intrs[vector]; if (i == NULL) @@ -211,6 +235,11 @@ powerpc_enable_intr(void) PIC_ENABLE(pic, i->irq, vector); } +#ifdef SMP + /* Send ourself a test IPI message. */ + ipi_self(IPI_PPC_TEST); +#endif + return (0); } diff --git a/sys/powerpc/powerpc/mp_machdep.c b/sys/powerpc/powerpc/mp_machdep.c index 0c9dbe0..c21f9db 100644 --- a/sys/powerpc/powerpc/mp_machdep.c +++ b/sys/powerpc/powerpc/mp_machdep.c @@ -1,75 +1,316 @@ /*- - * Copyright (c) 2000 Doug Rabson + * Copyright (c) 2008 Marcel Moolenaar * All rights reserved. * * Redistribution and use in source and binary forms, with or without * modification, are permitted provided that the following conditions * are met: + * * 1. Redistributions of source code must retain the above copyright * notice, this list of conditions and the following disclaimer. * 2. Redistributions in binary form must reproduce the above copyright * notice, this list of conditions and the following disclaimer in the * documentation and/or other materials provided with the distribution. * - * THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND - * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE - * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE - * ARE DISCLAIMED. IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE - * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL - * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS - * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) - * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT - * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY - * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF - * SUCH DAMAGE. - * - * $FreeBSD$ + * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR + * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES + * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. + * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, + * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT + * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, + * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY + * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT + * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF + * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. */ +#include <sys/cdefs.h> +__FBSDID("$FreeBSD$"); + #include <sys/param.h> #include <sys/systm.h> -#include <sys/ktr.h> -#include <sys/proc.h> -#include <sys/lock.h> -#include <sys/malloc.h> -#include <sys/mutex.h> #include <sys/kernel.h> +#include <sys/bus.h> +#include <sys/pcpu.h> +#include <sys/proc.h> +#include <sys/sched.h> #include <sys/smp.h> -#include <vm/vm.h> -#include <vm/pmap.h> -#include <vm/vm_map.h> +#include <machine/bus.h> +#include <machine/cpu.h> +#include <machine/intr_machdep.h> +#include <machine/smp.h> + +#include "pic_if.h" + +extern struct pcpu __pcpu[MAXCPU]; + +volatile static int ap_awake; +volatile static u_int ap_state; +volatile static uint32_t ap_decr; + +int mp_ipi_test = 0; + +void +machdep_ap_bootstrap(volatile uint32_t *trcp) +{ + + trcp[0] = 0x3000; + trcp[1] = (uint32_t)&machdep_ap_bootstrap; + + // __asm __volatile("mtspr 1023,%0" :: "r"(PCPU_GET(cpuid))); + __asm __volatile("mfspr %0,1023" : "=r"(pcpup->pc_pir)); + pcpup->pc_awake = 1; + + while (ap_state == 0) + ; + + __asm __volatile("mtdec %0" :: "r"(ap_decr)); + + ap_awake++; -#include <machine/atomic.h> -#include <machine/pmap.h> + /* Initialize curthread. */ + PCPU_SET(curthread, PCPU_GET(idlethread)); -int boot_cpu_id; + mtmsr(mfmsr() | PSL_EE | PSL_RI); + sched_throw(NULL); +} struct cpu_group * cpu_topo(void) { - return smp_topo_none(); + return (smp_topo_none()); } void cpu_mp_setmaxid(void) { + struct cpuref cpuref; + int error; + + mp_ncpus = 0; + error = powerpc_smp_first_cpu(&cpuref); + while (!error) { + mp_ncpus++; + error = powerpc_smp_next_cpu(&cpuref); + } + /* Sanity. */ + if (mp_ncpus == 0) + mp_ncpus = 1; + + /* + * Set the largest cpuid we're going to use. This is necessary + * for VM initialization. + */ + mp_maxid = min(mp_ncpus, MAXCPU) - 1; } int cpu_mp_probe(void) { - all_cpus = 1; /* needed for MB init code */ - return 0; + + /* + * We're not going to enable SMP if there's only 1 processor. + */ + return (mp_ncpus > 1); } void cpu_mp_start(void) { + struct cpuref bsp, cpu; + struct pcpu *pc; + int error; + + error = powerpc_smp_get_bsp(&bsp); + KASSERT(error == 0, ("Don't know BSP")); + KASSERT(bsp.cr_cpuid == 0, ("%s: cpuid != 0", __func__)); + + error = powerpc_smp_first_cpu(&cpu); + while (!error) { + if (cpu.cr_cpuid >= MAXCPU) { + printf("SMP: cpu%d: skipped -- ID out of range\n", + cpu.cr_cpuid); + goto next; + } + if (all_cpus & (1 << cpu.cr_cpuid)) { + printf("SMP: cpu%d: skipped - duplicate ID\n", + cpu.cr_cpuid); + goto next; + } + if (cpu.cr_cpuid != bsp.cr_cpuid) { + pc = &__pcpu[cpu.cr_cpuid]; + pcpu_init(pc, cpu.cr_cpuid, sizeof(*pc)); + } else { + pc = pcpup; + pc->pc_cpuid = bsp.cr_cpuid; + pc->pc_bsp = 1; + } + pc->pc_cpumask = 1 << pc->pc_cpuid; + pc->pc_hwref = cpu.cr_hwref; + all_cpus |= pc->pc_cpumask; + + next: + error = powerpc_smp_next_cpu(&cpu); + } } void cpu_mp_announce(void) { + struct pcpu *pc; + int i; + + for (i = 0; i <= mp_maxid; i++) { + pc = pcpu_find(i); + if (pc == NULL) + continue; + printf("cpu%d: dev=%x", i, pc->pc_hwref); + if (pc->pc_bsp) + printf(" (BSP)"); + printf("\n"); + } +} + +static void +cpu_mp_unleash(void *dummy) +{ + struct pcpu *pc; + int cpus; + + if (mp_ncpus <= 1) + return; + + if (mp_ipi_test != 1) { + printf("SMP: ERROR: sending of a test IPI failed\n"); + return; + } + + cpus = 0; + smp_cpus = 0; + SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { + cpus++; + pc->pc_other_cpus = all_cpus & ~pc->pc_cpumask; + if (!pc->pc_bsp) { + printf("Waking up CPU %d (dev=%x)\n", pc->pc_cpuid, + pc->pc_hwref); + powerpc_smp_start_cpu(pc); + } else { + __asm __volatile("mfspr %0,1023" : "=r"(pc->pc_pir)); + pc->pc_awake = 1; + } + if (pc->pc_awake) + smp_cpus++; + } + + ap_awake = 1; + __asm __volatile("mfdec %0" : "=r"(ap_decr)); + ap_state++; + + while (ap_awake < smp_cpus) + ; + + if (smp_cpus != cpus || cpus != mp_ncpus) { + printf("SMP: %d CPUs found; %d CPUs usable; %d CPUs woken\n", + mp_ncpus, cpus, smp_cpus); + } + + smp_active = 1; + smp_started = 1; +} + +SYSINIT(start_aps, SI_SUB_SMP, SI_ORDER_FIRST, cpu_mp_unleash, NULL); + +static u_int ipi_msg_cnt[32]; + +int +powerpc_ipi_handler(void *arg) +{ + cpumask_t self; + uint32_t ipimask; + int msg; + + ipimask = atomic_readandclear_32(&(pcpup->pc_ipimask)); + if (ipimask == 0) + return (FILTER_STRAY); + while ((msg = ffs(ipimask) - 1) != -1) { + ipimask &= ~(1u << msg); + ipi_msg_cnt[msg]++; + switch (msg) { + case IPI_AST: + break; + case IPI_PREEMPT: + sched_preempt(curthread); + break; + case IPI_RENDEZVOUS: + smp_rendezvous_action(); + break; + case IPI_STOP: + self = PCPU_GET(cpumask); + savectx(PCPU_GET(curpcb)); + atomic_set_int(&stopped_cpus, self); + while ((started_cpus & self) == 0) + cpu_spinwait(); + atomic_clear_int(&started_cpus, self); + atomic_clear_int(&stopped_cpus, self); + break; + case IPI_PPC_TEST: + mp_ipi_test++; + break; + } + } + + return (FILTER_HANDLED); +} + +static void +ipi_send(struct pcpu *pc, int ipi) +{ + + atomic_set_32(&pc->pc_ipimask, (1 << ipi)); + PIC_IPI(pic, pc->pc_cpuid); +} + +/* Send an IPI to a set of cpus. */ +void +ipi_selected(cpumask_t cpus, int ipi) +{ + struct pcpu *pc; + + SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { + if (cpus & pc->pc_cpumask) + ipi_send(pc, ipi); + } +} + +/* Send an IPI to all CPUs, including myself. */ +void +ipi_all(int ipi) +{ + struct pcpu *pc; + + SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { + ipi_send(pc, ipi); + } +} + +/* Send an IPI to all CPUs EXCEPT myself. */ +void +ipi_all_but_self(int ipi) +{ + struct pcpu *pc; + + SLIST_FOREACH(pc, &cpuhead, pc_allcpu) { + if (pc != pcpup) + ipi_send(pc, ipi); + } +} + +/* Send an IPI to myself. */ +void +ipi_self(int ipi) +{ + + ipi_send(pcpup, ipi); } diff --git a/sys/powerpc/powerpc/openpic.c b/sys/powerpc/powerpc/openpic.c index c778a24..88d2614 100644 --- a/sys/powerpc/powerpc/openpic.c +++ b/sys/powerpc/powerpc/openpic.c @@ -82,7 +82,7 @@ int openpic_attach(device_t dev) { struct openpic_softc *sc; - u_int ipi, irq; + u_int cpu, ipi, irq; u_int32_t x; sc = device_get_softc(dev); @@ -132,6 +132,9 @@ openpic_attach(device_t dev) "Version %s, supports %d CPUs and %d irqs\n", sc->sc_version, sc->sc_ncpu, sc->sc_nirq); + for (cpu = 0; cpu < sc->sc_ncpu; cpu++) + openpic_write(sc, OPENPIC_PCPU_TPR(cpu), 15); + /* Reset and disable all interrupts. */ for (irq = 0; irq < sc->sc_nirq; irq++) { x = irq; /* irq == vector. */ @@ -150,8 +153,6 @@ openpic_attach(device_t dev) openpic_write(sc, OPENPIC_IPI_VECTOR(ipi), x); } - openpic_set_priority(sc, 15); - /* we don't need 8259 passthrough mode */ x = openpic_read(sc, OPENPIC_CONFIG); x |= OPENPIC_CONFIG_8259_PASSTHRU_DISABLE; @@ -161,9 +162,8 @@ openpic_attach(device_t dev) for (irq = 0; irq < sc->sc_nirq; irq++) openpic_write(sc, OPENPIC_IDEST(irq), 1 << 0); - /* XXX set spurious intr vector */ - - openpic_set_priority(sc, 0); + for (cpu = 0; cpu < sc->sc_ncpu; cpu++) + openpic_write(sc, OPENPIC_PCPU_TPR(cpu), 0); /* clear all pending interrupts */ for (irq = 0; irq < sc->sc_nirq; irq++) { @@ -203,15 +203,25 @@ openpic_config(device_t dev, u_int irq, enum intr_trigger trig, void openpic_dispatch(device_t dev, struct trapframe *tf) { + static int once = 0; struct openpic_softc *sc; u_int vector; + if (once == 0 && PCPU_GET(cpuid) != 0) { + printf("XXX: got interrupt!\n"); + once++; + } + sc = device_get_softc(dev); while (1) { vector = openpic_read(sc, OPENPIC_PCPU_IACK(PCPU_GET(cpuid))); vector &= OPENPIC_VECTOR_MASK; if (vector == 255) break; + if (once == 1 && PCPU_GET(cpuid) != 0) { + printf("XXX: got vector %u\n", vector); + once++; + } powerpc_dispatch_intr(vector, tf); } } |