summaryrefslogtreecommitdiffstats
path: root/sys/powerpc/aim
diff options
context:
space:
mode:
Diffstat (limited to 'sys/powerpc/aim')
-rw-r--r--sys/powerpc/aim/copyinout.c175
-rw-r--r--sys/powerpc/aim/locore.S211
-rw-r--r--sys/powerpc/aim/locore32.S207
-rw-r--r--sys/powerpc/aim/locore64.S369
-rw-r--r--sys/powerpc/aim/machdep.c91
-rw-r--r--sys/powerpc/aim/mmu_oea.c4
-rw-r--r--sys/powerpc/aim/mmu_oea64.c955
-rw-r--r--sys/powerpc/aim/mp_cpudep.c35
-rw-r--r--sys/powerpc/aim/ofw_machdep.c258
-rw-r--r--sys/powerpc/aim/slb.c303
-rw-r--r--sys/powerpc/aim/swtch32.S (renamed from sys/powerpc/aim/swtch.S)11
-rw-r--r--sys/powerpc/aim/swtch64.S291
-rw-r--r--sys/powerpc/aim/trap.c149
-rw-r--r--sys/powerpc/aim/trap_subr32.S678
-rw-r--r--sys/powerpc/aim/trap_subr64.S634
-rw-r--r--sys/powerpc/aim/vm_machdep.c29
16 files changed, 3751 insertions, 649 deletions
diff --git a/sys/powerpc/aim/copyinout.c b/sys/powerpc/aim/copyinout.c
index e023b3f..35d0ee4 100644
--- a/sys/powerpc/aim/copyinout.c
+++ b/sys/powerpc/aim/copyinout.c
@@ -57,6 +57,8 @@
__FBSDID("$FreeBSD$");
#include <sys/param.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
#include <sys/systm.h>
#include <sys/proc.h>
@@ -66,20 +68,45 @@ __FBSDID("$FreeBSD$");
#include <machine/pcb.h>
#include <machine/sr.h>
+#include <machine/slb.h>
int setfault(faultbuf); /* defined in locore.S */
/*
* Makes sure that the right segment of userspace is mapped in.
*/
+
+#ifdef __powerpc64__
+static __inline void
+set_user_sr(pmap_t pm, const void *addr)
+{
+ register_t esid, vsid, slb1, slb2;
+
+ esid = USER_ADDR >> ADDR_SR_SHFT;
+ PMAP_LOCK(pm);
+ vsid = va_to_vsid(pm, (vm_offset_t)addr);
+ PMAP_UNLOCK(pm);
+
+ slb1 = vsid << SLBV_VSID_SHIFT;
+ slb2 = (esid << SLBE_ESID_SHIFT) | SLBE_VALID | USER_SR;
+
+ __asm __volatile ("slbie %0; slbmte %1, %2" :: "r"(esid << 28),
+ "r"(slb1), "r"(slb2));
+ isync();
+}
+#else
static __inline void
-set_user_sr(register_t vsid)
+set_user_sr(pmap_t pm, const void *addr)
{
+ register_t vsid;
+
+ vsid = va_to_vsid(pm, (vm_offset_t)addr);
isync();
__asm __volatile ("mtsr %0,%1" :: "n"(USER_SR), "r"(vsid));
isync();
}
+#endif
int
copyout(const void *kaddr, void *udaddr, size_t len)
@@ -103,13 +130,13 @@ copyout(const void *kaddr, void *udaddr, size_t len)
up = udaddr;
while (len > 0) {
- p = (char *)USER_ADDR + ((u_int)up & ~SEGMENT_MASK);
+ p = (char *)USER_ADDR + ((uintptr_t)up & ~SEGMENT_MASK);
l = ((char *)USER_ADDR + SEGMENT_LENGTH) - p;
if (l > len)
l = len;
- set_user_sr(pm->pm_sr[(u_int)up >> ADDR_SR_SHFT]);
+ set_user_sr(pm,up);
bcopy(kp, p, l);
@@ -144,13 +171,13 @@ copyin(const void *udaddr, void *kaddr, size_t len)
up = udaddr;
while (len > 0) {
- p = (char *)USER_ADDR + ((u_int)up & ~SEGMENT_MASK);
+ p = (char *)USER_ADDR + ((uintptr_t)up & ~SEGMENT_MASK);
l = ((char *)USER_ADDR + SEGMENT_LENGTH) - p;
if (l > len)
l = len;
- set_user_sr(pm->pm_sr[(u_int)up >> ADDR_SR_SHFT]);
+ set_user_sr(pm,up);
bcopy(p, kp, l);
@@ -218,14 +245,14 @@ subyte(void *addr, int byte)
td = PCPU_GET(curthread);
pm = &td->td_proc->p_vmspace->vm_pmap;
- p = (char *)((u_int)USER_ADDR + ((u_int)addr & ~SEGMENT_MASK));
+ p = (char *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
if (setfault(env)) {
td->td_pcb->pcb_onfault = NULL;
return (-1);
}
- set_user_sr(pm->pm_sr[(u_int)addr >> ADDR_SR_SHFT]);
+ set_user_sr(pm,addr);
*p = (char)byte;
@@ -233,6 +260,33 @@ subyte(void *addr, int byte)
return (0);
}
+#ifdef __powerpc64__
+int
+suword32(void *addr, int word)
+{
+ struct thread *td;
+ pmap_t pm;
+ faultbuf env;
+ int *p;
+
+ td = PCPU_GET(curthread);
+ pm = &td->td_proc->p_vmspace->vm_pmap;
+ p = (int *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
+
+ if (setfault(env)) {
+ td->td_pcb->pcb_onfault = NULL;
+ return (-1);
+ }
+
+ set_user_sr(pm,addr);
+
+ *p = word;
+
+ td->td_pcb->pcb_onfault = NULL;
+ return (0);
+}
+#endif
+
int
suword(void *addr, long word)
{
@@ -243,14 +297,14 @@ suword(void *addr, long word)
td = PCPU_GET(curthread);
pm = &td->td_proc->p_vmspace->vm_pmap;
- p = (long *)((u_int)USER_ADDR + ((u_int)addr & ~SEGMENT_MASK));
+ p = (long *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
if (setfault(env)) {
td->td_pcb->pcb_onfault = NULL;
return (-1);
}
- set_user_sr(pm->pm_sr[(u_int)addr >> ADDR_SR_SHFT]);
+ set_user_sr(pm,addr);
*p = word;
@@ -258,12 +312,19 @@ suword(void *addr, long word)
return (0);
}
+#ifdef __powerpc64__
+int
+suword64(void *addr, int64_t word)
+{
+ return (suword(addr, (long)word));
+}
+#else
int
suword32(void *addr, int32_t word)
{
return (suword(addr, (long)word));
}
-
+#endif
int
fubyte(const void *addr)
@@ -276,20 +337,47 @@ fubyte(const void *addr)
td = PCPU_GET(curthread);
pm = &td->td_proc->p_vmspace->vm_pmap;
- p = (u_char *)((u_int)USER_ADDR + ((u_int)addr & ~SEGMENT_MASK));
+ p = (u_char *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
+
+ if (setfault(env)) {
+ td->td_pcb->pcb_onfault = NULL;
+ return (-1);
+ }
+
+ set_user_sr(pm,addr);
+
+ val = *p;
+
+ td->td_pcb->pcb_onfault = NULL;
+ return (val);
+}
+
+#ifdef __powerpc64__
+int32_t
+fuword32(const void *addr)
+{
+ struct thread *td;
+ pmap_t pm;
+ faultbuf env;
+ int32_t *p, val;
+
+ td = PCPU_GET(curthread);
+ pm = &td->td_proc->p_vmspace->vm_pmap;
+ p = (int32_t *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
if (setfault(env)) {
td->td_pcb->pcb_onfault = NULL;
return (-1);
}
- set_user_sr(pm->pm_sr[(u_int)addr >> ADDR_SR_SHFT]);
+ set_user_sr(pm,addr);
val = *p;
td->td_pcb->pcb_onfault = NULL;
return (val);
}
+#endif
long
fuword(const void *addr)
@@ -301,14 +389,14 @@ fuword(const void *addr)
td = PCPU_GET(curthread);
pm = &td->td_proc->p_vmspace->vm_pmap;
- p = (long *)((u_int)USER_ADDR + ((u_int)addr & ~SEGMENT_MASK));
+ p = (long *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
if (setfault(env)) {
td->td_pcb->pcb_onfault = NULL;
return (-1);
}
- set_user_sr(pm->pm_sr[(u_int)addr >> ADDR_SR_SHFT]);
+ set_user_sr(pm,addr);
val = *p;
@@ -316,18 +404,59 @@ fuword(const void *addr)
return (val);
}
+#ifndef __powerpc64__
int32_t
fuword32(const void *addr)
{
return ((int32_t)fuword(addr));
}
+#endif
uint32_t
-casuword32(volatile uint32_t *base, uint32_t oldval, uint32_t newval)
+casuword32(volatile uint32_t *addr, uint32_t old, uint32_t new)
{
- return (casuword((volatile u_long *)base, oldval, newval));
+ struct thread *td;
+ pmap_t pm;
+ faultbuf env;
+ uint32_t *p, val;
+
+ td = PCPU_GET(curthread);
+ pm = &td->td_proc->p_vmspace->vm_pmap;
+ p = (uint32_t *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
+
+ set_user_sr(pm,(const void *)(vm_offset_t)addr);
+
+ if (setfault(env)) {
+ td->td_pcb->pcb_onfault = NULL;
+ return (-1);
+ }
+
+ __asm __volatile (
+ "1:\tlwarx %0, 0, %2\n\t" /* load old value */
+ "cmplw %3, %0\n\t" /* compare */
+ "bne 2f\n\t" /* exit if not equal */
+ "stwcx. %4, 0, %2\n\t" /* attempt to store */
+ "bne- 1b\n\t" /* spin if failed */
+ "b 3f\n\t" /* we've succeeded */
+ "2:\n\t"
+ "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
+ "3:\n\t"
+ : "=&r" (val), "=m" (*p)
+ : "r" (p), "r" (old), "r" (new), "m" (*p)
+ : "cc", "memory");
+
+ td->td_pcb->pcb_onfault = NULL;
+
+ return (val);
}
+#ifndef __powerpc64__
+u_long
+casuword(volatile u_long *addr, u_long old, u_long new)
+{
+ return (casuword32((volatile uint32_t *)addr, old, new));
+}
+#else
u_long
casuword(volatile u_long *addr, u_long old, u_long new)
{
@@ -338,9 +467,9 @@ casuword(volatile u_long *addr, u_long old, u_long new)
td = PCPU_GET(curthread);
pm = &td->td_proc->p_vmspace->vm_pmap;
- p = (u_long *)((u_int)USER_ADDR + ((u_int)addr & ~SEGMENT_MASK));
+ p = (u_long *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
- set_user_sr(pm->pm_sr[(u_int)addr >> ADDR_SR_SHFT]);
+ set_user_sr(pm,(const void *)(vm_offset_t)addr);
if (setfault(env)) {
td->td_pcb->pcb_onfault = NULL;
@@ -348,14 +477,14 @@ casuword(volatile u_long *addr, u_long old, u_long new)
}
__asm __volatile (
- "1:\tlwarx %0, 0, %2\n\t" /* load old value */
- "cmplw %3, %0\n\t" /* compare */
+ "1:\tldarx %0, 0, %2\n\t" /* load old value */
+ "cmpld %3, %0\n\t" /* compare */
"bne 2f\n\t" /* exit if not equal */
- "stwcx. %4, 0, %2\n\t" /* attempt to store */
+ "stdcx. %4, 0, %2\n\t" /* attempt to store */
"bne- 1b\n\t" /* spin if failed */
"b 3f\n\t" /* we've succeeded */
"2:\n\t"
- "stwcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
+ "stdcx. %0, 0, %2\n\t" /* clear reservation (74xx) */
"3:\n\t"
: "=&r" (val), "=m" (*p)
: "r" (p), "r" (old), "r" (new), "m" (*p)
@@ -365,3 +494,5 @@ casuword(volatile u_long *addr, u_long old, u_long new)
return (val);
}
+#endif
+
diff --git a/sys/powerpc/aim/locore.S b/sys/powerpc/aim/locore.S
index 60e5e9d..0cb51f2 100644
--- a/sys/powerpc/aim/locore.S
+++ b/sys/powerpc/aim/locore.S
@@ -1,209 +1,8 @@
/* $FreeBSD$ */
-/* $NetBSD: locore.S,v 1.24 2000/05/31 05:09:17 thorpej Exp $ */
-/*-
- * Copyright (C) 2001 Benno Rice
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-/*-
- * Copyright (C) 1995, 1996 Wolfgang Solfrank.
- * Copyright (C) 1995, 1996 TooLs GmbH.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- * notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- * notice, this list of conditions and the following disclaimer in the
- * documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- * must display the following acknowledgement:
- * This product includes software developed by TooLs GmbH.
- * 4. The name of TooLs GmbH may not be used to endorse or promote products
- * derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
+#ifdef __powerpc64__
+#include <powerpc/aim/locore64.S>
+#else
+#include <powerpc/aim/locore32.S>
+#endif
-#include "assym.s"
-
-#include <sys/syscall.h>
-
-#include <machine/trap.h>
-#include <machine/param.h>
-#include <machine/sr.h>
-#include <machine/spr.h>
-#include <machine/psl.h>
-#include <machine/asm.h>
-
-/* Locate the per-CPU data structure */
-#define GET_CPUINFO(r) \
- mfsprg0 r
-
-/*
- * Compiled KERNBASE location and the kernel load address
- */
- .globl kernbase
- .set kernbase, KERNBASE
-
-#define TMPSTKSZ 8192 /* 8K temporary stack */
-
-/*
- * Globals
- */
- .data
- .align 4
-GLOBAL(tmpstk)
- .space TMPSTKSZ
-GLOBAL(esym)
- .long 0 /* end of symbol table */
-
-GLOBAL(ofmsr)
- .long 0, 0, 0, 0, 0 /* msr/sprg0-3 used in Open Firmware */
-
-#define INTRCNT_COUNT 256 /* max(HROWPIC_IRQMAX,OPENPIC_IRQMAX) */
-GLOBAL(intrnames)
- .space INTRCNT_COUNT * (MAXCOMLEN + 1) * 2
-GLOBAL(eintrnames)
- .align 4
-GLOBAL(intrcnt)
- .space INTRCNT_COUNT * 4 * 2
-GLOBAL(eintrcnt)
-
-/*
- * File-scope for locore.S
- */
-idle_u:
- .long 0 /* fake uarea during idle after exit */
-openfirmware_entry:
- .long 0 /* Open Firmware entry point */
-srsave:
- .long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
-
- .text
- .globl btext
-btext:
-
-/*
- * This symbol is here for the benefit of kvm_mkdb, and is supposed to
- * mark the start of kernel text.
- */
- .globl kernel_text
-kernel_text:
-
-/*
- * Startup entry. Note, this must be the first thing in the text
- * segment!
- */
- .text
- .globl __start
-__start:
- li 8,0
- li 9,0x100
- mtctr 9
-1:
- dcbf 0,8
- icbi 0,8
- addi 8,8,0x20
- bdnz 1b
- sync
- isync
-
- /* Save the argument pointer and length */
- mr 20,6
- mr 21,7
-
- lis 8,openfirmware_entry@ha
- stw 5,openfirmware_entry@l(8) /* save client interface handler */
-
- lis 1,(tmpstk+TMPSTKSZ-16)@ha
- addi 1,1,(tmpstk+TMPSTKSZ-16)@l
-
- mfmsr 0
- lis 9,ofmsr@ha
- stwu 0,ofmsr@l(9)
-
- mfsprg0 0 /* save SPRG0-3 */
- stw 0,4(9) /* ofmsr[1] = sprg0 */
- mfsprg1 0
- stw 0,8(9) /* ofmsr[2] = sprg1 */
- mfsprg2 0
- stw 0,12(9) /* ofmsr[3] = sprg2 */
- mfsprg3 0
- stw 0,16(9) /* ofmsr[4] = sprg3 */
-
- bl OF_initial_setup
-
- lis 4,end@ha
- addi 4,4,end@l
- mr 5,4
-
- lis 3,kernel_text@ha
- addi 3,3,kernel_text@l
-
- /* Restore the argument pointer and length */
- mr 6,20
- mr 7,21
-
- bl powerpc_init
- mr %r1, %r3
- li %r3, 0
- stw %r3, 0(%r1)
- bl mi_startup
- b OF_exit
-
-/*
- * int setfault()
- *
- * Similar to setjmp to setup for handling faults on accesses to user memory.
- * Any routine using this may only call bcopy, either the form below,
- * or the (currently used) C code optimized, so it doesn't use any non-volatile
- * registers.
- */
- .globl setfault
-setfault:
- mflr 0
- mfcr 12
- mfsprg 4,0
- lwz 4,PC_CURTHREAD(4)
- lwz 4,TD_PCB(4)
- stw 3,PCB_ONFAULT(4)
- stw 0,0(3)
- stw 1,4(3)
- stw 2,8(3)
- stmw 12,12(3)
- xor 3,3,3
- blr
-
-#include <powerpc/aim/trap_subr.S>
diff --git a/sys/powerpc/aim/locore32.S b/sys/powerpc/aim/locore32.S
new file mode 100644
index 0000000..64bf81e
--- /dev/null
+++ b/sys/powerpc/aim/locore32.S
@@ -0,0 +1,207 @@
+/* $FreeBSD$ */
+/* $NetBSD: locore.S,v 1.24 2000/05/31 05:09:17 thorpej Exp $ */
+
+/*-
+ * Copyright (C) 2001 Benno Rice
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (C) 1995, 1996 Wolfgang Solfrank.
+ * Copyright (C) 1995, 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "assym.s"
+
+#include <sys/syscall.h>
+
+#include <machine/trap.h>
+#include <machine/param.h>
+#include <machine/spr.h>
+#include <machine/asm.h>
+
+/* Locate the per-CPU data structure */
+#define GET_CPUINFO(r) \
+ mfsprg0 r
+
+/*
+ * Compiled KERNBASE location and the kernel load address
+ */
+ .globl kernbase
+ .set kernbase, KERNBASE
+
+#define TMPSTKSZ 8192 /* 8K temporary stack */
+
+/*
+ * Globals
+ */
+ .data
+ .align 4
+GLOBAL(tmpstk)
+ .space TMPSTKSZ
+GLOBAL(esym)
+ .long 0 /* end of symbol table */
+
+GLOBAL(ofmsr)
+ .long 0, 0, 0, 0, 0 /* msr/sprg0-3 used in Open Firmware */
+
+#define INTRCNT_COUNT 256 /* max(HROWPIC_IRQMAX,OPENPIC_IRQMAX) */
+GLOBAL(intrnames)
+ .space INTRCNT_COUNT * (MAXCOMLEN + 1) * 2
+GLOBAL(eintrnames)
+ .align 4
+GLOBAL(intrcnt)
+ .space INTRCNT_COUNT * 4 * 2
+GLOBAL(eintrcnt)
+
+/*
+ * File-scope for locore.S
+ */
+idle_u:
+ .long 0 /* fake uarea during idle after exit */
+openfirmware_entry:
+ .long 0 /* Open Firmware entry point */
+srsave:
+ .long 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+
+ .text
+ .globl btext
+btext:
+
+/*
+ * This symbol is here for the benefit of kvm_mkdb, and is supposed to
+ * mark the start of kernel text.
+ */
+ .globl kernel_text
+kernel_text:
+
+/*
+ * Startup entry. Note, this must be the first thing in the text
+ * segment!
+ */
+ .text
+ .globl __start
+__start:
+ li 8,0
+ li 9,0x100
+ mtctr 9
+1:
+ dcbf 0,8
+ icbi 0,8
+ addi 8,8,0x20
+ bdnz 1b
+ sync
+ isync
+
+ /* Save the argument pointer and length */
+ mr 20,6
+ mr 21,7
+
+ lis 8,openfirmware_entry@ha
+ stw 5,openfirmware_entry@l(8) /* save client interface handler */
+
+ lis 1,(tmpstk+TMPSTKSZ-16)@ha
+ addi 1,1,(tmpstk+TMPSTKSZ-16)@l
+
+ mfmsr 0
+ lis 9,ofmsr@ha
+ stwu 0,ofmsr@l(9)
+
+ mfsprg0 0 /* save SPRG0-3 */
+ stw 0,4(9) /* ofmsr[1] = sprg0 */
+ mfsprg1 0
+ stw 0,8(9) /* ofmsr[2] = sprg1 */
+ mfsprg2 0
+ stw 0,12(9) /* ofmsr[3] = sprg2 */
+ mfsprg3 0
+ stw 0,16(9) /* ofmsr[4] = sprg3 */
+
+ bl OF_initial_setup
+
+ lis 4,end@ha
+ addi 4,4,end@l
+ mr 5,4
+
+ lis 3,kernel_text@ha
+ addi 3,3,kernel_text@l
+
+ /* Restore the argument pointer and length */
+ mr 6,20
+ mr 7,21
+
+ bl powerpc_init
+ mr %r1, %r3
+ li %r3, 0
+ stw %r3, 0(%r1)
+ bl mi_startup
+ b OF_exit
+
+/*
+ * int setfault()
+ *
+ * Similar to setjmp to setup for handling faults on accesses to user memory.
+ * Any routine using this may only call bcopy, either the form below,
+ * or the (currently used) C code optimized, so it doesn't use any non-volatile
+ * registers.
+ */
+ .globl setfault
+setfault:
+ mflr 0
+ mfcr 12
+ mfsprg 4,0
+ lwz 4,PC_CURTHREAD(4)
+ lwz 4,TD_PCB(4)
+ stw 3,PCB_ONFAULT(4)
+ stw 0,0(3)
+ stw 1,4(3)
+ stw 2,8(3)
+ stmw 12,12(3)
+ xor 3,3,3
+ blr
+
+#include <powerpc/aim/trap_subr32.S>
diff --git a/sys/powerpc/aim/locore64.S b/sys/powerpc/aim/locore64.S
new file mode 100644
index 0000000..234cded
--- /dev/null
+++ b/sys/powerpc/aim/locore64.S
@@ -0,0 +1,369 @@
+/* $FreeBSD$ */
+/* $NetBSD: locore.S,v 1.24 2000/05/31 05:09:17 thorpej Exp $ */
+
+/*-
+ * Copyright (C) 2001 Benno Rice
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+/*-
+ * Copyright (C) 1995, 1996 Wolfgang Solfrank.
+ * Copyright (C) 1995, 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "assym.s"
+
+#include <sys/syscall.h>
+
+#include <machine/trap.h>
+#include <machine/param.h>
+#include <machine/spr.h>
+#include <machine/asm.h>
+
+/* Locate the per-CPU data structure */
+#define GET_CPUINFO(r) \
+ mfsprg0 r
+
+/*
+ * Compiled KERNBASE location and the kernel load address
+ */
+ .globl kernbase
+ .set kernbase, KERNBASE
+
+#define TMPSTKSZ 8192 /* 8K temporary stack */
+#define OFWSTKSZ 4096 /* 4K Open Firmware stack */
+
+/*
+ * Globals
+ */
+ .data
+ .align 4
+GLOBAL(tmpstk)
+ .space TMPSTKSZ
+GLOBAL(ofwstk)
+ .space OFWSTKSZ
+GLOBAL(esym)
+ .llong 0 /* end of symbol table */
+
+GLOBAL(ofmsr)
+ .llong 0, 0, 0, 0, 0 /* msr/sprg0-3 used in Open Firmware */
+
+#define INTRCNT_COUNT 256 /* max(HROWPIC_IRQMAX,OPENPIC_IRQMAX) */
+GLOBAL(intrnames)
+ .space INTRCNT_COUNT * (MAXCOMLEN + 1) * 2
+GLOBAL(eintrnames)
+ .align 4
+GLOBAL(intrcnt)
+ .space INTRCNT_COUNT * 4 * 2
+GLOBAL(eintrcnt)
+
+/*
+ * File-scope for locore.S
+ */
+idle_u:
+ .llong 0 /* fake uarea during idle after exit */
+openfirmware_entry:
+ .llong 0 /* Open Firmware entry point */
+srsave:
+ .llong 0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0
+
+ .text
+ .globl btext
+btext:
+
+/*
+ * This symbol is here for the benefit of kvm_mkdb, and is supposed to
+ * mark the start of kernel text.
+ */
+ .globl kernel_text
+kernel_text:
+
+/*
+ * Startup entry. Note, this must be the first thing in the text
+ * segment!
+ */
+ .text
+ASENTRY(__start)
+ li 8,0
+ li 9,0x100
+ mtctr 9
+1:
+ dcbf 0,8
+ icbi 0,8
+ addi 8,8,0x20
+ bdnz 1b
+ sync
+ isync
+
+ /* Save the argument pointer and length */
+ mr 20,6
+ mr 21,7
+
+ lis 8,openfirmware_entry@ha
+ std 5,openfirmware_entry@l(8) /* save client interface handler */
+
+ /* Set up the stack pointer */
+ lis 1,(tmpstk+TMPSTKSZ-48)@ha
+ addi 1,1,(tmpstk+TMPSTKSZ-48)@l
+
+ /* Set up the TOC pointer */
+ lis 2,tocbase@ha
+ ld 2,tocbase@l(2)
+
+ mfmsr 0
+ lis 9,ofmsr@ha
+ stdu 0,ofmsr@l(9)
+
+ mfsprg0 0 /* save SPRG0-3 */
+ std 0,8(9) /* ofmsr[1] = sprg0 */
+ mfsprg1 0
+ std 0,16(9) /* ofmsr[2] = sprg1 */
+ mfsprg2 0
+ std 0,24(9) /* ofmsr[3] = sprg2 */
+ mfsprg3 0
+ std 0,32(9) /* ofmsr[4] = sprg3 */
+
+ /* Switch to 64-bit mode */
+ mfmsr 9
+ li 8,1
+ insrdi 9,8,1,0
+ mtmsrd 9
+
+ bl .OF_initial_setup
+ nop
+
+ lis 4,end@ha
+ addi 4,4,end@l
+ mr 5,4
+
+ lis 3,kernbase@ha
+ addi 3,3,kernbase@l
+
+ /* Restore the argument pointer and length */
+ mr 6,20
+ mr 7,21
+
+ bl .powerpc_init
+ nop
+ mr %r1, %r3
+ li %r3, 0
+ std %r3, 0(%r1)
+ bl .mi_startup
+ nop
+ b .OF_exit
+ nop
+
+/*
+ * PPC64 ABI TOC base
+ */
+
+ .align 3
+ .globl tocbase
+tocbase:
+ .llong .TOC.@tocbase
+
+/*
+ * Open Firmware Real-mode Entry Point. This is a huge pain.
+ */
+
+ASENTRY(ofw_32bit_mode_entry)
+ mflr %r0
+ std %r0,16(%r1)
+ stdu %r1,-208(%r1)
+
+ /*
+ * We need to save the following, because OF's register save/
+ * restore code assumes that the contents of registers are
+ * at most 32 bits wide: lr, cr, r2, r13-r31, the old MSR. These
+ * get placed in that order in the stack.
+ */
+
+ mfcr %r4
+ std %r4,48(%r1)
+ std %r13,56(%r1)
+ std %r14,64(%r1)
+ std %r15,72(%r1)
+ std %r16,80(%r1)
+ std %r17,88(%r1)
+ std %r18,96(%r1)
+ std %r19,104(%r1)
+ std %r20,112(%r1)
+ std %r21,120(%r1)
+ std %r22,128(%r1)
+ std %r23,136(%r1)
+ std %r24,144(%r1)
+ std %r25,152(%r1)
+ std %r26,160(%r1)
+ std %r27,168(%r1)
+ std %r28,176(%r1)
+ std %r29,184(%r1)
+ std %r30,192(%r1)
+ std %r31,200(%r1)
+
+ /* Record the old MSR */
+ mfmsr %r6
+
+ /* read client interface handler */
+ lis %r4,openfirmware_entry@ha
+ ld %r4,openfirmware_entry@l(%r4)
+
+ /*
+ * Set the MSR to the OF value. This has the side effect of disabling
+ * exceptions, which is important for the next few steps.
+ */
+
+ lis %r5,ofmsr@ha
+ ld %r5,ofmsr@l(%r5)
+ mtmsrd %r5
+ isync
+
+ /*
+ * Set up OF stack. This needs to be accessible in real mode and
+ * use the 32-bit ABI stack frame format. The pointer to the current
+ * kernel stack is placed at the very top of the stack along with
+ * the old MSR so we can get them back later.
+ */
+ mr %r5,%r1
+ lis %r1,(ofwstk+OFWSTKSZ-32)@ha
+ addi %r1,%r1,(ofwstk+OFWSTKSZ-32)@l
+ std %r5,8(%r1) /* Save real stack pointer */
+ std %r2,16(%r1) /* Save old TOC */
+ std %r6,24(%r1) /* Save old MSR */
+ li %r5,0
+ stw %r5,4(%r1)
+ stw %r5,0(%r1)
+
+ /* Finally, branch to OF */
+ mtctr %r4
+ bctrl
+
+ /* Reload stack pointer and MSR from the OFW stack */
+ ld %r6,24(%r1)
+ ld %r2,16(%r1)
+ ld %r1,8(%r1)
+
+ /* Now set the real MSR */
+ mtmsrd %r6
+ isync
+
+ /* Sign-extend the return value from OF */
+ extsw %r3,%r3
+
+ /* Restore all the non-volatile registers */
+ ld %r5,48(%r1)
+ mtcr %r5
+ ld %r13,56(%r1)
+ ld %r14,64(%r1)
+ ld %r15,72(%r1)
+ ld %r16,80(%r1)
+ ld %r17,88(%r1)
+ ld %r18,96(%r1)
+ ld %r19,104(%r1)
+ ld %r20,112(%r1)
+ ld %r21,120(%r1)
+ ld %r22,128(%r1)
+ ld %r23,136(%r1)
+ ld %r24,144(%r1)
+ ld %r25,152(%r1)
+ ld %r26,160(%r1)
+ ld %r27,168(%r1)
+ ld %r28,176(%r1)
+ ld %r29,184(%r1)
+ ld %r30,192(%r1)
+ ld %r31,200(%r1)
+
+ /* Restore the stack and link register */
+ ld %r1,0(%r1)
+ ld %r0,16(%r1)
+ mtlr %r0
+ blr
+
+/*
+ * int setfault()
+ *
+ * Similar to setjmp to setup for handling faults on accesses to user memory.
+ * Any routine using this may only call bcopy, either the form below,
+ * or the (currently used) C code optimized, so it doesn't use any non-volatile
+ * registers.
+ */
+ASENTRY(setfault)
+ mflr 0
+ mfcr 12
+ mfsprg 4,0
+ ld 4,PC_CURTHREAD(4)
+ ld 4,TD_PCB(4)
+ std 3,PCB_ONFAULT(4)
+ std 0,0(3)
+ std 1,8(3)
+ std 2,16(3)
+
+ std %r12,24(%r3) /* Save the non-volatile GP regs. */
+ std %r13,24+1*8(%r3)
+ std %r14,24+2*8(%r3)
+ std %r15,24+3*8(%r3)
+ std %r16,24+4*8(%r3)
+ std %r17,24+5*8(%r3)
+ std %r18,24+6*8(%r3)
+ std %r19,24+7*8(%r3)
+ std %r20,24+8*8(%r3)
+ std %r21,24+9*8(%r3)
+ std %r22,24+10*8(%r3)
+ std %r23,24+11*8(%r3)
+ std %r24,24+12*8(%r3)
+ std %r25,24+13*8(%r3)
+ std %r26,24+14*8(%r3)
+ std %r27,24+15*8(%r3)
+ std %r28,24+16*8(%r3)
+ std %r29,24+17*8(%r3)
+ std %r30,24+18*8(%r3)
+ std %r31,24+19*8(%r3)
+
+ xor 3,3,3
+ blr
+
+#include <powerpc/aim/trap_subr64.S>
diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index 45d9418..a2f13ef 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -105,7 +105,9 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_pager.h>
#include <machine/altivec.h>
+#ifndef __powerpc64__
#include <machine/bat.h>
+#endif
#include <machine/cpu.h>
#include <machine/elf.h>
#include <machine/fpu.h>
@@ -130,7 +132,11 @@ extern vm_offset_t ksym_start, ksym_end;
#endif
int cold = 1;
+#ifdef __powerpc64__
+int cacheline_size = 128;
+#else
int cacheline_size = 32;
+#endif
int hw_direct_map = 1;
struct pcpu __pcpu[MAXCPU];
@@ -146,24 +152,18 @@ SYSINIT(cpu, SI_SUB_CPU, SI_ORDER_FIRST, cpu_startup, NULL);
SYSCTL_INT(_machdep, CPU_CACHELINE, cacheline_size,
CTLFLAG_RD, &cacheline_size, 0, "");
-u_int powerpc_init(u_int, u_int, u_int, void *);
-
-int save_ofw_mapping(void);
-int restore_ofw_mapping(void);
-
-void install_extint(void (*)(void));
+uintptr_t powerpc_init(vm_offset_t, vm_offset_t, vm_offset_t, void *);
int setfault(faultbuf); /* defined in locore.S */
-void asm_panic(char *);
-
long Maxmem = 0;
long realmem = 0;
struct pmap ofw_pmap;
-extern int ofmsr;
+#ifndef __powerpc64__
struct bat battable[16];
+#endif
struct kva_md_info kmi;
@@ -210,9 +210,14 @@ cpu_startup(void *dummy)
printf("Physical memory chunk(s):\n");
for (indx = 0; phys_avail[indx + 1] != 0; indx += 2) {
- int size1 = phys_avail[indx + 1] - phys_avail[indx];
-
- printf("0x%08x - 0x%08x, %d bytes (%d pages)\n",
+ vm_offset_t size1 =
+ phys_avail[indx + 1] - phys_avail[indx];
+
+ #ifdef __powerpc64__
+ printf("0x%16lx - 0x%16lx, %ld bytes (%ld pages)\n",
+ #else
+ printf("0x%08x - 0x%08x, %d bytes (%ld pages)\n",
+ #endif
phys_avail[indx], phys_avail[indx + 1] - 1, size1,
size1 / PAGE_SIZE);
}
@@ -235,21 +240,27 @@ cpu_startup(void *dummy)
extern char kernel_text[], _end[];
+#ifndef __powerpc64__
+/* Bits for running on 64-bit systems in 32-bit mode. */
extern void *testppc64, *testppc64size;
extern void *restorebridge, *restorebridgesize;
extern void *rfid_patch, *rfi_patch1, *rfi_patch2;
+extern void *trapcode64;
+#endif
+
#ifdef SMP
extern void *rstcode, *rstsize;
#endif
-extern void *trapcode, *trapcode64, *trapsize;
+extern void *trapcode, *trapsize;
extern void *alitrap, *alisize;
extern void *dsitrap, *dsisize;
extern void *decrint, *decrsize;
extern void *extint, *extsize;
extern void *dblow, *dbsize;
-u_int
-powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
+uintptr_t
+powerpc_init(vm_offset_t startkernel, vm_offset_t endkernel,
+ vm_offset_t basekernel, void *mdp)
{
struct pcpu *pc;
vm_offset_t end;
@@ -257,9 +268,11 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
size_t trap_offset;
void *kmdp;
char *env;
- uint32_t msr, scratch;
+ register_t msr, scratch;
uint8_t *cache_check;
+ #ifndef __powerpc64__
int ppc64;
+ #endif
end = 0;
kmdp = NULL;
@@ -346,9 +359,9 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
case IBM970FX:
case IBM970MP:
case IBM970GX:
- scratch = mfspr64upper(SPR_HID5,msr);
+ scratch = mfspr(SPR_HID5);
scratch &= ~HID5_970_DCBZ_SIZE_HI;
- mtspr64(SPR_HID5, scratch, mfspr(SPR_HID5), msr);
+ mtspr(SPR_HID5, scratch);
break;
}
@@ -390,6 +403,7 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
cacheline_size = 32;
}
+ #ifndef __powerpc64__
/*
* Figure out whether we need to use the 64 bit PMAP. This works by
* executing an instruction that is only legal on 64-bit PPC (mtmsrd),
@@ -449,6 +463,11 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
generictrap = &trapcode;
}
+ #else /* powerpc64 */
+ cpu_features |= PPC_FEATURE_64;
+ generictrap = &trapcode;
+ #endif
+
#ifdef SMP
bcopy(&rstcode, (void *)(EXC_RST + trap_offset), (size_t)&rstsize);
#else
@@ -466,9 +485,13 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
bcopy(generictrap, (void *)EXC_TRC, (size_t)&trapsize);
bcopy(generictrap, (void *)EXC_BPT, (size_t)&trapsize);
#endif
- bcopy(&dsitrap, (void *)(EXC_DSI + trap_offset), (size_t)&dsisize);
bcopy(&alitrap, (void *)(EXC_ALI + trap_offset), (size_t)&alisize);
+ bcopy(&dsitrap, (void *)(EXC_DSI + trap_offset), (size_t)&dsisize);
bcopy(generictrap, (void *)EXC_ISI, (size_t)&trapsize);
+ #ifdef __powerpc64__
+ bcopy(generictrap, (void *)EXC_DSE, (size_t)&trapsize);
+ bcopy(generictrap, (void *)EXC_ISE, (size_t)&trapsize);
+ #endif
bcopy(generictrap, (void *)EXC_EXI, (size_t)&trapsize);
bcopy(generictrap, (void *)EXC_FPU, (size_t)&trapsize);
bcopy(generictrap, (void *)EXC_DECR, (size_t)&trapsize);
@@ -524,7 +547,7 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
*/
thread0.td_pcb = (struct pcb *)
((thread0.td_kstack + thread0.td_kstack_pages * PAGE_SIZE -
- sizeof(struct pcb)) & ~15);
+ sizeof(struct pcb)) & ~15UL);
bzero((void *)thread0.td_pcb, sizeof(struct pcb));
pc->pc_curpcb = thread0.td_pcb;
@@ -537,7 +560,8 @@ powerpc_init(u_int startkernel, u_int endkernel, u_int basekernel, void *mdp)
"Boot flags requested debugger");
#endif
- return (((uintptr_t)thread0.td_pcb - 16) & ~15);
+ return (((uintptr_t)thread0.td_pcb -
+ (sizeof(struct callframe) - 3*sizeof(register_t))) & ~15UL);
}
void
@@ -614,7 +638,7 @@ cpu_halt(void)
void
cpu_idle(int busy)
{
- uint32_t msr;
+ register_t msr;
uint16_t vers;
msr = mfmsr();
@@ -623,7 +647,7 @@ cpu_idle(int busy)
#ifdef INVARIANTS
if ((msr & PSL_EE) != PSL_EE) {
struct thread *td = curthread;
- printf("td msr %x\n", td->td_md.md_saved_msr);
+ printf("td msr %#lx\n", (u_long)td->td_md.md_saved_msr);
panic("ints disabled in idleproc!");
}
#endif
@@ -710,7 +734,10 @@ kdb_cpu_set_singlestep(void)
void
cpu_pcpu_init(struct pcpu *pcpu, int cpuid, size_t sz)
{
-
+#ifdef __powerpc64__
+/* Copy the SLB contents from the current CPU */
+memcpy(pcpu->pc_slb, PCPU_GET(slb), sizeof(pcpu->pc_slb));
+#endif
}
void
@@ -767,12 +794,6 @@ kcopy(const void *src, void *dst, size_t len)
return (0);
}
-void
-asm_panic(char *pstr)
-{
- panic(pstr);
-}
-
int db_trap_glue(struct trapframe *); /* Called from trap_subr.S */
int
@@ -793,3 +814,13 @@ db_trap_glue(struct trapframe *frame)
return (0);
}
+
+#ifndef __powerpc64__
+
+uint64_t
+va_to_vsid(pmap_t pm, vm_offset_t va)
+{
+ return ((pm->pm_sr[(uintptr_t)va >> ADDR_SR_SHFT]) & SR_VSID_MASK);
+}
+
+#endif
diff --git a/sys/powerpc/aim/mmu_oea.c b/sys/powerpc/aim/mmu_oea.c
index de83462..2d090db 100644
--- a/sys/powerpc/aim/mmu_oea.c
+++ b/sys/powerpc/aim/mmu_oea.c
@@ -193,8 +193,8 @@ struct ofw_map {
*/
static struct mem_region *regions;
static struct mem_region *pregions;
-u_int phys_avail_count;
-int regions_sz, pregions_sz;
+static u_int phys_avail_count;
+static int regions_sz, pregions_sz;
static struct ofw_map *translations;
extern struct pmap ofw_pmap;
diff --git a/sys/powerpc/aim/mmu_oea64.c b/sys/powerpc/aim/mmu_oea64.c
index 8672ab2..33b3ed9 100644
--- a/sys/powerpc/aim/mmu_oea64.c
+++ b/sys/powerpc/aim/mmu_oea64.c
@@ -142,12 +142,14 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_pager.h>
#include <vm/uma.h>
+#include <machine/_inttypes.h>
#include <machine/cpu.h>
#include <machine/platform.h>
#include <machine/frame.h>
#include <machine/md_var.h>
#include <machine/psl.h>
#include <machine/bat.h>
+#include <machine/hid.h>
#include <machine/pte.h>
#include <machine/sr.h>
#include <machine/trap.h>
@@ -158,20 +160,16 @@ __FBSDID("$FreeBSD$");
#define MOEA_DEBUG
#define TODO panic("%s: not implemented", __func__);
+void moea64_release_vsid(uint64_t vsid);
+uintptr_t moea64_get_unique_vsid(void);
-static __inline u_int32_t
-cntlzw(volatile u_int32_t a) {
- u_int32_t b;
- __asm ("cntlzw %0, %1" : "=r"(b) : "r"(a));
+static __inline register_t
+cntlzd(volatile register_t a) {
+ register_t b;
+ __asm ("cntlzd %0, %1" : "=r"(b) : "r"(a));
return b;
}
-static __inline uint64_t
-va_to_vsid(pmap_t pm, vm_offset_t va)
-{
- return ((pm->pm_sr[(uintptr_t)va >> ADDR_SR_SHFT]) & SR_VSID_MASK);
-}
-
#define PTESYNC() __asm __volatile("ptesync");
#define TLBSYNC() __asm __volatile("tlbsync; ptesync");
#define SYNC() __asm __volatile("sync");
@@ -186,21 +184,29 @@ va_to_vsid(pmap_t pm, vm_offset_t va)
struct mtx tlbie_mutex;
static __inline void
-TLBIE(pmap_t pmap, vm_offset_t va) {
- uint64_t vpn;
+TLBIE(uint64_t vpn) {
+#ifndef __powerpc64__
register_t vpn_hi, vpn_lo;
register_t msr;
register_t scratch;
+#endif
- vpn = (uint64_t)(va & ADDR_PIDX);
- if (pmap != NULL)
- vpn |= (va_to_vsid(pmap,va) << 28);
+ vpn <<= ADDR_PIDX_SHFT;
vpn &= ~(0xffffULL << 48);
+ mtx_lock_spin(&tlbie_mutex);
+#ifdef __powerpc64__
+ __asm __volatile("\
+ ptesync; \
+ tlbie %0; \
+ eieio; \
+ tlbsync; \
+ ptesync;"
+ :: "r"(vpn) : "memory");
+#else
vpn_hi = (uint32_t)(vpn >> 32);
vpn_lo = (uint32_t)vpn;
- mtx_lock_spin(&tlbie_mutex);
__asm __volatile("\
mfmsr %0; \
mr %1, %0; \
@@ -218,6 +224,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
ptesync;"
: "=r"(msr), "=r"(scratch) : "r"(vpn_hi), "r"(vpn_lo), "r"(32), "r"(1)
: "memory");
+#endif
mtx_unlock_spin(&tlbie_mutex);
}
@@ -225,7 +232,6 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
#define ENABLE_TRANS(msr) mtmsr(msr); isync()
#define VSID_MAKE(sr, hash) ((sr) | (((hash) & 0xfffff) << 4))
-#define VSID_TO_SR(vsid) ((vsid) & 0xf)
#define VSID_TO_HASH(vsid) (((vsid) >> 4) & 0xfffff)
#define VSID_HASH_MASK 0x0000007fffffffffULL
@@ -236,6 +242,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
#define PVO_BOOTSTRAP 0x080UL /* PVO entry allocated during
bootstrap */
#define PVO_FAKE 0x100UL /* fictitious phys page */
+#define PVO_LARGE 0x200UL /* large page */
#define PVO_VADDR(pvo) ((pvo)->pvo_vaddr & ~ADDR_POFF)
#define PVO_ISFAKE(pvo) ((pvo)->pvo_vaddr & PVO_FAKE)
#define PVO_PTEGIDX_GET(pvo) ((pvo)->pvo_vaddr & PVO_PTEGIDX_MASK)
@@ -244,6 +251,7 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
((void)((pvo)->pvo_vaddr &= ~(PVO_PTEGIDX_VALID|PVO_PTEGIDX_MASK)))
#define PVO_PTEGIDX_SET(pvo, i) \
((void)((pvo)->pvo_vaddr |= (i)|PVO_PTEGIDX_VALID))
+#define PVO_VSID(pvo) ((pvo)->pvo_vpn >> 16)
#define MOEA_PVO_CHECK(pvo)
@@ -252,11 +260,11 @@ TLBIE(pmap_t pmap, vm_offset_t va) {
#define ASSERT_TABLE_LOCK() mtx_assert(&moea64_table_mutex, MA_OWNED)
struct ofw_map {
- vm_offset_t om_va;
- vm_size_t om_len;
- vm_offset_t om_pa_hi;
- vm_offset_t om_pa_lo;
- u_int om_mode;
+ cell_t om_va;
+ cell_t om_len;
+ cell_t om_pa_hi;
+ cell_t om_pa_lo;
+ cell_t om_mode;
};
/*
@@ -264,8 +272,8 @@ struct ofw_map {
*/
static struct mem_region *regions;
static struct mem_region *pregions;
-extern u_int phys_avail_count;
-extern int regions_sz, pregions_sz;
+static u_int phys_avail_count;
+static int regions_sz, pregions_sz;
extern int ofw_real_mode;
extern struct pmap ofw_pmap;
@@ -303,7 +311,14 @@ static struct pvo_entry *moea64_bpvo_pool;
static int moea64_bpvo_pool_index = 0;
#define VSID_NBPW (sizeof(u_int32_t) * 8)
-static u_int moea64_vsid_bitmap[NPMAPS / VSID_NBPW];
+#ifdef __powerpc64__
+#define NVSIDS (NPMAPS * 16)
+#define VSID_HASHMASK 0xffffffffUL
+#else
+#define NVSIDS NPMAPS
+#define VSID_HASHMASK 0xfffffUL
+#endif
+static u_int moea64_vsid_bitmap[NVSIDS / VSID_NBPW];
static boolean_t moea64_initialized = FALSE;
@@ -327,9 +342,14 @@ SYSCTL_INT(_machdep, OID_AUTO, moea64_pvo_remove_calls, CTLFLAG_RD,
&moea64_pvo_remove_calls, 0, "");
vm_offset_t moea64_scratchpage_va[2];
+uint64_t moea64_scratchpage_vpn[2];
struct lpte *moea64_scratchpage_pte[2];
struct mtx moea64_scratchpage_mtx;
+uint64_t moea64_large_page_mask = 0;
+int moea64_large_page_size = 0;
+int moea64_large_page_shift = 0;
+
/*
* Allocate physical memory for use in moea64_bootstrap.
*/
@@ -345,16 +365,16 @@ static int moea64_pte_insert(u_int, struct lpte *);
*/
static int moea64_pvo_enter(pmap_t, uma_zone_t, struct pvo_head *,
vm_offset_t, vm_offset_t, uint64_t, int);
-static void moea64_pvo_remove(struct pvo_entry *, int);
-static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t, int *);
-static struct lpte *moea64_pvo_to_pte(const struct pvo_entry *, int);
+static void moea64_pvo_remove(struct pvo_entry *);
+static struct pvo_entry *moea64_pvo_find_va(pmap_t, vm_offset_t);
+static struct lpte *moea64_pvo_to_pte(const struct pvo_entry *);
/*
* Utility routines.
*/
-static void moea64_bridge_bootstrap(mmu_t mmup,
+static void moea64_bootstrap(mmu_t mmup,
vm_offset_t kernelstart, vm_offset_t kernelend);
-static void moea64_bridge_cpu_bootstrap(mmu_t, int ap);
+static void moea64_cpu_bootstrap(mmu_t, int ap);
static void moea64_enter_locked(pmap_t, vm_offset_t, vm_page_t,
vm_prot_t, boolean_t);
static boolean_t moea64_query_bit(vm_page_t, u_int64_t);
@@ -363,6 +383,9 @@ static void moea64_kremove(mmu_t, vm_offset_t);
static void moea64_syncicache(pmap_t pmap, vm_offset_t va,
vm_offset_t pa, vm_size_t sz);
static void tlbia(void);
+#ifdef __powerpc64__
+static void slbia(void);
+#endif
/*
* Kernel MMU interface
@@ -405,7 +428,7 @@ void moea64_kenter(mmu_t, vm_offset_t, vm_offset_t);
boolean_t moea64_dev_direct_mapped(mmu_t, vm_offset_t, vm_size_t);
static void moea64_sync_icache(mmu_t, pmap_t, vm_offset_t, vm_size_t);
-static mmu_method_t moea64_bridge_methods[] = {
+static mmu_method_t moea64_methods[] = {
MMUMETHOD(mmu_change_wiring, moea64_change_wiring),
MMUMETHOD(mmu_clear_modify, moea64_clear_modify),
MMUMETHOD(mmu_clear_reference, moea64_clear_reference),
@@ -439,8 +462,8 @@ static mmu_method_t moea64_bridge_methods[] = {
MMUMETHOD(mmu_deactivate, moea64_deactivate),
/* Internal interfaces */
- MMUMETHOD(mmu_bootstrap, moea64_bridge_bootstrap),
- MMUMETHOD(mmu_cpu_bootstrap, moea64_bridge_cpu_bootstrap),
+ MMUMETHOD(mmu_bootstrap, moea64_bootstrap),
+ MMUMETHOD(mmu_cpu_bootstrap, moea64_cpu_bootstrap),
MMUMETHOD(mmu_mapdev, moea64_mapdev),
MMUMETHOD(mmu_unmapdev, moea64_unmapdev),
MMUMETHOD(mmu_kextract, moea64_kextract),
@@ -450,20 +473,22 @@ static mmu_method_t moea64_bridge_methods[] = {
{ 0, 0 }
};
-static mmu_def_t oea64_bridge_mmu = {
+static mmu_def_t oea64_mmu = {
MMU_TYPE_G5,
- moea64_bridge_methods,
+ moea64_methods,
0
};
-MMU_DEF(oea64_bridge_mmu);
+MMU_DEF(oea64_mmu);
static __inline u_int
-va_to_pteg(uint64_t vsid, vm_offset_t addr)
+va_to_pteg(uint64_t vsid, vm_offset_t addr, int large)
{
uint64_t hash;
+ int shift;
+ shift = large ? moea64_large_page_shift : ADDR_PIDX_SHFT;
hash = (vsid & VSID_HASH_MASK) ^ (((uint64_t)addr & ADDR_PIDX) >>
- ADDR_PIDX_SHFT);
+ shift);
return (hash & moea64_pteg_mask);
}
@@ -515,8 +540,9 @@ moea64_attr_save(vm_page_t m, u_int64_t ptebit)
static __inline void
moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va,
- uint64_t pte_lo)
+ uint64_t pte_lo, int flags)
{
+
ASSERT_TABLE_LOCK();
/*
@@ -528,6 +554,9 @@ moea64_pte_create(struct lpte *pt, uint64_t vsid, vm_offset_t va,
pt->pte_hi = (vsid << LPTE_VSID_SHIFT) |
(((uint64_t)(va & ADDR_PIDX) >> ADDR_API_SHFT64) & LPTE_API);
+ if (flags & PVO_LARGE)
+ pt->pte_hi |= LPTE_BIG;
+
pt->pte_lo = pte_lo;
}
@@ -541,7 +570,7 @@ moea64_pte_synch(struct lpte *pt, struct lpte *pvo_pt)
}
static __inline void
-moea64_pte_clear(struct lpte *pt, pmap_t pmap, vm_offset_t va, u_int64_t ptebit)
+moea64_pte_clear(struct lpte *pt, uint64_t vpn, u_int64_t ptebit)
{
ASSERT_TABLE_LOCK();
@@ -549,7 +578,7 @@ moea64_pte_clear(struct lpte *pt, pmap_t pmap, vm_offset_t va, u_int64_t ptebit)
* As shown in Section 7.6.3.2.3
*/
pt->pte_lo &= ~ptebit;
- TLBIE(pmap,va);
+ TLBIE(vpn);
}
static __inline void
@@ -572,7 +601,7 @@ moea64_pte_set(struct lpte *pt, struct lpte *pvo_pt)
}
static __inline void
-moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va)
+moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn)
{
ASSERT_TABLE_LOCK();
pvo_pt->pte_hi &= ~LPTE_VALID;
@@ -586,7 +615,7 @@ moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t
* Invalidate the pte.
*/
pt->pte_hi &= ~LPTE_VALID;
- TLBIE(pmap,va);
+ TLBIE(vpn);
/*
* Save the reg & chg bits.
@@ -596,16 +625,14 @@ moea64_pte_unset(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t
}
static __inline void
-moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, pmap_t pmap, vm_offset_t va)
+moea64_pte_change(struct lpte *pt, struct lpte *pvo_pt, uint64_t vpn)
{
/*
* Invalidate the PTE
*/
- moea64_pte_unset(pt, pvo_pt, pmap, va);
+ moea64_pte_unset(pt, pvo_pt, vpn);
moea64_pte_set(pt, pvo_pt);
- if (pmap == kernel_pmap)
- isync();
}
static __inline uint64_t
@@ -674,21 +701,45 @@ om_cmp(const void *a, const void *b)
}
static void
-moea64_bridge_cpu_bootstrap(mmu_t mmup, int ap)
+moea64_cpu_bootstrap(mmu_t mmup, int ap)
{
int i = 0;
+ #ifdef __powerpc64__
+ struct slb *slb = PCPU_GET(slb);
+ #endif
/*
* Initialize segment registers and MMU
*/
mtmsr(mfmsr() & ~PSL_DR & ~PSL_IR); isync();
- for (i = 0; i < 16; i++) {
- mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
- }
+
+ /*
+ * Install kernel SLB entries
+ */
+
+ #ifdef __powerpc64__
+ slbia();
+
+ for (i = 0; i < 64; i++) {
+ if (!(slb[i].slbe & SLBE_VALID))
+ continue;
+
+ __asm __volatile ("slbmte %0, %1" ::
+ "r"(slb[i].slbv), "r"(slb[i].slbe));
+ }
+ #else
+ for (i = 0; i < 16; i++)
+ mtsrin(i << ADDR_SR_SHFT, kernel_pmap->pm_sr[i]);
+ #endif
+
+ /*
+ * Install page table
+ */
+
__asm __volatile ("ptesync; mtsdr1 %0; isync"
- :: "r"((u_int)moea64_pteg_table
- | (32 - cntlzw(moea64_pteg_mask >> 11))));
+ :: "r"((uintptr_t)moea64_pteg_table
+ | (64 - cntlzd(moea64_pteg_mask >> 11))));
tlbia();
}
@@ -717,15 +768,23 @@ moea64_add_ofw_mappings(mmu_t mmup, phandle_t mmu, size_t sz)
if (translations[i].om_pa_lo % PAGE_SIZE)
panic("OFW translation not page-aligned!");
+ pa_base = translations[i].om_pa_lo;
+
+ #ifdef __powerpc64__
+ pa_base += (vm_offset_t)translations[i].om_pa_hi << 32;
+ #else
if (translations[i].om_pa_hi)
panic("OFW translations above 32-bit boundary!");
-
- pa_base = translations[i].om_pa_lo;
+ #endif
/* Now enter the pages for this mapping */
DISABLE_TRANS(msr);
for (off = 0; off < translations[i].om_len; off += PAGE_SIZE) {
+ if (moea64_pvo_find_va(kernel_pmap,
+ translations[i].om_va + off) != NULL)
+ continue;
+
moea64_kenter(mmup, translations[i].om_va + off,
pa_base + off);
@@ -735,8 +794,130 @@ moea64_add_ofw_mappings(mmu_t mmup, phandle_t mmu, size_t sz)
}
}
+#ifdef __powerpc64__
+static void
+moea64_probe_large_page(void)
+{
+ uint16_t pvr = mfpvr() >> 16;
+
+ switch (pvr) {
+ case IBM970:
+ case IBM970FX:
+ case IBM970MP:
+ powerpc_sync(); isync();
+ mtspr(SPR_HID4, mfspr(SPR_HID4) & ~HID4_970_DISABLE_LG_PG);
+ powerpc_sync(); isync();
+
+ /* FALLTHROUGH */
+ case IBMCELLBE:
+ moea64_large_page_size = 0x1000000; /* 16 MB */
+ moea64_large_page_shift = 24;
+ break;
+ default:
+ moea64_large_page_size = 0;
+ }
+
+ moea64_large_page_mask = moea64_large_page_size - 1;
+}
+
+static void
+moea64_bootstrap_slb_prefault(vm_offset_t va, int large)
+{
+ struct slb *cache;
+ struct slb entry;
+ uint64_t esid, slbe;
+ uint64_t i;
+
+ cache = PCPU_GET(slb);
+ esid = va >> ADDR_SR_SHFT;
+ slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
+
+ for (i = 0; i < 64; i++) {
+ if (cache[i].slbe == (slbe | i))
+ return;
+ }
+
+ entry.slbe = slbe;
+ entry.slbv = KERNEL_VSID(esid, large) << SLBV_VSID_SHIFT;
+ if (large)
+ entry.slbv |= SLBV_L;
+
+ slb_insert(kernel_pmap, cache, &entry);
+}
+#endif
+
+static void
+moea64_setup_direct_map(mmu_t mmup, vm_offset_t kernelstart,
+ vm_offset_t kernelend)
+{
+ register_t msr;
+ vm_paddr_t pa;
+ vm_offset_t size, off;
+ uint64_t pte_lo;
+ int i;
+
+ if (moea64_large_page_size == 0)
+ hw_direct_map = 0;
+
+ DISABLE_TRANS(msr);
+ if (hw_direct_map) {
+ PMAP_LOCK(kernel_pmap);
+ for (i = 0; i < pregions_sz; i++) {
+ for (pa = pregions[i].mr_start; pa < pregions[i].mr_start +
+ pregions[i].mr_size; pa += moea64_large_page_size) {
+ pte_lo = LPTE_M;
+
+ /*
+ * Set memory access as guarded if prefetch within
+ * the page could exit the available physmem area.
+ */
+ if (pa & moea64_large_page_mask) {
+ pa &= moea64_large_page_mask;
+ pte_lo |= LPTE_G;
+ }
+ if (pa + moea64_large_page_size >
+ pregions[i].mr_start + pregions[i].mr_size)
+ pte_lo |= LPTE_G;
+
+ moea64_pvo_enter(kernel_pmap, moea64_upvo_zone,
+ &moea64_pvo_kunmanaged, pa, pa,
+ pte_lo, PVO_WIRED | PVO_LARGE |
+ VM_PROT_EXECUTE);
+ }
+ }
+ PMAP_UNLOCK(kernel_pmap);
+ } else {
+ size = moea64_pteg_count * sizeof(struct lpteg);
+ off = (vm_offset_t)(moea64_pteg_table);
+ for (pa = off; pa < off + size; pa += PAGE_SIZE)
+ moea64_kenter(mmup, pa, pa);
+ size = sizeof(struct pvo_head) * moea64_pteg_count;
+ off = (vm_offset_t)(moea64_pvo_table);
+ for (pa = off; pa < off + size; pa += PAGE_SIZE)
+ moea64_kenter(mmup, pa, pa);
+ size = BPVO_POOL_SIZE*sizeof(struct pvo_entry);
+ off = (vm_offset_t)(moea64_bpvo_pool);
+ for (pa = off; pa < off + size; pa += PAGE_SIZE)
+ moea64_kenter(mmup, pa, pa);
+
+ /*
+ * Map certain important things, like ourselves.
+ *
+ * NOTE: We do not map the exception vector space. That code is
+ * used only in real mode, and leaving it unmapped allows us to
+ * catch NULL pointer deferences, instead of making NULL a valid
+ * address.
+ */
+
+ for (pa = kernelstart & ~PAGE_MASK; pa < kernelend;
+ pa += PAGE_SIZE)
+ moea64_kenter(mmup, pa, pa);
+ }
+ ENABLE_TRANS(msr);
+}
+
static void
-moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
+moea64_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernelend)
{
ihandle_t mmui;
phandle_t chosen;
@@ -744,10 +925,11 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
size_t sz;
int i, j;
vm_size_t size, physsz, hwphyssz;
- vm_offset_t pa, va, off;
+ vm_offset_t pa, va;
register_t msr;
void *dpcpu;
+#ifndef __powerpc64__
/* We don't have a direct map since there is no BAT */
hw_direct_map = 0;
@@ -756,6 +938,15 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
battable[i].batu = 0;
battable[i].batl = 0;
}
+#else
+ moea64_probe_large_page();
+
+ /* Use a direct map if we have large page support */
+ if (moea64_large_page_size > 0)
+ hw_direct_map = 1;
+ else
+ hw_direct_map = 0;
+#endif
/* Get physical memory regions from firmware */
mem_regions(&pregions, &pregions_sz, &regions, &regions_sz);
@@ -789,6 +980,38 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
phys_avail_count++;
physsz += regions[i].mr_size;
}
+
+ /* Check for overlap with the kernel and exception vectors */
+ for (j = 0; j < 2*phys_avail_count; j+=2) {
+ if (phys_avail[j] < EXC_LAST)
+ phys_avail[j] += EXC_LAST;
+
+ if (kernelstart >= phys_avail[j] &&
+ kernelstart < phys_avail[j+1]) {
+ if (kernelend < phys_avail[j+1]) {
+ phys_avail[2*phys_avail_count] =
+ (kernelend & ~PAGE_MASK) + PAGE_SIZE;
+ phys_avail[2*phys_avail_count + 1] =
+ phys_avail[j+1];
+ phys_avail_count++;
+ }
+
+ phys_avail[j+1] = kernelstart & ~PAGE_MASK;
+ }
+
+ if (kernelend >= phys_avail[j] &&
+ kernelend < phys_avail[j+1]) {
+ if (kernelstart > phys_avail[j]) {
+ phys_avail[2*phys_avail_count] = phys_avail[j];
+ phys_avail[2*phys_avail_count + 1] =
+ kernelstart & ~PAGE_MASK;
+ phys_avail_count++;
+ }
+
+ phys_avail[j] = (kernelend & ~PAGE_MASK) + PAGE_SIZE;
+ }
+ }
+
physmem = btoc(physsz);
/*
@@ -801,6 +1024,8 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
while (moea64_pteg_count < physmem)
moea64_pteg_count <<= 1;
+
+ moea64_pteg_count >>= 1;
#endif /* PTEGCOUNT */
size = moea64_pteg_count * sizeof(struct lpteg);
@@ -859,15 +1084,24 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
/*
* Make sure kernel vsid is allocated as well as VSID 0.
*/
- moea64_vsid_bitmap[(KERNEL_VSIDBITS & (NPMAPS - 1)) / VSID_NBPW]
+ #ifndef __powerpc64__
+ moea64_vsid_bitmap[(KERNEL_VSIDBITS & (NVSIDS - 1)) / VSID_NBPW]
|= 1 << (KERNEL_VSIDBITS % VSID_NBPW);
moea64_vsid_bitmap[0] |= 1;
+ #endif
/*
* Initialize the kernel pmap (which is statically allocated).
*/
+ #ifdef __powerpc64__
+ for (i = 0; i < 64; i++) {
+ pcpup->pc_slb[i].slbv = 0;
+ pcpup->pc_slb[i].slbe = 0;
+ }
+ #else
for (i = 0; i < 16; i++)
kernel_pmap->pm_sr[i] = EMPTY_SEGMENT + i;
+ #endif
kernel_pmap->pmap_phys = kernel_pmap;
kernel_pmap->pm_active = ~0;
@@ -878,45 +1112,25 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
* Now map in all the other buffers we allocated earlier
*/
- DISABLE_TRANS(msr);
- size = moea64_pteg_count * sizeof(struct lpteg);
- off = (vm_offset_t)(moea64_pteg_table);
- for (pa = off; pa < off + size; pa += PAGE_SIZE)
- moea64_kenter(mmup, pa, pa);
- size = sizeof(struct pvo_head) * moea64_pteg_count;
- off = (vm_offset_t)(moea64_pvo_table);
- for (pa = off; pa < off + size; pa += PAGE_SIZE)
- moea64_kenter(mmup, pa, pa);
- size = BPVO_POOL_SIZE*sizeof(struct pvo_entry);
- off = (vm_offset_t)(moea64_bpvo_pool);
- for (pa = off; pa < off + size; pa += PAGE_SIZE)
- moea64_kenter(mmup, pa, pa);
+ moea64_setup_direct_map(mmup, kernelstart, kernelend);
/*
- * Map certain important things, like ourselves.
- *
- * NOTE: We do not map the exception vector space. That code is
- * used only in real mode, and leaving it unmapped allows us to
- * catch NULL pointer deferences, instead of making NULL a valid
- * address.
+ * Set up the Open Firmware pmap and add its mappings if not in real
+ * mode.
*/
- for (pa = kernelstart & ~PAGE_MASK; pa < kernelend; pa += PAGE_SIZE)
- moea64_kenter(mmup, pa, pa);
- ENABLE_TRANS(msr);
-
if (!ofw_real_mode) {
- /*
- * Set up the Open Firmware pmap and add its mappings.
- */
-
+ #ifndef __powerpc64__
moea64_pinit(mmup, &ofw_pmap);
+
for (i = 0; i < 16; i++)
ofw_pmap.pm_sr[i] = kernel_pmap->pm_sr[i];
+ #endif
if ((chosen = OF_finddevice("/chosen")) == -1)
panic("moea64_bootstrap: can't find /chosen");
OF_getprop(chosen, "mmu", &mmui, 4);
+
if ((mmu = OF_instance_to_package(mmui)) == -1)
panic("moea64_bootstrap: can't get mmu package");
if ((sz = OF_getproplen(mmu, "translations")) == -1)
@@ -941,7 +1155,7 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
/*
* Initialize MMU and remap early physical mappings
*/
- moea64_bridge_cpu_bootstrap(mmup,0);
+ moea64_cpu_bootstrap(mmup,0);
mtmsr(mfmsr() | PSL_DR | PSL_IR); isync();
pmap_bootstrapped++;
bs_remap_earlyboot();
@@ -953,6 +1167,14 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
virtual_end = VM_MAX_SAFE_KERNEL_ADDRESS;
/*
+ * Map the entire KVA range into the SLB. We must not fault there.
+ */
+ #ifdef __powerpc64__
+ for (va = virtual_avail; va < virtual_end; va += SEGMENT_LENGTH)
+ moea64_bootstrap_slb_prefault(va, 0);
+ #endif
+
+ /*
* Figure out how far we can extend virtual_end into segment 16
* without running into existing mappings. Segment 16 is guaranteed
* to contain neither RAM nor devices (at least on Apple hardware),
@@ -960,10 +1182,13 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
* step on.
*/
+ #ifndef __powerpc64__ /* KVA is in high memory on PPC64 */
PMAP_LOCK(kernel_pmap);
- while (moea64_pvo_find_va(kernel_pmap, virtual_end+1, NULL) == NULL)
+ while (virtual_end < VM_MAX_KERNEL_ADDRESS &&
+ moea64_pvo_find_va(kernel_pmap, virtual_end+1) == NULL)
virtual_end += PAGE_SIZE;
PMAP_UNLOCK(kernel_pmap);
+ #endif
/*
* Allocate some things for page zeroing. We put this directly
@@ -972,31 +1197,38 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
* from even knowing that this hack exists.
*/
- mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL, MTX_DEF);
- for (i = 0; i < 2; i++) {
- struct lpte pt;
- uint64_t vsid;
- int pteidx, ptegidx;
-
- moea64_scratchpage_va[i] = (virtual_end+1) - PAGE_SIZE;
- virtual_end -= PAGE_SIZE;
-
- LOCK_TABLE();
-
- vsid = va_to_vsid(kernel_pmap, moea64_scratchpage_va[i]);
- moea64_pte_create(&pt, vsid, moea64_scratchpage_va[i],
- LPTE_NOEXEC);
- pt.pte_hi |= LPTE_LOCKED;
-
- ptegidx = va_to_pteg(vsid, moea64_scratchpage_va[i]);
- pteidx = moea64_pte_insert(ptegidx, &pt);
- if (pt.pte_hi & LPTE_HID)
- ptegidx ^= moea64_pteg_mask;
-
- moea64_scratchpage_pte[i] =
- &moea64_pteg_table[ptegidx].pt[pteidx];
-
- UNLOCK_TABLE();
+ if (!hw_direct_map) {
+ mtx_init(&moea64_scratchpage_mtx, "pvo zero page", NULL,
+ MTX_DEF);
+ for (i = 0; i < 2; i++) {
+ struct lpte pt;
+ uint64_t vsid;
+ int pteidx, ptegidx;
+
+ moea64_scratchpage_va[i] = (virtual_end+1) - PAGE_SIZE;
+ virtual_end -= PAGE_SIZE;
+
+ LOCK_TABLE();
+
+ vsid = va_to_vsid(kernel_pmap,
+ moea64_scratchpage_va[i]);
+ moea64_pte_create(&pt, vsid, moea64_scratchpage_va[i],
+ LPTE_NOEXEC, 0);
+ pt.pte_hi |= LPTE_LOCKED;
+
+ moea64_scratchpage_vpn[i] = (vsid << 16) |
+ ((moea64_scratchpage_va[i] & ADDR_PIDX) >>
+ ADDR_PIDX_SHFT);
+ ptegidx = va_to_pteg(vsid, moea64_scratchpage_va[i], 0);
+ pteidx = moea64_pte_insert(ptegidx, &pt);
+ if (pt.pte_hi & LPTE_HID)
+ ptegidx ^= moea64_pteg_mask;
+
+ moea64_scratchpage_pte[i] =
+ &moea64_pteg_table[ptegidx].pt[pteidx];
+
+ UNLOCK_TABLE();
+ }
}
/*
@@ -1033,6 +1265,7 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
*/
pa = moea64_bootstrap_alloc(DPCPU_SIZE, PAGE_SIZE);
dpcpu = (void *)virtual_avail;
+ va = virtual_avail;
virtual_avail += DPCPU_SIZE;
while (va < virtual_avail) {
moea64_kenter(mmup, va, pa);
@@ -1043,23 +1276,22 @@ moea64_bridge_bootstrap(mmu_t mmup, vm_offset_t kernelstart, vm_offset_t kernele
}
/*
- * Activate a user pmap. The pmap must be activated before it's address
+ * Activate a user pmap. The pmap must be activated before its address
* space can be accessed in any way.
*/
void
moea64_activate(mmu_t mmu, struct thread *td)
{
- pmap_t pm, pmr;
+ pmap_t pm;
- /*
- * Load all the data we need up front to encourage the compiler to
- * not issue any loads while we have interrupts disabled below.
- */
pm = &td->td_proc->p_vmspace->vm_pmap;
- pmr = pm->pmap_phys;
-
pm->pm_active |= PCPU_GET(cpumask);
- PCPU_SET(curpmap, pmr);
+
+ #ifdef __powerpc64__
+ PCPU_SET(userslb, pm->pm_slb);
+ #else
+ PCPU_SET(curpmap, pm->pmap_phys);
+ #endif
}
void
@@ -1069,27 +1301,61 @@ moea64_deactivate(mmu_t mmu, struct thread *td)
pm = &td->td_proc->p_vmspace->vm_pmap;
pm->pm_active &= ~(PCPU_GET(cpumask));
+ #ifdef __powerpc64__
+ PCPU_SET(userslb, NULL);
+ #else
PCPU_SET(curpmap, NULL);
+ #endif
}
void
moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
{
struct pvo_entry *pvo;
+ struct lpte *pt;
+ uint64_t vsid;
+ int i, ptegidx;
PMAP_LOCK(pm);
- pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+ pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF);
if (pvo != NULL) {
+ LOCK_TABLE();
+ pt = moea64_pvo_to_pte(pvo);
+
if (wired) {
if ((pvo->pvo_vaddr & PVO_WIRED) == 0)
pm->pm_stats.wired_count++;
pvo->pvo_vaddr |= PVO_WIRED;
+ pvo->pvo_pte.lpte.pte_hi |= LPTE_WIRED;
} else {
if ((pvo->pvo_vaddr & PVO_WIRED) != 0)
pm->pm_stats.wired_count--;
pvo->pvo_vaddr &= ~PVO_WIRED;
+ pvo->pvo_pte.lpte.pte_hi &= ~LPTE_WIRED;
+ }
+
+ if (pt != NULL) {
+ /* Update wiring flag in page table. */
+ moea64_pte_change(pt, &pvo->pvo_pte.lpte,
+ pvo->pvo_vpn);
+ } else if (wired) {
+ /*
+ * If we are wiring the page, and it wasn't in the
+ * page table before, add it.
+ */
+ vsid = PVO_VSID(pvo);
+ ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo),
+ pvo->pvo_vaddr & PVO_LARGE);
+
+ i = moea64_pte_insert(ptegidx, &pvo->pvo_pte.lpte);
+ if (i >= 0) {
+ PVO_PTEGIDX_CLR(pvo);
+ PVO_PTEGIDX_SET(pvo, i);
+ }
}
+
+ UNLOCK_TABLE();
}
PMAP_UNLOCK(pm);
}
@@ -1104,10 +1370,11 @@ moea64_change_wiring(mmu_t mmu, pmap_t pm, vm_offset_t va, boolean_t wired)
static __inline
void moea64_set_scratchpage_pa(int which, vm_offset_t pa) {
+ KASSERT(!hw_direct_map, ("Using OEA64 scratchpage with a direct map!"));
mtx_assert(&moea64_scratchpage_mtx, MA_OWNED);
moea64_scratchpage_pte[which]->pte_hi &= ~LPTE_VALID;
- TLBIE(kernel_pmap, moea64_scratchpage_va[which]);
+ TLBIE(moea64_scratchpage_vpn[which]);
moea64_scratchpage_pte[which]->pte_lo &=
~(LPTE_WIMG | LPTE_RPGN);
@@ -1128,15 +1395,19 @@ moea64_copy_page(mmu_t mmu, vm_page_t msrc, vm_page_t mdst)
dst = VM_PAGE_TO_PHYS(mdst);
src = VM_PAGE_TO_PHYS(msrc);
- mtx_lock(&moea64_scratchpage_mtx);
+ if (hw_direct_map) {
+ kcopy((void *)src, (void *)dst, PAGE_SIZE);
+ } else {
+ mtx_lock(&moea64_scratchpage_mtx);
- moea64_set_scratchpage_pa(0,src);
- moea64_set_scratchpage_pa(1,dst);
+ moea64_set_scratchpage_pa(0,src);
+ moea64_set_scratchpage_pa(1,dst);
- kcopy((void *)moea64_scratchpage_va[0],
- (void *)moea64_scratchpage_va[1], PAGE_SIZE);
+ kcopy((void *)moea64_scratchpage_va[0],
+ (void *)moea64_scratchpage_va[1], PAGE_SIZE);
- mtx_unlock(&moea64_scratchpage_mtx);
+ mtx_unlock(&moea64_scratchpage_mtx);
+ }
}
void
@@ -1145,15 +1416,18 @@ moea64_zero_page_area(mmu_t mmu, vm_page_t m, int off, int size)
vm_offset_t pa = VM_PAGE_TO_PHYS(m);
if (!moea64_initialized)
- panic("moea64_zero_page: can't zero pa %#x", pa);
+ panic("moea64_zero_page: can't zero pa %#" PRIxPTR, pa);
if (size + off > PAGE_SIZE)
panic("moea64_zero_page: size + off > PAGE_SIZE");
- mtx_lock(&moea64_scratchpage_mtx);
-
- moea64_set_scratchpage_pa(0,pa);
- bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
- mtx_unlock(&moea64_scratchpage_mtx);
+ if (hw_direct_map) {
+ bzero((caddr_t)pa + off, size);
+ } else {
+ mtx_lock(&moea64_scratchpage_mtx);
+ moea64_set_scratchpage_pa(0,pa);
+ bzero((caddr_t)moea64_scratchpage_va[0] + off, size);
+ mtx_unlock(&moea64_scratchpage_mtx);
+ }
}
/*
@@ -1163,18 +1437,25 @@ void
moea64_zero_page(mmu_t mmu, vm_page_t m)
{
vm_offset_t pa = VM_PAGE_TO_PHYS(m);
- vm_offset_t off;
+ vm_offset_t va, off;
if (!moea64_initialized)
- panic("moea64_zero_page: can't zero pa %#x", pa);
+ panic("moea64_zero_page: can't zero pa %#zx", pa);
+
+ if (!hw_direct_map) {
+ mtx_lock(&moea64_scratchpage_mtx);
- mtx_lock(&moea64_scratchpage_mtx);
+ moea64_set_scratchpage_pa(0,pa);
+ va = moea64_scratchpage_va[0];
+ } else {
+ va = pa;
+ }
- moea64_set_scratchpage_pa(0,pa);
for (off = 0; off < PAGE_SIZE; off += cacheline_size)
- __asm __volatile("dcbz 0,%0" ::
- "r"(moea64_scratchpage_va[0] + off));
- mtx_unlock(&moea64_scratchpage_mtx);
+ __asm __volatile("dcbz 0,%0" :: "r"(va + off));
+
+ if (!hw_direct_map)
+ mtx_unlock(&moea64_scratchpage_mtx);
}
void
@@ -1298,6 +1579,8 @@ moea64_syncicache(pmap_t pmap, vm_offset_t va, vm_offset_t pa, vm_size_t sz)
__syncicache((void *)pa, sz);
} else if (pmap == kernel_pmap) {
__syncicache((void *)va, sz);
+ } else if (hw_direct_map) {
+ __syncicache((void *)pa, sz);
} else {
/* Use the scratch page to set up a temp mapping */
@@ -1363,11 +1646,12 @@ moea64_extract(mmu_t mmu, pmap_t pm, vm_offset_t va)
vm_paddr_t pa;
PMAP_LOCK(pm);
- pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+ pvo = moea64_pvo_find_va(pm, va);
if (pvo == NULL)
pa = 0;
else
- pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va & ADDR_POFF);
+ pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) |
+ (va - PVO_VADDR(pvo));
PMAP_UNLOCK(pm);
return (pa);
}
@@ -1388,7 +1672,7 @@ moea64_extract_and_hold(mmu_t mmu, pmap_t pmap, vm_offset_t va, vm_prot_t prot)
pa = 0;
PMAP_LOCK(pmap);
retry:
- pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF, NULL);
+ pvo = moea64_pvo_find_va(pmap, va & ~ADDR_POFF);
if (pvo != NULL && (pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) &&
((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) == LPTE_RW ||
(prot & VM_PROT_WRITE) == 0)) {
@@ -1565,7 +1849,7 @@ moea64_remove_write(mmu_t mmu, vm_page_t m)
PMAP_LOCK(pmap);
LOCK_TABLE();
if ((pvo->pvo_pte.lpte.pte_lo & LPTE_PP) != LPTE_BR) {
- pt = moea64_pvo_to_pte(pvo, -1);
+ pt = moea64_pvo_to_pte(pvo);
pvo->pvo_pte.lpte.pte_lo &= ~LPTE_PP;
pvo->pvo_pte.lpte.pte_lo |= LPTE_BR;
if (pt != NULL) {
@@ -1573,7 +1857,9 @@ moea64_remove_write(mmu_t mmu, vm_page_t m)
lo |= pvo->pvo_pte.lpte.pte_lo;
pvo->pvo_pte.lpte.pte_lo &= ~LPTE_CHG;
moea64_pte_change(pt, &pvo->pvo_pte.lpte,
- pvo->pvo_pmap, PVO_VADDR(pvo));
+ pvo->pvo_vpn);
+ if (pvo->pvo_pmap == kernel_pmap)
+ isync();
}
}
UNLOCK_TABLE();
@@ -1620,7 +1906,8 @@ moea64_kenter(mmu_t mmu, vm_offset_t va, vm_offset_t pa)
#if 0
if (!pmap_bootstrapped) {
if (va >= VM_MIN_KERNEL_ADDRESS && va < virtual_end)
- panic("Trying to enter an address in KVA -- %#x!\n",pa);
+ panic("Trying to enter an address in KVA -- %#"
+ PRIxPTR "!\n",pa);
}
#endif
@@ -1632,7 +1919,7 @@ moea64_kenter(mmu_t mmu, vm_offset_t va, vm_offset_t pa)
PVO_WIRED | VM_PROT_EXECUTE);
if (error != 0 && error != ENOENT)
- panic("moea64_kenter: failed to enter va %#x pa %#x: %d", va,
+ panic("moea64_kenter: failed to enter va %#zx pa %#zx: %d", va,
pa, error);
/*
@@ -1662,9 +1949,10 @@ moea64_kextract(mmu_t mmu, vm_offset_t va)
return (va);
PMAP_LOCK(kernel_pmap);
- pvo = moea64_pvo_find_va(kernel_pmap, va & ~ADDR_POFF, NULL);
- KASSERT(pvo != NULL, ("moea64_kextract: no addr found"));
- pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) | (va & ADDR_POFF);
+ pvo = moea64_pvo_find_va(kernel_pmap, va);
+ KASSERT(pvo != NULL, ("moea64_kextract: no addr found for %#" PRIxPTR,
+ va));
+ pa = (pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) + (va - PVO_VADDR(pvo));
PMAP_UNLOCK(kernel_pmap);
return (pa);
}
@@ -1754,29 +2042,20 @@ moea64_page_wired_mappings(mmu_t mmu, vm_page_t m)
return (count);
}
-static u_int moea64_vsidcontext;
+static uintptr_t moea64_vsidcontext;
-void
-moea64_pinit(mmu_t mmu, pmap_t pmap)
-{
- int i, mask;
- u_int entropy;
-
- PMAP_LOCK_INIT(pmap);
+uintptr_t
+moea64_get_unique_vsid(void) {
+ u_int entropy;
+ register_t hash;
+ uint32_t mask;
+ int i;
entropy = 0;
__asm __volatile("mftb %0" : "=r"(entropy));
- if (pmap_bootstrapped)
- pmap->pmap_phys = (pmap_t)moea64_kextract(mmu, (vm_offset_t)pmap);
- else
- pmap->pmap_phys = pmap;
-
- /*
- * Allocate some segment registers for this pmap.
- */
- for (i = 0; i < NPMAPS; i += VSID_NBPW) {
- u_int hash, n;
+ for (i = 0; i < NVSIDS; i += VSID_NBPW) {
+ u_int n;
/*
* Create a new value by mutiplying by a prime and adding in
@@ -1786,12 +2065,12 @@ moea64_pinit(mmu_t mmu, pmap_t pmap)
* instead of a multiply.)
*/
moea64_vsidcontext = (moea64_vsidcontext * 0x1105) + entropy;
- hash = moea64_vsidcontext & (NPMAPS - 1);
+ hash = moea64_vsidcontext & (NVSIDS - 1);
if (hash == 0) /* 0 is special, avoid it */
continue;
n = hash >> 5;
mask = 1 << (hash & (VSID_NBPW - 1));
- hash = (moea64_vsidcontext & 0xfffff);
+ hash = (moea64_vsidcontext & VSID_HASHMASK);
if (moea64_vsid_bitmap[n] & mask) { /* collision? */
/* anything free in this bucket? */
if (moea64_vsid_bitmap[n] == 0xffffffff) {
@@ -1800,18 +2079,49 @@ moea64_pinit(mmu_t mmu, pmap_t pmap)
}
i = ffs(~moea64_vsid_bitmap[i]) - 1;
mask = 1 << i;
- hash &= 0xfffff & ~(VSID_NBPW - 1);
+ hash &= VSID_HASHMASK & ~(VSID_NBPW - 1);
hash |= i;
}
moea64_vsid_bitmap[n] |= mask;
- for (i = 0; i < 16; i++) {
- pmap->pm_sr[i] = VSID_MAKE(i, hash);
- }
- return;
+ return (hash);
}
- panic("moea64_pinit: out of segments");
+ panic("%s: out of segments",__func__);
+}
+
+#ifdef __powerpc64__
+void
+moea64_pinit(mmu_t mmu, pmap_t pmap)
+{
+ PMAP_LOCK_INIT(pmap);
+
+ SPLAY_INIT(&pmap->pm_slbtree);
+ pmap->pm_slb = slb_alloc_user_cache();
+}
+#else
+void
+moea64_pinit(mmu_t mmu, pmap_t pmap)
+{
+ int i;
+ register_t hash;
+
+ PMAP_LOCK_INIT(pmap);
+
+ if (pmap_bootstrapped)
+ pmap->pmap_phys = (pmap_t)moea64_kextract(mmu,
+ (vm_offset_t)pmap);
+ else
+ pmap->pmap_phys = pmap;
+
+ /*
+ * Allocate some segment registers for this pmap.
+ */
+ hash = moea64_get_unique_vsid();
+
+ for (i = 0; i < 16; i++)
+ pmap->pm_sr[i] = VSID_MAKE(i, hash);
}
+#endif
/*
* Initialize the pmap associated with process 0.
@@ -1832,7 +2142,6 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
{
struct pvo_entry *pvo;
struct lpte *pt;
- int pteidx;
CTR4(KTR_PMAP, "moea64_protect: pm=%p sva=%#x eva=%#x prot=%#x", pm, sva,
eva, prot);
@@ -1849,7 +2158,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
vm_page_lock_queues();
PMAP_LOCK(pm);
for (; sva < eva; sva += PAGE_SIZE) {
- pvo = moea64_pvo_find_va(pm, sva, &pteidx);
+ pvo = moea64_pvo_find_va(pm, sva);
if (pvo == NULL)
continue;
@@ -1858,7 +2167,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
* copy.
*/
LOCK_TABLE();
- pt = moea64_pvo_to_pte(pvo, pteidx);
+ pt = moea64_pvo_to_pte(pvo);
/*
* Change the protection of the page.
@@ -1873,8 +2182,7 @@ moea64_protect(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva,
* If the PVO is in the page table, update that pte as well.
*/
if (pt != NULL) {
- moea64_pte_change(pt, &pvo->pvo_pte.lpte,
- pvo->pvo_pmap, PVO_VADDR(pvo));
+ moea64_pte_change(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
if ((pvo->pvo_pte.lpte.pte_lo &
(LPTE_I | LPTE_G | LPTE_NOEXEC)) == 0) {
moea64_syncicache(pm, sva,
@@ -1917,20 +2225,34 @@ moea64_qremove(mmu_t mmu, vm_offset_t va, int count)
}
void
-moea64_release(mmu_t mmu, pmap_t pmap)
+moea64_release_vsid(uint64_t vsid)
{
int idx, mask;
+
+ idx = vsid & (NVSIDS-1);
+ mask = 1 << (idx % VSID_NBPW);
+ idx /= VSID_NBPW;
+ moea64_vsid_bitmap[idx] &= ~mask;
+}
+
+
+void
+moea64_release(mmu_t mmu, pmap_t pmap)
+{
/*
- * Free segment register's VSID
+ * Free segment registers' VSIDs
*/
+ #ifdef __powerpc64__
+ free_vsids(pmap);
+ slb_free_user_cache(pmap->pm_slb);
+ #else
if (pmap->pm_sr[0] == 0)
panic("moea64_release");
- idx = VSID_TO_HASH(pmap->pm_sr[0]) & (NPMAPS-1);
- mask = 1 << (idx % VSID_NBPW);
- idx /= VSID_NBPW;
- moea64_vsid_bitmap[idx] &= ~mask;
+ moea64_release_vsid(pmap->pm_sr[0]);
+ #endif
+
PMAP_LOCK_DESTROY(pmap);
}
@@ -1941,15 +2263,13 @@ void
moea64_remove(mmu_t mmu, pmap_t pm, vm_offset_t sva, vm_offset_t eva)
{
struct pvo_entry *pvo;
- int pteidx;
vm_page_lock_queues();
PMAP_LOCK(pm);
for (; sva < eva; sva += PAGE_SIZE) {
- pvo = moea64_pvo_find_va(pm, sva, &pteidx);
- if (pvo != NULL) {
- moea64_pvo_remove(pvo, pteidx);
- }
+ pvo = moea64_pvo_find_va(pm, sva);
+ if (pvo != NULL)
+ moea64_pvo_remove(pvo);
}
vm_page_unlock_queues();
PMAP_UNLOCK(pm);
@@ -1974,7 +2294,7 @@ moea64_remove_all(mmu_t mmu, vm_page_t m)
MOEA_PVO_CHECK(pvo); /* sanity check */
pmap = pvo->pvo_pmap;
PMAP_LOCK(pmap);
- moea64_pvo_remove(pvo, -1);
+ moea64_pvo_remove(pvo);
PMAP_UNLOCK(pmap);
}
if ((m->flags & PG_WRITEABLE) && moea64_is_modified(mmu, m)) {
@@ -2032,25 +2352,45 @@ static void
tlbia(void)
{
vm_offset_t i;
+ #ifndef __powerpc64__
register_t msr, scratch;
+ #endif
+
+ TLBSYNC();
for (i = 0; i < 0xFF000; i += 0x00001000) {
+ #ifdef __powerpc64__
+ __asm __volatile("tlbiel %0" :: "r"(i));
+ #else
__asm __volatile("\
mfmsr %0; \
mr %1, %0; \
insrdi %1,%3,1,0; \
mtmsrd %1; \
- ptesync; \
+ isync; \
\
tlbiel %2; \
\
mtmsrd %0; \
- eieio; \
- tlbsync; \
- ptesync;"
+ isync;"
: "=r"(msr), "=r"(scratch) : "r"(i), "r"(1));
+ #endif
}
+
+ EIEIO();
+ TLBSYNC();
+}
+
+#ifdef __powerpc64__
+static void
+slbia(void)
+{
+ register_t seg0;
+
+ __asm __volatile ("slbia");
+ __asm __volatile ("slbmfee %0,%1; slbie %0;" : "=r"(seg0) : "r"(0));
}
+#endif
static int
moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
@@ -2084,7 +2424,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
*/
va &= ~ADDR_POFF;
vsid = va_to_vsid(pm, va);
- ptegidx = va_to_pteg(vsid, va);
+ ptegidx = va_to_pteg(vsid, va, flags & PVO_LARGE);
/*
* Remove any existing mapping for this page. Reuse the pvo entry if
@@ -2097,10 +2437,18 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
if ((pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) == pa &&
(pvo->pvo_pte.lpte.pte_lo & LPTE_PP) ==
(pte_lo & LPTE_PP)) {
+ if (!(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID)) {
+ /* Re-insert if spilled */
+ i = moea64_pte_insert(ptegidx,
+ &pvo->pvo_pte.lpte);
+ if (i >= 0)
+ PVO_PTEGIDX_SET(pvo, i);
+ moea64_pte_overflow--;
+ }
UNLOCK_TABLE();
return (0);
}
- moea64_pvo_remove(pvo, -1);
+ moea64_pvo_remove(pvo);
break;
}
}
@@ -2110,7 +2458,7 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
*/
if (bootstrap) {
if (moea64_bpvo_pool_index >= BPVO_POOL_SIZE) {
- panic("moea64_enter: bpvo pool exhausted, %d, %d, %d",
+ panic("moea64_enter: bpvo pool exhausted, %d, %d, %zd",
moea64_bpvo_pool_index, BPVO_POOL_SIZE,
BPVO_POOL_SIZE * sizeof(struct pvo_entry));
}
@@ -2136,6 +2484,8 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
moea64_pvo_entries++;
pvo->pvo_vaddr = va;
+ pvo->pvo_vpn = (uint64_t)((va & ADDR_PIDX) >> ADDR_PIDX_SHFT)
+ | (vsid << 16);
pvo->pvo_pmap = pm;
LIST_INSERT_HEAD(&moea64_pvo_table[ptegidx], pvo, pvo_olink);
pvo->pvo_vaddr &= ~ADDR_POFF;
@@ -2150,9 +2500,11 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
pvo->pvo_vaddr |= PVO_BOOTSTRAP;
if (flags & PVO_FAKE)
pvo->pvo_vaddr |= PVO_FAKE;
+ if (flags & PVO_LARGE)
+ pvo->pvo_vaddr |= PVO_LARGE;
moea64_pte_create(&pvo->pvo_pte.lpte, vsid, va,
- (uint64_t)(pa) | pte_lo);
+ (uint64_t)(pa) | pte_lo, flags);
/*
* Remember if the list was empty and therefore will be the first
@@ -2162,8 +2514,10 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
first = 1;
LIST_INSERT_HEAD(pvo_head, pvo, pvo_vlink);
- if (pvo->pvo_vaddr & PVO_WIRED)
+ if (pvo->pvo_vaddr & PVO_WIRED) {
+ pvo->pvo_pte.lpte.pte_hi |= LPTE_WIRED;
pm->pm_stats.wired_count++;
+ }
pm->pm_stats.resident_count++;
/*
@@ -2182,11 +2536,20 @@ moea64_pvo_enter(pmap_t pm, uma_zone_t zone, struct pvo_head *pvo_head,
UNLOCK_TABLE();
+#ifdef __powerpc64__
+ /*
+ * Make sure all our bootstrap mappings are in the SLB as soon
+ * as virtual memory is switched on.
+ */
+ if (!pmap_bootstrapped)
+ moea64_bootstrap_slb_prefault(va, flags & PVO_LARGE);
+#endif
+
return (first ? ENOENT : 0);
}
static void
-moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
+moea64_pvo_remove(struct pvo_entry *pvo)
{
struct lpte *pt;
@@ -2195,10 +2558,9 @@ moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
* save the ref & cfg bits).
*/
LOCK_TABLE();
- pt = moea64_pvo_to_pte(pvo, pteidx);
+ pt = moea64_pvo_to_pte(pvo);
if (pt != NULL) {
- moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_pmap,
- PVO_VADDR(pvo));
+ moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
PVO_PTEGIDX_CLR(pvo);
} else {
moea64_pte_overflow--;
@@ -2244,39 +2606,35 @@ moea64_pvo_remove(struct pvo_entry *pvo, int pteidx)
moea64_pvo_remove_calls++;
}
-static __inline int
-moea64_pvo_pte_index(const struct pvo_entry *pvo, int ptegidx)
-{
-
- /*
- * We can find the actual pte entry without searching by grabbing
- * the PTEG index from 3 unused bits in pvo_vaddr and by
- * noticing the HID bit.
- */
- if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
- ptegidx ^= moea64_pteg_mask;
-
- return ((ptegidx << 3) | PVO_PTEGIDX_GET(pvo));
-}
-
static struct pvo_entry *
-moea64_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p)
+moea64_pvo_find_va(pmap_t pm, vm_offset_t va)
{
struct pvo_entry *pvo;
int ptegidx;
uint64_t vsid;
+ #ifdef __powerpc64__
+ struct slb slb;
+ /* The page is not mapped if the segment isn't */
+ if (va_to_slb_entry(pm, va, &slb) != 0)
+ return NULL;
+
+ vsid = (slb.slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT;
+ if (slb.slbv & SLBV_L)
+ va &= ~moea64_large_page_mask;
+ else
+ va &= ~ADDR_POFF;
+ ptegidx = va_to_pteg(vsid, va, slb.slbv & SLBV_L);
+ #else
va &= ~ADDR_POFF;
vsid = va_to_vsid(pm, va);
- ptegidx = va_to_pteg(vsid, va);
+ ptegidx = va_to_pteg(vsid, va, 0);
+ #endif
LOCK_TABLE();
LIST_FOREACH(pvo, &moea64_pvo_table[ptegidx], pvo_olink) {
- if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va) {
- if (pteidx_p)
- *pteidx_p = moea64_pvo_pte_index(pvo, ptegidx);
+ if (pvo->pvo_pmap == pm && PVO_VADDR(pvo) == va)
break;
- }
}
UNLOCK_TABLE();
@@ -2284,23 +2642,34 @@ moea64_pvo_find_va(pmap_t pm, vm_offset_t va, int *pteidx_p)
}
static struct lpte *
-moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
+moea64_pvo_to_pte(const struct pvo_entry *pvo)
{
- struct lpte *pt;
+ struct lpte *pt;
+ int pteidx, ptegidx;
+ uint64_t vsid;
+
+ ASSERT_TABLE_LOCK();
+
+ /* If the PTEG index is not set, then there is no page table entry */
+ if (!PVO_PTEGIDX_ISSET(pvo))
+ return (NULL);
/*
- * If we haven't been supplied the ptegidx, calculate it.
+ * Calculate the ptegidx
*/
- if (pteidx == -1) {
- int ptegidx;
- uint64_t vsid;
+ vsid = PVO_VSID(pvo);
+ ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo),
+ pvo->pvo_vaddr & PVO_LARGE);
- vsid = va_to_vsid(pvo->pvo_pmap, PVO_VADDR(pvo));
- ptegidx = va_to_pteg(vsid, PVO_VADDR(pvo));
- pteidx = moea64_pvo_pte_index(pvo, ptegidx);
- }
+ /*
+ * We can find the actual pte entry without searching by grabbing
+ * the PTEG index from 3 unused bits in pvo_vaddr and by
+ * noticing the HID bit.
+ */
+ if (pvo->pvo_pte.lpte.pte_hi & LPTE_HID)
+ ptegidx ^= moea64_pteg_mask;
- pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7];
+ pteidx = (ptegidx << 3) | PVO_PTEGIDX_GET(pvo);
if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) &&
!PVO_PTEGIDX_ISSET(pvo)) {
@@ -2314,6 +2683,7 @@ moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
"pvo but no valid pte", pvo);
}
+ pt = &moea64_pteg_table[pteidx >> 3].pt[pteidx & 7];
if ((pt->pte_hi ^ (pvo->pvo_pte.lpte.pte_hi & ~LPTE_VALID)) ==
LPTE_VALID) {
if ((pvo->pvo_pte.lpte.pte_hi & LPTE_VALID) == 0) {
@@ -2329,7 +2699,6 @@ moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
(uint32_t)(pt->pte_lo ^ pvo->pvo_pte.lpte.pte_lo));
}
- ASSERT_TABLE_LOCK();
return (pt);
}
@@ -2341,10 +2710,37 @@ moea64_pvo_to_pte(const struct pvo_entry *pvo, int pteidx)
return (NULL);
}
+static __inline int
+moea64_pte_spillable_ident(u_int ptegidx)
+{
+ struct lpte *pt;
+ int i, j, k;
+
+ /* Start at a random slot */
+ i = mftb() % 8;
+ k = -1;
+ for (j = 0; j < 8; j++) {
+ pt = &moea64_pteg_table[ptegidx].pt[(i + j) % 8];
+ if (pt->pte_hi & (LPTE_LOCKED | LPTE_WIRED))
+ continue;
+
+ /* This is a candidate, so remember it */
+ k = (i + j) % 8;
+
+ /* Try to get a page that has not been used lately */
+ if (!(pt->pte_lo & LPTE_REF))
+ return (k);
+ }
+
+ return (k);
+}
+
static int
moea64_pte_insert(u_int ptegidx, struct lpte *pvo_pt)
{
struct lpte *pt;
+ struct pvo_entry *pvo;
+ u_int pteg_bktidx;
int i;
ASSERT_TABLE_LOCK();
@@ -2352,9 +2748,9 @@ moea64_pte_insert(u_int ptegidx, struct lpte *pvo_pt)
/*
* First try primary hash.
*/
- for (pt = moea64_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
- if ((pt->pte_hi & LPTE_VALID) == 0 &&
- (pt->pte_hi & LPTE_LOCKED) == 0) {
+ pteg_bktidx = ptegidx;
+ for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) {
+ if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) {
pvo_pt->pte_hi &= ~LPTE_HID;
moea64_pte_set(pt, pvo_pt);
return (i);
@@ -2364,19 +2760,66 @@ moea64_pte_insert(u_int ptegidx, struct lpte *pvo_pt)
/*
* Now try secondary hash.
*/
- ptegidx ^= moea64_pteg_mask;
-
- for (pt = moea64_pteg_table[ptegidx].pt, i = 0; i < 8; i++, pt++) {
- if ((pt->pte_hi & LPTE_VALID) == 0 &&
- (pt->pte_hi & LPTE_LOCKED) == 0) {
+ pteg_bktidx ^= moea64_pteg_mask;
+ for (pt = moea64_pteg_table[pteg_bktidx].pt, i = 0; i < 8; i++, pt++) {
+ if ((pt->pte_hi & (LPTE_VALID | LPTE_LOCKED)) == 0) {
pvo_pt->pte_hi |= LPTE_HID;
moea64_pte_set(pt, pvo_pt);
return (i);
}
}
- panic("moea64_pte_insert: overflow");
- return (-1);
+ /*
+ * Out of luck. Find a PTE to sacrifice.
+ */
+ pteg_bktidx = ptegidx;
+ i = moea64_pte_spillable_ident(pteg_bktidx);
+ if (i < 0) {
+ pteg_bktidx ^= moea64_pteg_mask;
+ i = moea64_pte_spillable_ident(pteg_bktidx);
+ }
+
+ if (i < 0) {
+ /* No freeable slots in either PTEG? We're hosed. */
+ panic("moea64_pte_insert: overflow");
+ return (-1);
+ }
+
+ if (pteg_bktidx == ptegidx)
+ pvo_pt->pte_hi &= ~LPTE_HID;
+ else
+ pvo_pt->pte_hi |= LPTE_HID;
+
+ /*
+ * Synchronize the sacrifice PTE with its PVO, then mark both
+ * invalid. The PVO will be reused when/if the VM system comes
+ * here after a fault.
+ */
+ pt = &moea64_pteg_table[pteg_bktidx].pt[i];
+
+ if (pt->pte_hi & LPTE_HID)
+ pteg_bktidx ^= moea64_pteg_mask; /* PTEs indexed by primary */
+
+ LIST_FOREACH(pvo, &moea64_pvo_table[pteg_bktidx], pvo_olink) {
+ if (pvo->pvo_pte.lpte.pte_hi == pt->pte_hi) {
+ KASSERT(pvo->pvo_pte.lpte.pte_hi & LPTE_VALID,
+ ("Invalid PVO for valid PTE!"));
+ moea64_pte_unset(pt, &pvo->pvo_pte.lpte, pvo->pvo_vpn);
+ PVO_PTEGIDX_CLR(pvo);
+ moea64_pte_overflow++;
+ break;
+ }
+ }
+
+ KASSERT(pvo->pvo_pte.lpte.pte_hi == pt->pte_hi,
+ ("Unable to find PVO for spilled PTE"));
+
+ /*
+ * Set the new PTE.
+ */
+ moea64_pte_set(pt, pvo_pt);
+
+ return (i);
}
static boolean_t
@@ -2420,7 +2863,7 @@ moea64_query_bit(vm_page_t m, u_int64_t ptebit)
* ptebit is set, cache it and return success.
*/
LOCK_TABLE();
- pt = moea64_pvo_to_pte(pvo, -1);
+ pt = moea64_pvo_to_pte(pvo);
if (pt != NULL) {
moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
@@ -2471,12 +2914,12 @@ moea64_clear_bit(vm_page_t m, u_int64_t ptebit)
MOEA_PVO_CHECK(pvo); /* sanity check */
LOCK_TABLE();
- pt = moea64_pvo_to_pte(pvo, -1);
+ pt = moea64_pvo_to_pte(pvo);
if (pt != NULL) {
moea64_pte_synch(pt, &pvo->pvo_pte.lpte);
if (pvo->pvo_pte.lpte.pte_lo & ptebit) {
count++;
- moea64_pte_clear(pt, pvo->pvo_pmap, PVO_VADDR(pvo), ptebit);
+ moea64_pte_clear(pt, pvo->pvo_vpn, ptebit);
}
}
pvo->pvo_pte.lpte.pte_lo &= ~ptebit;
@@ -2497,7 +2940,7 @@ moea64_dev_direct_mapped(mmu_t mmu, vm_offset_t pa, vm_size_t size)
PMAP_LOCK(kernel_pmap);
for (ppa = pa & ~ADDR_POFF; ppa < pa + size; ppa += PAGE_SIZE) {
- pvo = moea64_pvo_find_va(kernel_pmap, ppa, NULL);
+ pvo = moea64_pvo_find_va(kernel_pmap, ppa);
if (pvo == NULL ||
(pvo->pvo_pte.lpte.pte_lo & LPTE_RPGN) != ppa) {
error = EFAULT;
@@ -2563,7 +3006,7 @@ moea64_sync_icache(mmu_t mmu, pmap_t pm, vm_offset_t va, vm_size_t sz)
while (sz > 0) {
lim = round_page(va);
len = MIN(lim - va, sz);
- pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF, NULL);
+ pvo = moea64_pvo_find_va(pm, va & ~ADDR_POFF);
if (pvo != NULL) {
pa = (pvo->pvo_pte.pte.pte_lo & LPTE_RPGN) |
(va & ADDR_POFF);
diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c
index 8741463..b042365 100644
--- a/sys/powerpc/aim/mp_cpudep.c
+++ b/sys/powerpc/aim/mp_cpudep.c
@@ -55,6 +55,31 @@ static register_t bsp_state[8] __aligned(8);
static void cpudep_save_config(void *dummy);
SYSINIT(cpu_save_config, SI_SUB_CPU, SI_ORDER_ANY, cpudep_save_config, NULL);
+void
+cpudep_ap_early_bootstrap(void)
+{
+ register_t reg;
+
+ __asm __volatile("mtsprg 0, %0" :: "r"(ap_pcpu));
+ powerpc_sync();
+
+ switch (mfpvr() >> 16) {
+ case IBM970:
+ case IBM970FX:
+ case IBM970MP:
+ /* Restore HID4 and HID5, which are necessary for the MMU */
+
+ __asm __volatile("ld %0, 16(%2); sync; isync; \
+ mtspr %1, %0; sync; isync;"
+ : "=r"(reg) : "K"(SPR_HID4), "r"(bsp_state));
+ __asm __volatile("ld %0, 24(%2); sync; isync; \
+ mtspr %1, %0; sync; isync;"
+ : "=r"(reg) : "K"(SPR_HID5), "r"(bsp_state));
+ powerpc_sync();
+ break;
+ }
+}
+
uintptr_t
cpudep_ap_bootstrap(void)
{
@@ -64,9 +89,6 @@ cpudep_ap_bootstrap(void)
mtmsr(msr);
isync();
- __asm __volatile("mtsprg 0, %0" :: "r"(ap_pcpu));
- powerpc_sync();
-
pcpup->pc_curthread = pcpup->pc_idlethread;
pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb;
sp = pcpup->pc_curpcb->pcb_sp;
@@ -187,6 +209,12 @@ cpudep_save_config(void *dummy)
case IBM970:
case IBM970FX:
case IBM970MP:
+ #ifdef __powerpc64__
+ bsp_state[0] = mfspr(SPR_HID0);
+ bsp_state[1] = mfspr(SPR_HID1);
+ bsp_state[2] = mfspr(SPR_HID4);
+ bsp_state[3] = mfspr(SPR_HID5);
+ #else
__asm __volatile ("mfspr %0,%2; mr %1,%0; srdi %0,%0,32"
: "=r" (bsp_state[0]),"=r" (bsp_state[1]) : "K" (SPR_HID0));
__asm __volatile ("mfspr %0,%2; mr %1,%0; srdi %0,%0,32"
@@ -195,6 +223,7 @@ cpudep_save_config(void *dummy)
: "=r" (bsp_state[4]),"=r" (bsp_state[5]) : "K" (SPR_HID4));
__asm __volatile ("mfspr %0,%2; mr %1,%0; srdi %0,%0,32"
: "=r" (bsp_state[6]),"=r" (bsp_state[7]) : "K" (SPR_HID5));
+ #endif
powerpc_sync();
diff --git a/sys/powerpc/aim/ofw_machdep.c b/sys/powerpc/aim/ofw_machdep.c
index 358afaa..21b8847 100644
--- a/sys/powerpc/aim/ofw_machdep.c
+++ b/sys/powerpc/aim/ofw_machdep.c
@@ -63,12 +63,7 @@ __FBSDID("$FreeBSD$");
#define OFMEM_REGIONS 32
static struct mem_region OFmem[OFMEM_REGIONS + 1], OFavail[OFMEM_REGIONS + 3];
static struct mem_region OFfree[OFMEM_REGIONS + 3];
-
-struct mem_region64 {
- vm_offset_t mr_start_hi;
- vm_offset_t mr_start_lo;
- vm_size_t mr_size;
-};
+static int nOFmem;
extern register_t ofmsr[5];
extern struct pmap ofw_pmap;
@@ -76,6 +71,7 @@ static int (*ofwcall)(void *);
static void *fdt;
int ofw_real_mode;
+int ofw_32bit_mode_entry(void *);
static void ofw_quiesce(void);
static int openfirmware(void *args);
@@ -133,30 +129,23 @@ memr_overlap(struct mem_region *r1, struct mem_region *r2)
static void
memr_merge(struct mem_region *from, struct mem_region *to)
{
- int end;
- end = imax(to->mr_start + to->mr_size, from->mr_start + from->mr_size);
- to->mr_start = imin(from->mr_start, to->mr_start);
+ vm_offset_t end;
+ end = ulmax(to->mr_start + to->mr_size, from->mr_start + from->mr_size);
+ to->mr_start = ulmin(from->mr_start, to->mr_start);
to->mr_size = end - to->mr_start;
}
-/*
- * This is called during powerpc_init, before the system is really initialized.
- * It shall provide the total and the available regions of RAM.
- * Both lists must have a zero-size entry as terminator.
- * The available regions need not take the kernel into account, but needs
- * to provide space for two additional entry beyond the terminating one.
- */
-void
-ofw_mem_regions(struct mem_region **memp, int *memsz,
- struct mem_region **availp, int *availsz)
+static int
+parse_ofw_memory(phandle_t node, const char *prop, struct mem_region *output)
{
+ cell_t address_cells, size_cells;
+ cell_t OFmem[4*(OFMEM_REGIONS + 1)];
+ int sz, i, j;
+ int apple_hack_mode;
phandle_t phandle;
- int asz, msz, fsz;
- int i, j;
- int still_merging;
- cell_t address_cells;
- asz = msz = 0;
+ sz = 0;
+ apple_hack_mode = 0;
/*
* Get #address-cells from root node, defaulting to 1 if it cannot
@@ -166,71 +155,127 @@ ofw_mem_regions(struct mem_region **memp, int *memsz,
if (OF_getprop(phandle, "#address-cells", &address_cells,
sizeof(address_cells)) < sizeof(address_cells))
address_cells = 1;
+ if (OF_getprop(phandle, "#size-cells", &size_cells,
+ sizeof(size_cells)) < sizeof(size_cells))
+ size_cells = 1;
+
+ /*
+ * On Apple hardware, address_cells is always 1 for "available",
+ * even when it is explicitly set to 2. Then all memory above 4 GB
+ * should be added by hand to the available list. Detect Apple hardware
+ * by seeing if ofw_real_mode is set -- only Apple seems to use
+ * virtual-mode OF.
+ */
+ if (strcmp(prop, "available") == 0 && !ofw_real_mode)
+ apple_hack_mode = 1;
+ if (apple_hack_mode)
+ address_cells = 1;
+
/*
* Get memory.
*/
- if ((phandle = OF_finddevice("/memory")) == -1
- || (asz = OF_getprop(phandle, "available",
- OFavail, sizeof OFavail[0] * OFMEM_REGIONS)) <= 0)
- {
- if (ofw_real_mode) {
- /* XXX MAMBO */
- printf("Physical memory unknown -- guessing 128 MB\n");
+ if ((node == -1) || (sz = OF_getprop(node, prop,
+ OFmem, sizeof(OFmem[0]) * 4 * OFMEM_REGIONS)) <= 0)
+ panic("Physical memory map not found");
- /* Leave the first 0xA000000 bytes for the kernel */
- OFavail[0].mr_start = 0xA00000;
- OFavail[0].mr_size = 0x75FFFFF;
- asz = sizeof(OFavail[0]);
- } else {
- panic("no memory?");
+ i = 0;
+ j = 0;
+ while (i < sz/sizeof(cell_t)) {
+ #ifndef __powerpc64__
+ /* On 32-bit PPC, ignore regions starting above 4 GB */
+ if (address_cells > 1 && OFmem[i] > 0) {
+ i += address_cells + size_cells;
+ continue;
+ }
+ #endif
+
+ output[j].mr_start = OFmem[i++];
+ if (address_cells == 2) {
+ #ifdef __powerpc64__
+ output[j].mr_start <<= 32;
+ #endif
+ output[j].mr_start += OFmem[i++];
+ }
+
+ output[j].mr_size = OFmem[i++];
+ if (size_cells == 2) {
+ #ifdef __powerpc64__
+ output[j].mr_size <<= 32;
+ #endif
+ output[j].mr_size += OFmem[i++];
}
- }
- if (address_cells == 2) {
- struct mem_region64 OFmem64[OFMEM_REGIONS + 1];
- if ((phandle == -1) || (msz = OF_getprop(phandle, "reg",
- OFmem64, sizeof OFmem64[0] * OFMEM_REGIONS)) <= 0) {
- if (ofw_real_mode) {
- /* XXX MAMBO */
- OFmem64[0].mr_start_hi = 0;
- OFmem64[0].mr_start_lo = 0x0;
- OFmem64[0].mr_size = 0x7FFFFFF;
- msz = sizeof(OFmem64[0]);
- } else {
- panic("Physical memory map not found");
+ #ifndef __powerpc64__
+ /*
+ * Check for memory regions extending above 32-bit
+ * memory space, and restrict them to stay there.
+ */
+ if (((uint64_t)output[j].mr_start +
+ (uint64_t)output[j].mr_size) >
+ BUS_SPACE_MAXADDR_32BIT) {
+ output[j].mr_size = BUS_SPACE_MAXADDR_32BIT -
+ output[j].mr_start;
}
- }
-
- for (i = 0, j = 0; i < msz/sizeof(OFmem64[0]); i++) {
- if (OFmem64[i].mr_start_hi == 0) {
- OFmem[i].mr_start = OFmem64[i].mr_start_lo;
- OFmem[i].mr_size = OFmem64[i].mr_size;
-
- /*
- * Check for memory regions extending above 32-bit
- * memory space, and restrict them to stay there.
- */
- if (((uint64_t)OFmem[i].mr_start +
- (uint64_t)OFmem[i].mr_size) >
- BUS_SPACE_MAXADDR_32BIT) {
- OFmem[i].mr_size = BUS_SPACE_MAXADDR_32BIT -
- OFmem[i].mr_start;
+ #endif
+
+ j++;
+ }
+ sz = j*sizeof(output[0]);
+
+ #ifdef __powerpc64__
+ if (apple_hack_mode) {
+ /* Add in regions above 4 GB to the available list */
+ struct mem_region himem[OFMEM_REGIONS];
+ int hisz;
+
+ hisz = parse_ofw_memory(node, "reg", himem);
+ for (i = 0; i < hisz/sizeof(himem[0]); i++) {
+ if (himem[i].mr_start > BUS_SPACE_MAXADDR_32BIT) {
+ output[j].mr_start = himem[i].mr_start;
+ output[j].mr_size = himem[i].mr_size;
+ j++;
}
- j++;
}
- }
- msz = j*sizeof(OFmem[0]);
- } else {
- if ((msz = OF_getprop(phandle, "reg",
- OFmem, sizeof OFmem[0] * OFMEM_REGIONS)) <= 0)
- panic("Physical memory map not found");
+ sz = j*sizeof(output[0]);
}
+ #endif
+
+ return (sz);
+}
+
+/*
+ * This is called during powerpc_init, before the system is really initialized.
+ * It shall provide the total and the available regions of RAM.
+ * Both lists must have a zero-size entry as terminator.
+ * The available regions need not take the kernel into account, but needs
+ * to provide space for two additional entry beyond the terminating one.
+ */
+void
+ofw_mem_regions(struct mem_region **memp, int *memsz,
+ struct mem_region **availp, int *availsz)
+{
+ phandle_t phandle;
+ int asz, msz, fsz;
+ int i, j;
+ int still_merging;
+
+ asz = msz = 0;
+
+ /*
+ * Get memory.
+ */
+ phandle = OF_finddevice("/memory");
+ if (phandle == -1)
+ phandle = OF_finddevice("/memory@0");
+
+ msz = parse_ofw_memory(phandle, "reg", OFmem);
+ nOFmem = msz / sizeof(struct mem_region);
+ asz = parse_ofw_memory(phandle, "available", OFavail);
*memp = OFmem;
- *memsz = msz / sizeof(struct mem_region);
+ *memsz = nOFmem;
-
/*
* OFavail may have overlapping regions - collapse these
* and copy out remaining regions to OFfree
@@ -274,7 +319,19 @@ OF_initial_setup(void *fdt_ptr, void *junk, int (*openfirm)(void *))
else
ofw_real_mode = 1;
- ofwcall = openfirm;
+ ofwcall = NULL;
+
+ #ifdef __powerpc64__
+ /*
+ * For PPC64, we need to use some hand-written
+ * asm trampolines to get to OF.
+ */
+ if (openfirm != NULL)
+ ofwcall = ofw_32bit_mode_entry;
+ #else
+ ofwcall = openfirm;
+ #endif
+
fdt = fdt_ptr;
}
@@ -284,10 +341,15 @@ OF_bootstrap()
boolean_t status = FALSE;
if (ofwcall != NULL) {
- if (ofw_real_mode)
+ if (ofw_real_mode) {
status = OF_install(OFW_STD_REAL, 0);
- else
+ } else {
+ #ifdef __powerpc64__
+ status = OF_install(OFW_STD_32BIT, 0);
+ #else
status = OF_install(OFW_STD_DIRECT, 0);
+ #endif
+ }
if (status != TRUE)
return status;
@@ -347,26 +409,28 @@ ofw_quiesce(void)
static int
openfirmware_core(void *args)
{
- long oldmsr;
- int result;
- u_int srsave[16];
- u_int i;
-
- __asm __volatile( "\t"
- "sync\n\t"
- "mfmsr %0\n\t"
- "mtmsr %1\n\t"
- "isync\n"
- : "=r" (oldmsr)
- : "r" (ofmsr[0])
- );
+ int result;
+ register_t oldmsr;
+ #ifndef __powerpc64__
+ register_t srsave[16];
+ u_int i;
+ #endif
+
+ /*
+ * Turn off exceptions - we really don't want to end up
+ * anywhere unexpected with PCPU set to something strange,
+ * the stack pointer wrong, or the OFW mapping enabled.
+ */
+ oldmsr = intr_disable();
ofw_sprg_prepare();
+ #ifndef __powerpc64__
if (pmap_bootstrapped && !ofw_real_mode) {
/*
* Swap the kernel's address space with Open Firmware's
*/
+
for (i = 0; i < 16; i++) {
srsave[i] = mfsrin(i << ADDR_SR_SHFT);
mtsrin(i << ADDR_SR_SHFT, ofw_pmap.pm_sr[i]);
@@ -381,28 +445,28 @@ openfirmware_core(void *args)
}
isync();
}
+ #endif
result = ofwcall(args);
+ #ifndef __powerpc64__
if (pmap_bootstrapped && !ofw_real_mode) {
/*
* Restore the kernel's addr space. The isync() doesn;t
* work outside the loop unless mtsrin() is open-coded
* in an asm statement :(
*/
+
for (i = 0; i < 16; i++) {
mtsrin(i << ADDR_SR_SHFT, srsave[i]);
isync();
}
}
+ #endif
ofw_sprg_restore();
- __asm( "\t"
- "mtmsr %0\n\t"
- "isync\n"
- : : "r" (oldmsr)
- );
+ intr_restore(oldmsr);
return (result);
}
@@ -626,7 +690,7 @@ mem_valid(vm_offset_t addr, int len)
{
int i;
- for (i = 0; i < OFMEM_REGIONS; i++)
+ for (i = 0; i < nOFmem; i++)
if ((addr >= OFmem[i].mr_start)
&& (addr + len < OFmem[i].mr_start + OFmem[i].mr_size))
return (0);
diff --git a/sys/powerpc/aim/slb.c b/sys/powerpc/aim/slb.c
new file mode 100644
index 0000000..0990f94
--- /dev/null
+++ b/sys/powerpc/aim/slb.c
@@ -0,0 +1,303 @@
+/*-
+ * Copyright (c) 2010 Nathan Whitehorn
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ *
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
+ * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
+ * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+ * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+ * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+ * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
+ * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ *
+ * $FreeBSD$
+ */
+
+#include <sys/param.h>
+#include <sys/kernel.h>
+#include <sys/lock.h>
+#include <sys/mutex.h>
+#include <sys/proc.h>
+#include <sys/systm.h>
+#include <sys/tree.h>
+
+#include <vm/vm.h>
+#include <vm/pmap.h>
+#include <vm/uma.h>
+#include <vm/vm_map.h>
+
+#include <machine/md_var.h>
+#include <machine/pmap.h>
+#include <machine/vmparam.h>
+
+uintptr_t moea64_get_unique_vsid(void);
+void moea64_release_vsid(uint64_t vsid);
+
+struct slbcontainer {
+ struct slb slb;
+ SPLAY_ENTRY(slbcontainer) slb_node;
+};
+
+static int slb_compare(struct slbcontainer *a, struct slbcontainer *b);
+static void slb_zone_init(void *);
+
+SPLAY_PROTOTYPE(slb_tree, slbcontainer, slb_node, slb_compare);
+SPLAY_GENERATE(slb_tree, slbcontainer, slb_node, slb_compare);
+
+uma_zone_t slb_zone;
+uma_zone_t slb_cache_zone;
+
+SYSINIT(slb_zone_init, SI_SUB_KMEM, SI_ORDER_ANY, slb_zone_init, NULL);
+
+int
+va_to_slb_entry(pmap_t pm, vm_offset_t va, struct slb *slb)
+{
+ struct slbcontainer cont, *found;
+ uint64_t esid;
+
+ esid = (uintptr_t)va >> ADDR_SR_SHFT;
+ slb->slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
+
+ if (pm == kernel_pmap) {
+ /* Set kernel VSID to deterministic value */
+ slb->slbv = va_to_vsid(kernel_pmap, va) << SLBV_VSID_SHIFT;
+
+ /* Figure out if this is a large-page mapping */
+ if (hw_direct_map && va < VM_MIN_KERNEL_ADDRESS) {
+ /*
+ * XXX: If we have set up a direct map, assumes
+ * all physical memory is mapped with large pages.
+ */
+ if (mem_valid(va, 0) == 0)
+ slb->slbv |= SLBV_L;
+ }
+
+ return (0);
+ }
+
+ PMAP_LOCK_ASSERT(pm, MA_OWNED);
+
+ cont.slb.slbe = slb->slbe;
+ found = SPLAY_FIND(slb_tree, &pm->pm_slbtree, &cont);
+
+ if (found == NULL)
+ return (-1);
+
+ slb->slbv = found->slb.slbv;
+ return (0);
+}
+
+uint64_t
+va_to_vsid(pmap_t pm, vm_offset_t va)
+{
+ struct slb entry;
+ int large;
+
+ /* Shortcut kernel case */
+ if (pm == kernel_pmap) {
+ large = 0;
+ if (hw_direct_map && va < VM_MIN_KERNEL_ADDRESS &&
+ mem_valid(va, 0) == 0)
+ large = 1;
+
+ return (KERNEL_VSID((uintptr_t)va >> ADDR_SR_SHFT, large));
+ }
+
+ /*
+ * If there is no vsid for this VA, we need to add a new entry
+ * to the PMAP's segment table.
+ */
+
+ if (va_to_slb_entry(pm, va, &entry) != 0)
+ return (allocate_vsid(pm, (uintptr_t)va >> ADDR_SR_SHFT, 0));
+
+ return ((entry.slbv & SLBV_VSID_MASK) >> SLBV_VSID_SHIFT);
+}
+
+uint64_t
+allocate_vsid(pmap_t pm, uint64_t esid, int large)
+{
+ uint64_t vsid;
+ struct slbcontainer *slb_entry, kern_entry;
+ struct slb *prespill;
+
+ prespill = NULL;
+
+ if (pm == kernel_pmap) {
+ vsid = va_to_vsid(pm, esid << ADDR_SR_SHFT);
+ slb_entry = &kern_entry;
+ prespill = PCPU_GET(slb);
+ } else {
+ vsid = moea64_get_unique_vsid();
+ slb_entry = uma_zalloc(slb_zone, M_NOWAIT);
+
+ if (slb_entry == NULL)
+ panic("Could not allocate SLB mapping!");
+
+ prespill = pm->pm_slb;
+ }
+
+ slb_entry->slb.slbe = (esid << SLBE_ESID_SHIFT) | SLBE_VALID;
+ slb_entry->slb.slbv = vsid << SLBV_VSID_SHIFT;
+
+ if (large)
+ slb_entry->slb.slbv |= SLBV_L;
+
+ if (pm != kernel_pmap) {
+ PMAP_LOCK_ASSERT(pm, MA_OWNED);
+ SPLAY_INSERT(slb_tree, &pm->pm_slbtree, slb_entry);
+ }
+
+ /*
+ * Someone probably wants this soon, and it may be a wired
+ * SLB mapping, so pre-spill this entry.
+ */
+ if (prespill != NULL)
+ slb_insert(pm, prespill, &slb_entry->slb);
+
+ return (vsid);
+}
+
+/* Lock entries mapping kernel text and stacks */
+
+#define SLB_SPILLABLE(slbe) \
+ (((slbe & SLBE_ESID_MASK) < VM_MIN_KERNEL_ADDRESS && \
+ (slbe & SLBE_ESID_MASK) > 16*SEGMENT_LENGTH) || \
+ (slbe & SLBE_ESID_MASK) > VM_MAX_KERNEL_ADDRESS)
+void
+slb_insert(pmap_t pm, struct slb *slbcache, struct slb *slb_entry)
+{
+ uint64_t slbe, slbv;
+ int i, j, to_spill;
+
+ /* We don't want to be preempted while modifying the kernel map */
+ critical_enter();
+
+ to_spill = -1;
+ slbv = slb_entry->slbv;
+ slbe = slb_entry->slbe;
+
+ /* Hunt for a likely candidate */
+
+ for (i = mftb() % 64, j = 0; j < 64; j++, i = (i+1) % 64) {
+ if (pm == kernel_pmap && i == USER_SR)
+ continue;
+
+ if (!(slbcache[i].slbe & SLBE_VALID)) {
+ to_spill = i;
+ break;
+ }
+
+ if (to_spill < 0 && (pm != kernel_pmap ||
+ SLB_SPILLABLE(slbcache[i].slbe)))
+ to_spill = i;
+ }
+
+ if (to_spill < 0)
+ panic("SLB spill on ESID %#lx, but no available candidates!\n",
+ (slbe & SLBE_ESID_MASK) >> SLBE_ESID_SHIFT);
+
+ if (slbcache[to_spill].slbe & SLBE_VALID) {
+ /* Invalidate this first to avoid races */
+ slbcache[to_spill].slbe = 0;
+ mb();
+ }
+ slbcache[to_spill].slbv = slbv;
+ slbcache[to_spill].slbe = slbe | (uint64_t)to_spill;
+
+ /* If it is for this CPU, put it in the SLB right away */
+ if (pm == kernel_pmap && pmap_bootstrapped) {
+ /* slbie not required */
+ __asm __volatile ("slbmte %0, %1" ::
+ "r"(slbcache[to_spill].slbv),
+ "r"(slbcache[to_spill].slbe));
+ }
+
+ critical_exit();
+}
+
+int
+vsid_to_esid(pmap_t pm, uint64_t vsid, uint64_t *esid)
+{
+ uint64_t slbv;
+ struct slbcontainer *entry;
+
+#ifdef INVARIANTS
+ if (pm == kernel_pmap)
+ panic("vsid_to_esid only works on user pmaps");
+
+ PMAP_LOCK_ASSERT(pm, MA_OWNED);
+#endif
+
+ slbv = vsid << SLBV_VSID_SHIFT;
+
+ SPLAY_FOREACH(entry, slb_tree, &pm->pm_slbtree) {
+ if (slbv == entry->slb.slbv) {
+ *esid = entry->slb.slbe >> SLBE_ESID_SHIFT;
+ return (0);
+ }
+ }
+
+ return (-1);
+}
+
+void
+free_vsids(pmap_t pm)
+{
+ struct slbcontainer *entry;
+
+ while (!SPLAY_EMPTY(&pm->pm_slbtree)) {
+ entry = SPLAY_MIN(slb_tree, &pm->pm_slbtree);
+
+ SPLAY_REMOVE(slb_tree, &pm->pm_slbtree, entry);
+
+ moea64_release_vsid(entry->slb.slbv >> SLBV_VSID_SHIFT);
+ uma_zfree(slb_zone, entry);
+ }
+}
+
+static int
+slb_compare(struct slbcontainer *a, struct slbcontainer *b)
+{
+ if (a->slb.slbe == b->slb.slbe)
+ return (0);
+ else if (a->slb.slbe < b->slb.slbe)
+ return (-1);
+ else
+ return (1);
+}
+
+static void
+slb_zone_init(void *dummy)
+{
+
+ slb_zone = uma_zcreate("SLB segment", sizeof(struct slbcontainer),
+ NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
+ slb_cache_zone = uma_zcreate("SLB cache", 64*sizeof(struct slb),
+ NULL, NULL, NULL, NULL, UMA_ALIGN_PTR, UMA_ZONE_VM);
+}
+
+struct slb *
+slb_alloc_user_cache(void)
+{
+ return (uma_zalloc(slb_cache_zone, M_ZERO));
+}
+
+void
+slb_free_user_cache(struct slb *slb)
+{
+ uma_zfree(slb_cache_zone, slb);
+}
diff --git a/sys/powerpc/aim/swtch.S b/sys/powerpc/aim/swtch32.S
index 0797057..f5dba2a 100644
--- a/sys/powerpc/aim/swtch.S
+++ b/sys/powerpc/aim/swtch32.S
@@ -63,8 +63,6 @@
#include <machine/trap.h>
#include <machine/param.h>
-#include <machine/sr.h>
-#include <machine/psl.h>
#include <machine/asm.h>
/*
@@ -83,7 +81,6 @@ ENTRY(cpu_throw)
*/
ENTRY(cpu_switch)
lwz %r6,TD_PCB(%r3) /* Get the old thread's PCB ptr */
- mr %r12,%r2
stmw %r12,PCB_CONTEXT(%r6) /* Save the non-volatile GP regs.
These can now be used for scratch */
@@ -93,8 +90,9 @@ ENTRY(cpu_switch)
stw %r16,PCB_LR(%r6)
mfsr %r16,USER_SR /* Save USER_SR for copyin/out */
isync
- stw %r16,PCB_AIM_USR(%r6)
+ stw %r16,PCB_AIM_USR_VSID(%r6)
stw %r1,PCB_SP(%r6) /* Save the stack pointer */
+ stw %r2,PCB_TOC(%r6) /* Save the TOC pointer */
mr %r14,%r3 /* Copy the old thread ptr... */
mr %r15,%r4 /* and the new thread ptr in scratch */
@@ -159,15 +157,15 @@ blocked_loop:
.L4:
mr %r3,%r17 /* Recover PCB ptr */
lmw %r12,PCB_CONTEXT(%r3) /* Load the non-volatile GP regs */
- mr %r2,%r12
lwz %r5,PCB_CR(%r3) /* Load the condition register */
mtcr %r5
lwz %r5,PCB_LR(%r3) /* Load the link register */
mtlr %r5
- lwz %r5,PCB_AIM_USR(%r3) /* Load the USER_SR segment reg */
+ lwz %r5,PCB_AIM_USR_VSID(%r3) /* Load the USER_SR segment reg */
mtsr USER_SR,%r5
isync
lwz %r1,PCB_SP(%r3) /* Load the stack pointer */
+ lwz %r2,PCB_TOC(%r3) /* Load the TOC pointer */
/*
* Perform a dummy stwcx. to clear any reservations we may have
* inherited from the previous thread. It doesn't matter if the
@@ -181,7 +179,6 @@ blocked_loop:
* Update pcb, saving current processor state
*/
ENTRY(savectx)
- mr %r12,%r2
stmw %r12,PCB_CONTEXT(%r3) /* Save the non-volatile GP regs */
mfcr %r4 /* Save the condition register */
stw %r4,PCB_CR(%r3)
diff --git a/sys/powerpc/aim/swtch64.S b/sys/powerpc/aim/swtch64.S
new file mode 100644
index 0000000..6ba9843
--- /dev/null
+++ b/sys/powerpc/aim/swtch64.S
@@ -0,0 +1,291 @@
+/* $FreeBSD$ */
+/* $NetBSD: locore.S,v 1.24 2000/05/31 05:09:17 thorpej Exp $ */
+
+/*-
+ * Copyright (C) 2001 Benno Rice
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ *
+ * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+*/
+/*-
+ * Copyright (C) 1995, 1996 Wolfgang Solfrank.
+ * Copyright (C) 1995, 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+#include "assym.s"
+#include "opt_sched.h"
+
+#include <sys/syscall.h>
+
+#include <machine/trap.h>
+#include <machine/param.h>
+#include <machine/asm.h>
+
+/*
+ * void cpu_throw(struct thread *old, struct thread *new)
+ */
+ENTRY(cpu_throw)
+ mr %r15, %r4
+ b cpu_switchin
+
+/*
+ * void cpu_switch(struct thread *old,
+ * struct thread *new,
+ * struct mutex *mtx);
+ *
+ * Switch to a new thread saving the current state in the old thread.
+ */
+ENTRY(cpu_switch)
+ ld %r6,TD_PCB(%r3) /* Get the old thread's PCB ptr */
+ std %r12,PCB_CONTEXT(%r6) /* Save the non-volatile GP regs.
+ These can now be used for scratch */
+ std %r13,PCB_CONTEXT+1*8(%r6)
+ std %r14,PCB_CONTEXT+2*8(%r6)
+ std %r15,PCB_CONTEXT+3*8(%r6)
+ std %r16,PCB_CONTEXT+4*8(%r6)
+ std %r17,PCB_CONTEXT+5*8(%r6)
+ std %r18,PCB_CONTEXT+6*8(%r6)
+ std %r19,PCB_CONTEXT+7*8(%r6)
+ std %r20,PCB_CONTEXT+8*8(%r6)
+ std %r21,PCB_CONTEXT+9*8(%r6)
+ std %r22,PCB_CONTEXT+10*8(%r6)
+ std %r23,PCB_CONTEXT+11*8(%r6)
+ std %r24,PCB_CONTEXT+12*8(%r6)
+ std %r25,PCB_CONTEXT+13*8(%r6)
+ std %r26,PCB_CONTEXT+14*8(%r6)
+ std %r27,PCB_CONTEXT+15*8(%r6)
+ std %r28,PCB_CONTEXT+16*8(%r6)
+ std %r29,PCB_CONTEXT+17*8(%r6)
+ std %r30,PCB_CONTEXT+18*8(%r6)
+ std %r31,PCB_CONTEXT+19*8(%r6)
+
+ mfcr %r16 /* Save the condition register */
+ std %r16,PCB_CR(%r6)
+ mflr %r16 /* Save the link register */
+ std %r16,PCB_LR(%r6)
+ std %r1,PCB_SP(%r6) /* Save the stack pointer */
+ std %r2,PCB_TOC(%r6) /* Save the TOC pointer */
+
+ li %r14,0 /* Save USER_SR for copyin/out */
+ li %r15,0
+ li %r16,USER_SR
+ slbmfee %r14, %r16
+ slbmfev %r15, %r16
+ isync
+ std %r14,PCB_AIM_USR_ESID(%r6)
+ std %r15,PCB_AIM_USR_VSID(%r6)
+
+ mr %r14,%r3 /* Copy the old thread ptr... */
+ mr %r15,%r4 /* and the new thread ptr in scratch */
+ mr %r16,%r5 /* and the new lock */
+ mr %r17,%r6 /* and the PCB */
+
+ stdu %r1,-48(%r1)
+
+ lwz %r7,PCB_FLAGS(%r17)
+ /* Save FPU context if needed */
+ andi. %r7, %r7, PCB_FPU
+ beq .L1
+ bl .save_fpu
+ nop
+
+.L1:
+ mr %r3,%r14 /* restore old thread ptr */
+ lwz %r7,PCB_FLAGS(%r17)
+ /* Save Altivec context if needed */
+ andi. %r7, %r7, PCB_VEC
+ beq .L2
+ bl .save_vec
+ nop
+
+.L2:
+ mr %r3,%r14 /* restore old thread ptr */
+ bl .pmap_deactivate /* Deactivate the current pmap */
+ nop
+
+ addi %r1,%r1,48
+
+ std %r16,TD_LOCK(%r14) /* ULE: update old thread's lock */
+
+cpu_switchin:
+#if defined(SMP) && defined(SCHED_ULE)
+ /* Wait for the new thread to become unblocked */
+ lis %r6,blocked_lock@ha
+ addi %r6,%r6,blocked_lock@l
+blocked_loop:
+ ld %r7,TD_LOCK(%r15)
+ cmpd %r6,%r7
+ beq blocked_loop
+#endif
+
+ mfsprg %r7,0 /* Get the pcpu pointer */
+ std %r15,PC_CURTHREAD(%r7) /* Store new current thread */
+ ld %r17,TD_PCB(%r15) /* Store new current PCB */
+ std %r17,PC_CURPCB(%r7)
+
+ stdu %r1,-48(%r1)
+
+ mr %r3,%r15 /* Get new thread ptr */
+ bl .pmap_activate /* Activate the new address space */
+ nop
+
+ lwz %r6, PCB_FLAGS(%r17)
+ /* Restore FPU context if needed */
+ andi. %r6, %r6, PCB_FPU
+ beq .L3
+ mr %r3,%r15 /* Pass curthread to enable_fpu */
+ bl .enable_fpu
+ nop
+
+.L3:
+ lwz %r6, PCB_FLAGS(%r17)
+ /* Restore Altivec context if needed */
+ andi. %r6, %r6, PCB_VEC
+ beq .L4
+ mr %r3,%r15 /* Pass curthread to enable_vec */
+ bl .enable_vec
+ nop
+
+ /* thread to restore is in r3 */
+.L4:
+ addi %r1,%r1,48
+ mr %r3,%r17 /* Recover PCB ptr */
+ ld %r12,PCB_CONTEXT(%r3) /* Load the non-volatile GP regs. */
+ ld %r13,PCB_CONTEXT+1*8(%r3)
+ ld %r14,PCB_CONTEXT+2*8(%r3)
+ ld %r15,PCB_CONTEXT+3*8(%r3)
+ ld %r16,PCB_CONTEXT+4*8(%r3)
+ ld %r17,PCB_CONTEXT+5*8(%r3)
+ ld %r18,PCB_CONTEXT+6*8(%r3)
+ ld %r19,PCB_CONTEXT+7*8(%r3)
+ ld %r20,PCB_CONTEXT+8*8(%r3)
+ ld %r21,PCB_CONTEXT+9*8(%r3)
+ ld %r22,PCB_CONTEXT+10*8(%r3)
+ ld %r23,PCB_CONTEXT+11*8(%r3)
+ ld %r24,PCB_CONTEXT+12*8(%r3)
+ ld %r25,PCB_CONTEXT+13*8(%r3)
+ ld %r26,PCB_CONTEXT+14*8(%r3)
+ ld %r27,PCB_CONTEXT+15*8(%r3)
+ ld %r28,PCB_CONTEXT+16*8(%r3)
+ ld %r29,PCB_CONTEXT+17*8(%r3)
+ ld %r30,PCB_CONTEXT+18*8(%r3)
+ ld %r31,PCB_CONTEXT+19*8(%r3)
+ ld %r5,PCB_CR(%r3) /* Load the condition register */
+ mtcr %r5
+ ld %r5,PCB_LR(%r3) /* Load the link register */
+ mtlr %r5
+ ld %r1,PCB_SP(%r3) /* Load the stack pointer */
+ ld %r2,PCB_TOC(%r3) /* Load the TOC pointer */
+
+ lis %r5,USER_ADDR@highesta /* Load the USER_SR segment reg */
+ ori %r5,%r5,USER_ADDR@highera
+ sldi %r5,%r5,32
+ oris %r5,%r5,USER_ADDR@ha
+ slbie %r5
+ ld %r5,PCB_AIM_USR_VSID(%r3)
+ ld %r6,PCB_AIM_USR_ESID(%r3)
+ ori %r6,%r6,USER_SR
+ slbmte %r5,%r6
+
+ isync
+ /*
+ * Perform a dummy stdcx. to clear any reservations we may have
+ * inherited from the previous thread. It doesn't matter if the
+ * stdcx succeeds or not. pcb_context[0] can be clobbered.
+ */
+ stdcx. %r1, 0, %r3
+ blr
+
+/*
+ * savectx(pcb)
+ * Update pcb, saving current processor state
+ */
+ENTRY(savectx)
+ std %r12,PCB_CONTEXT(%r3) /* Save the non-volatile GP regs. */
+ std %r13,PCB_CONTEXT+1*8(%r3)
+ std %r14,PCB_CONTEXT+2*8(%r3)
+ std %r15,PCB_CONTEXT+3*8(%r3)
+ std %r16,PCB_CONTEXT+4*8(%r3)
+ std %r17,PCB_CONTEXT+5*8(%r3)
+ std %r18,PCB_CONTEXT+6*8(%r3)
+ std %r19,PCB_CONTEXT+7*8(%r3)
+ std %r20,PCB_CONTEXT+8*8(%r3)
+ std %r21,PCB_CONTEXT+9*8(%r3)
+ std %r22,PCB_CONTEXT+10*8(%r3)
+ std %r23,PCB_CONTEXT+11*8(%r3)
+ std %r24,PCB_CONTEXT+12*8(%r3)
+ std %r25,PCB_CONTEXT+13*8(%r3)
+ std %r26,PCB_CONTEXT+14*8(%r3)
+ std %r27,PCB_CONTEXT+15*8(%r3)
+ std %r28,PCB_CONTEXT+16*8(%r3)
+ std %r29,PCB_CONTEXT+17*8(%r3)
+ std %r30,PCB_CONTEXT+18*8(%r3)
+ std %r31,PCB_CONTEXT+19*8(%r3)
+
+ mfcr %r4 /* Save the condition register */
+ std %r4,PCB_CR(%r3)
+ std %r2,PCB_TOC(%r3) /* Save the TOC pointer */
+ blr
+
+/*
+ * fork_trampoline()
+ * Set up the return from cpu_fork()
+ */
+ENTRY(fork_trampoline)
+ ld %r3,CF_FUNC(%r1)
+ ld %r4,CF_ARG0(%r1)
+ ld %r5,CF_ARG1(%r1)
+
+ stdu %r1,-48(%r1)
+ bl .fork_exit
+ nop
+ addi %r1,%r1,48+CF_SIZE-FSP /* Allow 8 bytes in front of
+ trapframe to simulate FRAME_SETUP
+ does when allocating space for
+ a frame pointer/saved LR */
+ b trapexit
+ nop
diff --git a/sys/powerpc/aim/trap.c b/sys/powerpc/aim/trap.c
index 74e0f26..df5c971 100644
--- a/sys/powerpc/aim/trap.c
+++ b/sys/powerpc/aim/trap.c
@@ -65,6 +65,7 @@ __FBSDID("$FreeBSD$");
#include <vm/vm_map.h>
#include <vm/vm_page.h>
+#include <machine/_inttypes.h>
#include <machine/altivec.h>
#include <machine/cpu.h>
#include <machine/db_machdep.h>
@@ -86,6 +87,10 @@ static int ppc_instr_emulate(struct trapframe *frame);
static int handle_onfault(struct trapframe *frame);
static void syscall(struct trapframe *frame);
+#ifdef __powerpc64__
+static int handle_slb_spill(pmap_t pm, vm_offset_t addr);
+#endif
+
int setfault(faultbuf); /* defined in locore.S */
/* Why are these not defined in a header? */
@@ -101,7 +106,9 @@ static struct powerpc_exception powerpc_exceptions[] = {
{ 0x0100, "system reset" },
{ 0x0200, "machine check" },
{ 0x0300, "data storage interrupt" },
+ { 0x0380, "data segment exception" },
{ 0x0400, "instruction storage interrupt" },
+ { 0x0480, "instruction segment exception" },
{ 0x0500, "external interrupt" },
{ 0x0600, "alignment" },
{ 0x0700, "program" },
@@ -171,6 +178,15 @@ trap(struct trapframe *frame)
sig = SIGTRAP;
break;
+#ifdef __powerpc64__
+ case EXC_ISE:
+ case EXC_DSE:
+ if (handle_slb_spill(&p->p_vmspace->vm_pmap,
+ (type == EXC_ISE) ? frame->srr0 :
+ frame->cpu.aim.dar) != 0)
+ sig = SIGSEGV;
+ break;
+#endif
case EXC_DSI:
case EXC_ISI:
sig = trap_pfault(frame, 1);
@@ -227,6 +243,15 @@ trap(struct trapframe *frame)
if (trap_pfault(frame, 0) == 0)
return;
break;
+#ifdef __powerpc64__
+ case EXC_ISE:
+ case EXC_DSE:
+ if (handle_slb_spill(kernel_pmap,
+ (type == EXC_ISE) ? frame->srr0 :
+ frame->cpu.aim.dar) != 0)
+ panic("Fault handling kernel SLB miss");
+ return;
+#endif
case EXC_MCHK:
if (handle_onfault(frame))
return;
@@ -276,16 +301,19 @@ printtrap(u_int vector, struct trapframe *frame, int isfatal, int user)
printf(" exception = 0x%x (%s)\n", vector >> 8,
trapname(vector));
switch (vector) {
+ case EXC_DSE:
case EXC_DSI:
- printf(" virtual address = 0x%x\n", frame->cpu.aim.dar);
+ printf(" virtual address = 0x%" PRIxPTR "\n",
+ frame->cpu.aim.dar);
break;
+ case EXC_ISE:
case EXC_ISI:
- printf(" virtual address = 0x%x\n", frame->srr0);
+ printf(" virtual address = 0x%" PRIxPTR "\n", frame->srr0);
break;
}
- printf(" srr0 = 0x%x\n", frame->srr0);
- printf(" srr1 = 0x%x\n", frame->srr1);
- printf(" lr = 0x%x\n", frame->lr);
+ printf(" srr0 = 0x%" PRIxPTR "\n", frame->srr0);
+ printf(" srr1 = 0x%" PRIxPTR "\n", frame->srr1);
+ printf(" lr = 0x%" PRIxPTR "\n", frame->lr);
printf(" curthread = %p\n", curthread);
if (curthread != NULL)
printf(" pid = %d, comm = %s\n",
@@ -324,7 +352,8 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
struct proc *p;
struct trapframe *frame;
caddr_t params;
- int error, n;
+ size_t argsz;
+ int error, n, i;
p = td->td_proc;
frame = td->td_frame;
@@ -338,7 +367,7 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
* code is first argument,
* followed by actual args.
*/
- sa->code = *(u_int *) params;
+ sa->code = *(register_t *) params;
params += sizeof(register_t);
n -= 1;
} else if (sa->code == SYS___syscall) {
@@ -347,10 +376,16 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
* so as to maintain quad alignment
* for the rest of the args.
*/
- params += sizeof(register_t);
- sa->code = *(u_int *) params;
- params += sizeof(register_t);
- n -= 2;
+ if (p->p_sysent->sv_flags & SV_ILP32) {
+ params += sizeof(register_t);
+ sa->code = *(register_t *) params;
+ params += sizeof(register_t);
+ n -= 2;
+ } else {
+ sa->code = *(register_t *) params;
+ params += sizeof(register_t);
+ n -= 1;
+ }
}
if (p->p_sysent->sv_mask)
@@ -362,13 +397,34 @@ cpu_fetch_syscall_args(struct thread *td, struct syscall_args *sa)
sa->narg = sa->callp->sy_narg;
- bcopy(params, sa->args, n * sizeof(register_t));
- if (sa->narg > n) {
+ if (p->p_sysent->sv_flags & SV_ILP32) {
+ argsz = sizeof(uint32_t);
+
+ for (i = 0; i < n; i++)
+ sa->args[i] = ((u_register_t *)(params))[i] &
+ 0xffffffff;
+ } else {
+ argsz = sizeof(uint64_t);
+
+ for (i = 0; i < n; i++)
+ sa->args[i] = ((u_register_t *)(params))[i];
+ }
+
+ if (sa->narg > n)
error = copyin(MOREARGS(frame->fixreg[1]), sa->args + n,
- (sa->narg - n) * sizeof(register_t));
- } else
+ (sa->narg - n) * argsz);
+ else
error = 0;
+#ifdef __powerpc64__
+ if (p->p_sysent->sv_flags & SV_ILP32 && sa->narg > n) {
+ /* Expand the size of arguments copied from the stack */
+
+ for (i = sa->narg; i >= n; i--)
+ sa->args[i] = ((uint32_t *)(&sa->args[n]))[i-n];
+ }
+#endif
+
if (error == 0) {
td->td_retval[0] = 0;
td->td_retval[1] = frame->fixreg[FIRSTARG + 1];
@@ -390,6 +446,44 @@ syscall(struct trapframe *frame)
syscallret(td, error, &sa);
}
+#ifdef __powerpc64__
+static int
+handle_slb_spill(pmap_t pm, vm_offset_t addr)
+{
+ struct slb slb_entry;
+ int error, i;
+
+ if (pm == kernel_pmap) {
+ error = va_to_slb_entry(pm, addr, &slb_entry);
+ if (error)
+ return (error);
+
+ slb_insert(pm, PCPU_GET(slb), &slb_entry);
+ return (0);
+ }
+
+ PMAP_LOCK(pm);
+ error = va_to_slb_entry(pm, addr, &slb_entry);
+ if (error != 0)
+ (void)allocate_vsid(pm, (uintptr_t)addr >> ADDR_SR_SHFT, 0);
+ else {
+ /*
+ * Check that another CPU has not already mapped this.
+ * XXX: Per-thread SLB caches would be better.
+ */
+ for (i = 0; i < 64; i++)
+ if (pm->pm_slb[i].slbe == (slb_entry.slbe | i))
+ break;
+
+ if (i == 64)
+ slb_insert(pm, pm->pm_slb, &slb_entry);
+ }
+ PMAP_UNLOCK(pm);
+
+ return (0);
+}
+#endif
+
static int
trap_pfault(struct trapframe *frame, int user)
{
@@ -399,7 +493,7 @@ trap_pfault(struct trapframe *frame, int user)
vm_map_t map;
vm_prot_t ftype;
int rv;
- u_int user_sr;
+ register_t user_sr;
td = curthread;
p = td->td_proc;
@@ -417,16 +511,33 @@ trap_pfault(struct trapframe *frame, int user)
if (user) {
map = &p->p_vmspace->vm_map;
} else {
- if ((eva >> ADDR_SR_SHFT) == USER_SR) {
+ if ((eva >> ADDR_SR_SHFT) == (USER_ADDR >> ADDR_SR_SHFT)) {
if (p->p_vmspace == NULL)
return (SIGSEGV);
+ map = &p->p_vmspace->vm_map;
+
+ #ifdef __powerpc64__
+ user_sr = 0;
+ __asm ("slbmfev %0, %1"
+ : "=r"(user_sr)
+ : "r"(USER_SR));
+
+ PMAP_LOCK(&p->p_vmspace->vm_pmap);
+ user_sr >>= SLBV_VSID_SHIFT;
+ rv = vsid_to_esid(&p->p_vmspace->vm_pmap, user_sr,
+ &user_sr);
+ PMAP_UNLOCK(&p->p_vmspace->vm_pmap);
+
+ if (rv != 0)
+ return (SIGSEGV);
+ #else
__asm ("mfsr %0, %1"
: "=r"(user_sr)
: "K"(USER_SR));
+ #endif
eva &= ADDR_PIDX | ADDR_POFF;
eva |= user_sr << ADDR_SR_SHFT;
- map = &p->p_vmspace->vm_map;
} else {
map = kernel_map;
}
@@ -502,7 +613,7 @@ badaddr_read(void *addr, size_t size, int *rptr)
x = *(volatile int32_t *)addr;
break;
default:
- panic("badaddr: invalid size (%d)", size);
+ panic("badaddr: invalid size (%zd)", size);
}
/* Make sure we took the machine check, if we caused one. */
diff --git a/sys/powerpc/aim/trap_subr32.S b/sys/powerpc/aim/trap_subr32.S
new file mode 100644
index 0000000..6482553
--- /dev/null
+++ b/sys/powerpc/aim/trap_subr32.S
@@ -0,0 +1,678 @@
+/* $FreeBSD$ */
+/* $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $ */
+
+/*-
+ * Copyright (C) 1995, 1996 Wolfgang Solfrank.
+ * Copyright (C) 1995, 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * NOTICE: This is not a standalone file. to use it, #include it in
+ * your port's locore.S, like so:
+ *
+ * #include <powerpc/aim/trap_subr.S>
+ */
+
+/*
+ * Save/restore segment registers
+ */
+#define RESTORE_SRS(pmap,sr) mtsr 0,sr; \
+ lwz sr,1*4(pmap); mtsr 1,sr; \
+ lwz sr,2*4(pmap); mtsr 2,sr; \
+ lwz sr,3*4(pmap); mtsr 3,sr; \
+ lwz sr,4*4(pmap); mtsr 4,sr; \
+ lwz sr,5*4(pmap); mtsr 5,sr; \
+ lwz sr,6*4(pmap); mtsr 6,sr; \
+ lwz sr,7*4(pmap); mtsr 7,sr; \
+ lwz sr,8*4(pmap); mtsr 8,sr; \
+ lwz sr,9*4(pmap); mtsr 9,sr; \
+ lwz sr,10*4(pmap); mtsr 10,sr; \
+ lwz sr,11*4(pmap); mtsr 11,sr; \
+ lwz sr,12*4(pmap); mtsr 12,sr; \
+ lwz sr,13*4(pmap); mtsr 13,sr; \
+ lwz sr,14*4(pmap); mtsr 14,sr; \
+ lwz sr,15*4(pmap); mtsr 15,sr; isync;
+
+/*
+ * User SRs are loaded through a pointer to the current pmap.
+ */
+#define RESTORE_USER_SRS(pmap,sr) \
+ GET_CPUINFO(pmap); \
+ lwz pmap,PC_CURPMAP(pmap); \
+ lwzu sr,PM_SR(pmap); \
+ RESTORE_SRS(pmap,sr)
+
+/*
+ * Kernel SRs are loaded directly from kernel_pmap_
+ */
+#define RESTORE_KERN_SRS(pmap,sr) \
+ lis pmap,CNAME(kernel_pmap_store)@ha; \
+ lwzu sr,CNAME(kernel_pmap_store)+PM_SR@l(pmap); \
+ RESTORE_SRS(pmap,sr)
+
+/*
+ * FRAME_SETUP assumes:
+ * SPRG1 SP (1)
+ * SPRG3 trap type
+ * savearea r28-r31,DAR,DSISR (DAR & DSISR only for DSI traps)
+ * r28 LR
+ * r29 CR
+ * r30 scratch
+ * r31 scratch
+ * r1 kernel stack
+ * SRR0/1 as at start of trap
+ */
+#define FRAME_SETUP(savearea) \
+/* Have to enable translation to allow access of kernel stack: */ \
+ GET_CPUINFO(%r31); \
+ mfsrr0 %r30; \
+ stw %r30,(savearea+CPUSAVE_SRR0)(%r31); /* save SRR0 */ \
+ mfsrr1 %r30; \
+ stw %r30,(savearea+CPUSAVE_SRR1)(%r31); /* save SRR1 */ \
+ mfmsr %r30; \
+ ori %r30,%r30,(PSL_DR|PSL_IR|PSL_RI)@l; /* relocation on */ \
+ mtmsr %r30; /* stack can now be accessed */ \
+ isync; \
+ mfsprg1 %r31; /* get saved SP */ \
+ stwu %r31,-FRAMELEN(%r1); /* save it in the callframe */ \
+ stw %r0, FRAME_0+8(%r1); /* save r0 in the trapframe */ \
+ stw %r31,FRAME_1+8(%r1); /* save SP " " */ \
+ stw %r2, FRAME_2+8(%r1); /* save r2 " " */ \
+ stw %r28,FRAME_LR+8(%r1); /* save LR " " */ \
+ stw %r29,FRAME_CR+8(%r1); /* save CR " " */ \
+ GET_CPUINFO(%r2); \
+ lwz %r28,(savearea+CPUSAVE_R28)(%r2); /* get saved r28 */ \
+ lwz %r29,(savearea+CPUSAVE_R29)(%r2); /* get saved r29 */ \
+ lwz %r30,(savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \
+ lwz %r31,(savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \
+ stw %r3, FRAME_3+8(%r1); /* save r3-r31 */ \
+ stw %r4, FRAME_4+8(%r1); \
+ stw %r5, FRAME_5+8(%r1); \
+ stw %r6, FRAME_6+8(%r1); \
+ stw %r7, FRAME_7+8(%r1); \
+ stw %r8, FRAME_8+8(%r1); \
+ stw %r9, FRAME_9+8(%r1); \
+ stw %r10, FRAME_10+8(%r1); \
+ stw %r11, FRAME_11+8(%r1); \
+ stw %r12, FRAME_12+8(%r1); \
+ stw %r13, FRAME_13+8(%r1); \
+ stw %r14, FRAME_14+8(%r1); \
+ stw %r15, FRAME_15+8(%r1); \
+ stw %r16, FRAME_16+8(%r1); \
+ stw %r17, FRAME_17+8(%r1); \
+ stw %r18, FRAME_18+8(%r1); \
+ stw %r19, FRAME_19+8(%r1); \
+ stw %r20, FRAME_20+8(%r1); \
+ stw %r21, FRAME_21+8(%r1); \
+ stw %r22, FRAME_22+8(%r1); \
+ stw %r23, FRAME_23+8(%r1); \
+ stw %r24, FRAME_24+8(%r1); \
+ stw %r25, FRAME_25+8(%r1); \
+ stw %r26, FRAME_26+8(%r1); \
+ stw %r27, FRAME_27+8(%r1); \
+ stw %r28, FRAME_28+8(%r1); \
+ stw %r29, FRAME_29+8(%r1); \
+ stw %r30, FRAME_30+8(%r1); \
+ stw %r31, FRAME_31+8(%r1); \
+ lwz %r28,(savearea+CPUSAVE_AIM_DAR)(%r2); /* saved DAR */ \
+ lwz %r29,(savearea+CPUSAVE_AIM_DSISR)(%r2);/* saved DSISR */\
+ lwz %r30,(savearea+CPUSAVE_SRR0)(%r2); /* saved SRR0 */ \
+ lwz %r31,(savearea+CPUSAVE_SRR1)(%r2); /* saved SRR1 */ \
+ mfxer %r3; \
+ mfctr %r4; \
+ mfsprg3 %r5; \
+ stw %r3, FRAME_XER+8(1); /* save xer/ctr/exc */ \
+ stw %r4, FRAME_CTR+8(1); \
+ stw %r5, FRAME_EXC+8(1); \
+ stw %r28,FRAME_AIM_DAR+8(1); \
+ stw %r29,FRAME_AIM_DSISR+8(1); /* save dsisr/srr0/srr1 */ \
+ stw %r30,FRAME_SRR0+8(1); \
+ stw %r31,FRAME_SRR1+8(1)
+
+#define FRAME_LEAVE(savearea) \
+/* Now restore regs: */ \
+ lwz %r2,FRAME_SRR0+8(%r1); \
+ lwz %r3,FRAME_SRR1+8(%r1); \
+ lwz %r4,FRAME_CTR+8(%r1); \
+ lwz %r5,FRAME_XER+8(%r1); \
+ lwz %r6,FRAME_LR+8(%r1); \
+ GET_CPUINFO(%r7); \
+ stw %r2,(savearea+CPUSAVE_SRR0)(%r7); /* save SRR0 */ \
+ stw %r3,(savearea+CPUSAVE_SRR1)(%r7); /* save SRR1 */ \
+ lwz %r7,FRAME_CR+8(%r1); \
+ mtctr %r4; \
+ mtxer %r5; \
+ mtlr %r6; \
+ mtsprg1 %r7; /* save cr */ \
+ lwz %r31,FRAME_31+8(%r1); /* restore r0-31 */ \
+ lwz %r30,FRAME_30+8(%r1); \
+ lwz %r29,FRAME_29+8(%r1); \
+ lwz %r28,FRAME_28+8(%r1); \
+ lwz %r27,FRAME_27+8(%r1); \
+ lwz %r26,FRAME_26+8(%r1); \
+ lwz %r25,FRAME_25+8(%r1); \
+ lwz %r24,FRAME_24+8(%r1); \
+ lwz %r23,FRAME_23+8(%r1); \
+ lwz %r22,FRAME_22+8(%r1); \
+ lwz %r21,FRAME_21+8(%r1); \
+ lwz %r20,FRAME_20+8(%r1); \
+ lwz %r19,FRAME_19+8(%r1); \
+ lwz %r18,FRAME_18+8(%r1); \
+ lwz %r17,FRAME_17+8(%r1); \
+ lwz %r16,FRAME_16+8(%r1); \
+ lwz %r15,FRAME_15+8(%r1); \
+ lwz %r14,FRAME_14+8(%r1); \
+ lwz %r13,FRAME_13+8(%r1); \
+ lwz %r12,FRAME_12+8(%r1); \
+ lwz %r11,FRAME_11+8(%r1); \
+ lwz %r10,FRAME_10+8(%r1); \
+ lwz %r9, FRAME_9+8(%r1); \
+ lwz %r8, FRAME_8+8(%r1); \
+ lwz %r7, FRAME_7+8(%r1); \
+ lwz %r6, FRAME_6+8(%r1); \
+ lwz %r5, FRAME_5+8(%r1); \
+ lwz %r4, FRAME_4+8(%r1); \
+ lwz %r3, FRAME_3+8(%r1); \
+ lwz %r2, FRAME_2+8(%r1); \
+ lwz %r0, FRAME_0+8(%r1); \
+ lwz %r1, FRAME_1+8(%r1); \
+/* Can't touch %r1 from here on */ \
+ mtsprg2 %r2; /* save r2 & r3 */ \
+ mtsprg3 %r3; \
+/* Disable translation, machine check and recoverability: */ \
+ mfmsr %r2; \
+ andi. %r2,%r2,~(PSL_DR|PSL_IR|PSL_EE|PSL_ME|PSL_RI)@l; \
+ mtmsr %r2; \
+ isync; \
+/* Decide whether we return to user mode: */ \
+ GET_CPUINFO(%r2); \
+ lwz %r3,(savearea+CPUSAVE_SRR1)(%r2); \
+ mtcr %r3; \
+ bf 17,1f; /* branch if PSL_PR is false */ \
+/* Restore user SRs */ \
+ RESTORE_USER_SRS(%r2,%r3); \
+1: mfsprg1 %r2; /* restore cr */ \
+ mtcr %r2; \
+ GET_CPUINFO(%r2); \
+ lwz %r3,(savearea+CPUSAVE_SRR0)(%r2); /* restore srr0 */ \
+ mtsrr0 %r3; \
+ lwz %r3,(savearea+CPUSAVE_SRR1)(%r2); /* restore srr1 */ \
+ \
+ /* Make sure HV bit of MSR propagated to SRR1 */ \
+ mfmsr %r2; \
+ or %r3,%r2,%r3; \
+ \
+ mtsrr1 %r3; \
+ mfsprg2 %r2; /* restore r2 & r3 */ \
+ mfsprg3 %r3
+
+/*
+ * The next two routines are 64-bit glue code. The first is used to test if
+ * we are on a 64-bit system. By copying it to the illegal instruction
+ * handler, we can test for 64-bit mode by trying to execute a 64-bit
+ * instruction and seeing what happens. The second gets copied in front
+ * of all the other handlers to restore 32-bit bridge mode when traps
+ * are taken.
+ */
+
+/* 64-bit test code. Sets SPRG2 to 0 if an illegal instruction is executed */
+
+ .globl CNAME(testppc64),CNAME(testppc64size)
+CNAME(testppc64):
+ mtsprg1 %r31
+ mfsrr0 %r31
+ addi %r31, %r31, 4
+ mtsrr0 %r31
+
+ li %r31, 0
+ mtsprg2 %r31
+ mfsprg1 %r31
+
+ rfi
+CNAME(testppc64size) = .-CNAME(testppc64)
+
+
+/* 64-bit bridge mode restore snippet. Gets copied in front of everything else
+ * on 64-bit systems. */
+
+ .globl CNAME(restorebridge),CNAME(restorebridgesize)
+CNAME(restorebridge):
+ mtsprg1 %r31
+ mfmsr %r31
+ clrldi %r31,%r31,1
+ mtmsrd %r31
+ mfsprg1 %r31
+ isync
+CNAME(restorebridgesize) = .-CNAME(restorebridge)
+
+#ifdef SMP
+/*
+ * Processor reset exception handler. These are typically
+ * the first instructions the processor executes after a
+ * software reset. We do this in two bits so that we are
+ * not still hanging around in the trap handling region
+ * once the MMU is turned on.
+ */
+ .globl CNAME(rstcode), CNAME(rstsize)
+CNAME(rstcode):
+ ba cpu_reset
+CNAME(rstsize) = . - CNAME(rstcode)
+
+cpu_reset:
+ bl 1f
+
+ .space 124
+
+1:
+ mflr %r1
+ addi %r1,%r1,(124-16)@l
+
+ lis %r3,1@l
+ bla CNAME(cpudep_ap_early_bootstrap)
+ bla CNAME(pmap_cpu_bootstrap)
+ bla CNAME(cpudep_ap_bootstrap)
+ mr %r1,%r3
+ bla CNAME(machdep_ap_bootstrap)
+
+ /* Should not be reached */
+9:
+ b 9b
+#endif
+
+/*
+ * This code gets copied to all the trap vectors
+ * (except ISI/DSI, ALI, and the interrupts)
+ */
+
+ .globl CNAME(trapcode),CNAME(trapsize)
+CNAME(trapcode):
+ mtsprg1 %r1 /* save SP */
+ mflr %r1 /* Save the old LR in r1 */
+ mtsprg2 %r1 /* And then in SPRG2 */
+ li %r1, 0x20 /* How to get the vector from LR */
+ bla generictrap /* LR & SPRG3 is exception # */
+CNAME(trapsize) = .-CNAME(trapcode)
+
+/*
+ * 64-bit version of trapcode. Identical, except it calls generictrap64.
+ */
+ .globl CNAME(trapcode64)
+CNAME(trapcode64):
+ mtsprg1 %r1 /* save SP */
+ mflr %r1 /* Save the old LR in r1 */
+ mtsprg2 %r1 /* And then in SPRG2 */
+ li %r1, 0x20 /* How to get the vector from LR */
+ bla generictrap64 /* LR & SPRG3 is exception # */
+
+/*
+ * For ALI: has to save DSISR and DAR
+ */
+ .globl CNAME(alitrap),CNAME(alisize)
+CNAME(alitrap):
+ mtsprg1 %r1 /* save SP */
+ GET_CPUINFO(%r1)
+ stw %r28,(PC_TEMPSAVE+CPUSAVE_R28)(%r1) /* free r28-r31 */
+ stw %r29,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
+ stw %r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
+ stw %r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
+ mfdar %r30
+ mfdsisr %r31
+ stw %r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1)
+ stw %r31,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1)
+ mfsprg1 %r1 /* restore SP, in case of branch */
+ mflr %r28 /* save LR */
+ mfcr %r29 /* save CR */
+
+ /* Put our exception vector in SPRG3 */
+ li %r31, EXC_ALI
+ mtsprg3 %r31
+
+ /* Test whether we already had PR set */
+ mfsrr1 %r31
+ mtcr %r31
+ bla s_trap
+CNAME(alisize) = .-CNAME(alitrap)
+
+/*
+ * Similar to the above for DSI
+ * Has to handle BAT spills
+ * and standard pagetable spills
+ */
+ .globl CNAME(dsitrap),CNAME(dsisize)
+CNAME(dsitrap):
+ mtsprg1 %r1 /* save SP */
+ GET_CPUINFO(%r1)
+ stw %r28,(PC_DISISAVE+CPUSAVE_R28)(%r1) /* free r28-r31 */
+ stw %r29,(PC_DISISAVE+CPUSAVE_R29)(%r1)
+ stw %r30,(PC_DISISAVE+CPUSAVE_R30)(%r1)
+ stw %r31,(PC_DISISAVE+CPUSAVE_R31)(%r1)
+ mfsprg1 %r1 /* restore SP */
+ mfcr %r29 /* save CR */
+ mfxer %r30 /* save XER */
+ mtsprg2 %r30 /* in SPRG2 */
+ mfsrr1 %r31 /* test kernel mode */
+ mtcr %r31
+ bt 17,1f /* branch if PSL_PR is set */
+ mfdar %r31 /* get fault address */
+ rlwinm %r31,%r31,7,25,28 /* get segment * 8 */
+
+ /* get batu */
+ addis %r31,%r31,CNAME(battable)@ha
+ lwz %r30,CNAME(battable)@l(31)
+ mtcr %r30
+ bf 30,1f /* branch if supervisor valid is
+ false */
+ /* get batl */
+ lwz %r31,CNAME(battable)+4@l(31)
+/* We randomly use the highest two bat registers here */
+ mftb %r28
+ andi. %r28,%r28,1
+ bne 2f
+ mtdbatu 2,%r30
+ mtdbatl 2,%r31
+ b 3f
+2:
+ mtdbatu 3,%r30
+ mtdbatl 3,%r31
+3:
+ mfsprg2 %r30 /* restore XER */
+ mtxer %r30
+ mtcr %r29 /* restore CR */
+ mtsprg1 %r1
+ GET_CPUINFO(%r1)
+ lwz %r28,(PC_DISISAVE+CPUSAVE_R28)(%r1) /* restore r28-r31 */
+ lwz %r29,(PC_DISISAVE+CPUSAVE_R29)(%r1)
+ lwz %r30,(PC_DISISAVE+CPUSAVE_R30)(%r1)
+ lwz %r31,(PC_DISISAVE+CPUSAVE_R31)(%r1)
+ mfsprg1 %r1
+ rfi /* return to trapped code */
+1:
+ mflr %r28 /* save LR (SP already saved) */
+ bla disitrap
+CNAME(dsisize) = .-CNAME(dsitrap)
+
+/*
+ * Preamble code for DSI/ISI traps
+ */
+disitrap:
+ /* Write the trap vector to SPRG3 by computing LR & 0xff00 */
+ mflr %r1
+ andi. %r1,%r1,0xff00
+ mtsprg3 %r1
+
+ GET_CPUINFO(%r1)
+ lwz %r30,(PC_DISISAVE+CPUSAVE_R28)(%r1)
+ stw %r30,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
+ lwz %r31,(PC_DISISAVE+CPUSAVE_R29)(%r1)
+ stw %r31,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
+ lwz %r30,(PC_DISISAVE+CPUSAVE_R30)(%r1)
+ stw %r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
+ lwz %r31,(PC_DISISAVE+CPUSAVE_R31)(%r1)
+ stw %r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
+ mfdar %r30
+ mfdsisr %r31
+ stw %r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1)
+ stw %r31,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1)
+
+#ifdef KDB
+ /* Try and detect a kernel stack overflow */
+ mfsrr1 %r31
+ mtcr %r31
+ bt 17,realtrap /* branch is user mode */
+ mfsprg1 %r31 /* get old SP */
+ sub. %r30,%r31,%r30 /* SP - DAR */
+ bge 1f
+ neg %r30,%r30 /* modulo value */
+1: cmplwi %cr0,%r30,4096 /* is DAR within a page of SP? */
+ bge %cr0,realtrap /* no, too far away. */
+
+ /* Now convert this DSI into a DDB trap. */
+ GET_CPUINFO(%r1)
+ lwz %r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1) /* get DAR */
+ stw %r30,(PC_DBSAVE +CPUSAVE_AIM_DAR)(%r1) /* save DAR */
+ lwz %r30,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1) /* get DSISR */
+ stw %r30,(PC_DBSAVE +CPUSAVE_AIM_DSISR)(%r1) /* save DSISR */
+ lwz %r30,(PC_DISISAVE+CPUSAVE_R28)(%r1) /* get r28 */
+ stw %r30,(PC_DBSAVE +CPUSAVE_R28)(%r1) /* save r28 */
+ lwz %r31,(PC_DISISAVE+CPUSAVE_R29)(%r1) /* get r29 */
+ stw %r31,(PC_DBSAVE +CPUSAVE_R29)(%r1) /* save r29 */
+ lwz %r30,(PC_DISISAVE+CPUSAVE_R30)(%r1) /* get r30 */
+ stw %r30,(PC_DBSAVE +CPUSAVE_R30)(%r1) /* save r30 */
+ lwz %r31,(PC_DISISAVE+CPUSAVE_R31)(%r1) /* get r31 */
+ stw %r31,(PC_DBSAVE +CPUSAVE_R31)(%r1) /* save r31 */
+ b dbtrap
+#endif
+
+ /* XXX need stack probe here */
+realtrap:
+/* Test whether we already had PR set */
+ mfsrr1 %r1
+ mtcr %r1
+ mfsprg1 %r1 /* restore SP (might have been
+ overwritten) */
+ bf 17,k_trap /* branch if PSL_PR is false */
+ GET_CPUINFO(%r1)
+ lwz %r1,PC_CURPCB(%r1)
+ RESTORE_KERN_SRS(%r30,%r31) /* enable kernel mapping */
+ ba s_trap
+
+/*
+ * generictrap does some standard setup for trap handling to minimize
+ * the code that need be installed in the actual vectors. It expects
+ * the following conditions.
+ *
+ * R1 - Trap vector = LR & (0xff00 | R1)
+ * SPRG1 - Original R1 contents
+ * SPRG2 - Original LR
+ */
+
+generictrap64:
+ mtsprg3 %r31
+ mfmsr %r31
+ clrldi %r31,%r31,1
+ mtmsrd %r31
+ mfsprg3 %r31
+ isync
+
+generictrap:
+ /* Save R1 for computing the exception vector */
+ mtsprg3 %r1
+
+ /* Save interesting registers */
+ GET_CPUINFO(%r1)
+ stw %r28,(PC_TEMPSAVE+CPUSAVE_R28)(%r1) /* free r28-r31 */
+ stw %r29,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
+ stw %r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
+ stw %r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
+ mfsprg1 %r1 /* restore SP, in case of branch */
+ mfsprg2 %r28 /* save LR */
+ mfcr %r29 /* save CR */
+
+ /* Compute the exception vector from the link register */
+ mfsprg3 %r31
+ ori %r31,%r31,0xff00
+ mflr %r30
+ and %r30,%r30,%r31
+ mtsprg3 %r30
+
+ /* Test whether we already had PR set */
+ mfsrr1 %r31
+ mtcr %r31
+
+s_trap:
+ bf 17,k_trap /* branch if PSL_PR is false */
+ GET_CPUINFO(%r1)
+u_trap:
+ lwz %r1,PC_CURPCB(%r1)
+ RESTORE_KERN_SRS(%r30,%r31) /* enable kernel mapping */
+
+/*
+ * Now the common trap catching code.
+ */
+k_trap:
+ FRAME_SETUP(PC_TEMPSAVE)
+/* Call C interrupt dispatcher: */
+trapagain:
+ addi %r3,%r1,8
+ bl CNAME(powerpc_interrupt)
+ .globl CNAME(trapexit) /* backtrace code sentinel */
+CNAME(trapexit):
+
+/* Disable interrupts: */
+ mfmsr %r3
+ andi. %r3,%r3,~PSL_EE@l
+ mtmsr %r3
+/* Test AST pending: */
+ lwz %r5,FRAME_SRR1+8(%r1)
+ mtcr %r5
+ bf 17,1f /* branch if PSL_PR is false */
+
+ GET_CPUINFO(%r3) /* get per-CPU pointer */
+ lwz %r4, PC_CURTHREAD(%r3) /* deref to get curthread */
+ lwz %r4, TD_FLAGS(%r4) /* get thread flags value */
+ lis %r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@h
+ ori %r5,%r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@l
+ and. %r4,%r4,%r5
+ beq 1f
+ mfmsr %r3 /* re-enable interrupts */
+ ori %r3,%r3,PSL_EE@l
+ mtmsr %r3
+ isync
+ addi %r3,%r1,8
+ bl CNAME(ast)
+ .globl CNAME(asttrapexit) /* backtrace code sentinel #2 */
+CNAME(asttrapexit):
+ b trapexit /* test ast ret value ? */
+1:
+ FRAME_LEAVE(PC_TEMPSAVE)
+
+ .globl CNAME(rfi_patch1) /* replace rfi with rfid on ppc64 */
+CNAME(rfi_patch1):
+ rfi
+
+ .globl CNAME(rfid_patch)
+CNAME(rfid_patch):
+ rfid
+
+#if defined(KDB)
+/*
+ * Deliberate entry to dbtrap
+ */
+ .globl CNAME(breakpoint)
+CNAME(breakpoint):
+ mtsprg1 %r1
+ mfmsr %r3
+ mtsrr1 %r3
+ andi. %r3,%r3,~(PSL_EE|PSL_ME)@l
+ mtmsr %r3 /* disable interrupts */
+ isync
+ GET_CPUINFO(%r3)
+ stw %r28,(PC_DBSAVE+CPUSAVE_R28)(%r3)
+ stw %r29,(PC_DBSAVE+CPUSAVE_R29)(%r3)
+ stw %r30,(PC_DBSAVE+CPUSAVE_R30)(%r3)
+ stw %r31,(PC_DBSAVE+CPUSAVE_R31)(%r3)
+ mflr %r28
+ li %r29,EXC_BPT
+ mtlr %r29
+ mfcr %r29
+ mtsrr0 %r28
+
+/*
+ * Now the kdb trap catching code.
+ */
+dbtrap:
+ /* Write the trap vector to SPRG3 by computing LR & 0xff00 */
+ mflr %r1
+ andi. %r1,%r1,0xff00
+ mtsprg3 %r1
+
+ lis %r1,(tmpstk+TMPSTKSZ-16)@ha /* get new SP */
+ addi %r1,%r1,(tmpstk+TMPSTKSZ-16)@l
+
+ FRAME_SETUP(PC_DBSAVE)
+/* Call C trap code: */
+ addi %r3,%r1,8
+ bl CNAME(db_trap_glue)
+ or. %r3,%r3,%r3
+ bne dbleave
+/* This wasn't for KDB, so switch to real trap: */
+ lwz %r3,FRAME_EXC+8(%r1) /* save exception */
+ GET_CPUINFO(%r4)
+ stw %r3,(PC_DBSAVE+CPUSAVE_R31)(%r4)
+ FRAME_LEAVE(PC_DBSAVE)
+ mtsprg1 %r1 /* prepare for entrance to realtrap */
+ GET_CPUINFO(%r1)
+ stw %r28,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
+ stw %r29,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
+ stw %r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
+ stw %r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
+ mflr %r28
+ mfcr %r29
+ lwz %r31,(PC_DBSAVE+CPUSAVE_R31)(%r1)
+ mtsprg3 %r31 /* SPRG3 was clobbered by FRAME_LEAVE */
+ mfsprg1 %r1
+ b realtrap
+dbleave:
+ FRAME_LEAVE(PC_DBSAVE)
+ .globl CNAME(rfi_patch2) /* replace rfi with rfid on ppc64 */
+CNAME(rfi_patch2):
+ rfi
+
+/*
+ * In case of KDB we want a separate trap catcher for it
+ */
+ .globl CNAME(dblow),CNAME(dbsize)
+CNAME(dblow):
+ mtsprg1 %r1 /* save SP */
+ mtsprg2 %r29 /* save r29 */
+ mfcr %r29 /* save CR in r29 */
+ mfsrr1 %r1
+ mtcr %r1
+ bf 17,1f /* branch if privileged */
+
+ /* Unprivileged case */
+ mtcr %r29 /* put the condition register back */
+ mfsprg2 %r29 /* ... and r29 */
+ mflr %r1 /* save LR */
+ mtsprg2 %r1 /* And then in SPRG2 */
+ li %r1, 0 /* How to get the vector from LR */
+
+ bla generictrap /* and we look like a generic trap */
+1:
+ /* Privileged, so drop to KDB */
+ GET_CPUINFO(%r1)
+ stw %r28,(PC_DBSAVE+CPUSAVE_R28)(%r1) /* free r28 */
+ mfsprg2 %r28 /* r29 holds cr... */
+ stw %r28,(PC_DBSAVE+CPUSAVE_R29)(%r1) /* free r29 */
+ stw %r30,(PC_DBSAVE+CPUSAVE_R30)(%r1) /* free r30 */
+ stw %r31,(PC_DBSAVE+CPUSAVE_R31)(%r1) /* free r31 */
+ mflr %r28 /* save LR */
+ bla dbtrap
+CNAME(dbsize) = .-CNAME(dblow)
+#endif /* KDB */
diff --git a/sys/powerpc/aim/trap_subr64.S b/sys/powerpc/aim/trap_subr64.S
new file mode 100644
index 0000000..680de10
--- /dev/null
+++ b/sys/powerpc/aim/trap_subr64.S
@@ -0,0 +1,634 @@
+/* $FreeBSD$ */
+/* $NetBSD: trap_subr.S,v 1.20 2002/04/22 23:20:08 kleink Exp $ */
+
+/*-
+ * Copyright (C) 1995, 1996 Wolfgang Solfrank.
+ * Copyright (C) 1995, 1996 TooLs GmbH.
+ * All rights reserved.
+ *
+ * Redistribution and use in source and binary forms, with or without
+ * modification, are permitted provided that the following conditions
+ * are met:
+ * 1. Redistributions of source code must retain the above copyright
+ * notice, this list of conditions and the following disclaimer.
+ * 2. Redistributions in binary form must reproduce the above copyright
+ * notice, this list of conditions and the following disclaimer in the
+ * documentation and/or other materials provided with the distribution.
+ * 3. All advertising materials mentioning features or use of this software
+ * must display the following acknowledgement:
+ * This product includes software developed by TooLs GmbH.
+ * 4. The name of TooLs GmbH may not be used to endorse or promote products
+ * derived from this software without specific prior written permission.
+ *
+ * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
+ * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
+ * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
+ * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+ * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
+ * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
+ * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
+ * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
+ * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
+ * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
+ */
+
+/*
+ * NOTICE: This is not a standalone file. to use it, #include it in
+ * your port's locore.S, like so:
+ *
+ * #include <powerpc/aim/trap_subr.S>
+ */
+
+/*
+ * Save/restore segment registers
+ */
+
+/*
+ * Restore SRs for a pmap
+ *
+ * Requires that r28-r31 be scratch, with r28 initialized to the SLB cache
+ */
+
+restoresrs:
+ li %r29, 0 /* Set the counter to zero */
+
+ slbia
+ slbmfee %r31,%r29
+ clrrdi %r31,%r31,28
+ slbie %r31
+instslb:
+ ld %r31, 8(%r28); /* Load SLBE */
+
+ cmpli 0, %r31, 0; /* If SLBE is not valid, get the next */
+ beq nslb;
+
+ ld %r30, 0(%r28) /* Load SLBV */
+ slbmte %r30, %r31; /* Install SLB entry */
+
+nslb:
+ addi %r28, %r28, 16; /* Advance */
+ addi %r29, %r29, 1;
+ cmpli 0, %r29, 64; /* Repeat if we are not at the end */
+ blt instslb;
+
+ blr;
+
+/*
+ * User SRs are loaded through a pointer to the current pmap.
+ */
+#define RESTORE_USER_SRS() \
+ GET_CPUINFO(%r28); \
+ ld %r28,PC_USERSLB(%r28); \
+ bl restoresrs;
+
+/*
+ * Kernel SRs are loaded directly from kernel_pmap_
+ */
+#define RESTORE_KERN_SRS() \
+ GET_CPUINFO(%r28); \
+ addi %r28,%r28,PC_KERNSLB; \
+ bl restoresrs;
+
+/*
+ * FRAME_SETUP assumes:
+ * SPRG1 SP (1)
+ * SPRG3 trap type
+ * savearea r27-r31,DAR,DSISR (DAR & DSISR only for DSI traps)
+ * r28 LR
+ * r29 CR
+ * r30 scratch
+ * r31 scratch
+ * r1 kernel stack
+ * SRR0/1 as at start of trap
+ */
+#define FRAME_SETUP(savearea) \
+/* Have to enable translation to allow access of kernel stack: */ \
+ GET_CPUINFO(%r31); \
+ mfsrr0 %r30; \
+ std %r30,(savearea+CPUSAVE_SRR0)(%r31); /* save SRR0 */ \
+ mfsrr1 %r30; \
+ std %r30,(savearea+CPUSAVE_SRR1)(%r31); /* save SRR1 */ \
+ mfmsr %r30; \
+ ori %r30,%r30,(PSL_DR|PSL_IR|PSL_RI)@l; /* relocation on */ \
+ mtmsr %r30; /* stack can now be accessed */ \
+ isync; \
+ mfsprg1 %r31; /* get saved SP */ \
+ stdu %r31,-(FRAMELEN+288)(%r1); /* save it in the callframe */ \
+ std %r0, FRAME_0+48(%r1); /* save r0 in the trapframe */ \
+ std %r31,FRAME_1+48(%r1); /* save SP " " */ \
+ std %r2, FRAME_2+48(%r1); /* save r2 " " */ \
+ std %r28,FRAME_LR+48(%r1); /* save LR " " */ \
+ std %r29,FRAME_CR+48(%r1); /* save CR " " */ \
+ GET_CPUINFO(%r2); \
+ ld %r27,(savearea+CPUSAVE_R27)(%r2); /* get saved r27 */ \
+ ld %r28,(savearea+CPUSAVE_R28)(%r2); /* get saved r28 */ \
+ ld %r29,(savearea+CPUSAVE_R29)(%r2); /* get saved r29 */ \
+ ld %r30,(savearea+CPUSAVE_R30)(%r2); /* get saved r30 */ \
+ ld %r31,(savearea+CPUSAVE_R31)(%r2); /* get saved r31 */ \
+ std %r3, FRAME_3+48(%r1); /* save r3-r31 */ \
+ std %r4, FRAME_4+48(%r1); \
+ std %r5, FRAME_5+48(%r1); \
+ std %r6, FRAME_6+48(%r1); \
+ std %r7, FRAME_7+48(%r1); \
+ std %r8, FRAME_8+48(%r1); \
+ std %r9, FRAME_9+48(%r1); \
+ std %r10, FRAME_10+48(%r1); \
+ std %r11, FRAME_11+48(%r1); \
+ std %r12, FRAME_12+48(%r1); \
+ std %r13, FRAME_13+48(%r1); \
+ std %r14, FRAME_14+48(%r1); \
+ std %r15, FRAME_15+48(%r1); \
+ std %r16, FRAME_16+48(%r1); \
+ std %r17, FRAME_17+48(%r1); \
+ std %r18, FRAME_18+48(%r1); \
+ std %r19, FRAME_19+48(%r1); \
+ std %r20, FRAME_20+48(%r1); \
+ std %r21, FRAME_21+48(%r1); \
+ std %r22, FRAME_22+48(%r1); \
+ std %r23, FRAME_23+48(%r1); \
+ std %r24, FRAME_24+48(%r1); \
+ std %r25, FRAME_25+48(%r1); \
+ std %r26, FRAME_26+48(%r1); \
+ std %r27, FRAME_27+48(%r1); \
+ std %r28, FRAME_28+48(%r1); \
+ std %r29, FRAME_29+48(%r1); \
+ std %r30, FRAME_30+48(%r1); \
+ std %r31, FRAME_31+48(%r1); \
+ ld %r28,(savearea+CPUSAVE_AIM_DAR)(%r2); /* saved DAR */ \
+ ld %r29,(savearea+CPUSAVE_AIM_DSISR)(%r2);/* saved DSISR */\
+ ld %r30,(savearea+CPUSAVE_SRR0)(%r2); /* saved SRR0 */ \
+ ld %r31,(savearea+CPUSAVE_SRR1)(%r2); /* saved SRR1 */ \
+ mfxer %r3; \
+ mfctr %r4; \
+ mfsprg3 %r5; \
+ std %r3, FRAME_XER+48(1); /* save xer/ctr/exc */ \
+ std %r4, FRAME_CTR+48(1); \
+ std %r5, FRAME_EXC+48(1); \
+ std %r28,FRAME_AIM_DAR+48(1); \
+ std %r29,FRAME_AIM_DSISR+48(1); /* save dsisr/srr0/srr1 */ \
+ std %r30,FRAME_SRR0+48(1); \
+ std %r31,FRAME_SRR1+48(1)
+
+#define FRAME_LEAVE(savearea) \
+/* Now restore regs: */ \
+ ld %r2,FRAME_SRR0+48(%r1); \
+ ld %r3,FRAME_SRR1+48(%r1); \
+ ld %r4,FRAME_CTR+48(%r1); \
+ ld %r5,FRAME_XER+48(%r1); \
+ ld %r6,FRAME_LR+48(%r1); \
+ GET_CPUINFO(%r7); \
+ std %r2,(savearea+CPUSAVE_SRR0)(%r7); /* save SRR0 */ \
+ std %r3,(savearea+CPUSAVE_SRR1)(%r7); /* save SRR1 */ \
+ ld %r7,FRAME_CR+48(%r1); \
+ mtctr %r4; \
+ mtxer %r5; \
+ mtlr %r6; \
+ mtsprg1 %r7; /* save cr */ \
+ ld %r31,FRAME_31+48(%r1); /* restore r0-31 */ \
+ ld %r30,FRAME_30+48(%r1); \
+ ld %r29,FRAME_29+48(%r1); \
+ ld %r28,FRAME_28+48(%r1); \
+ ld %r27,FRAME_27+48(%r1); \
+ ld %r26,FRAME_26+48(%r1); \
+ ld %r25,FRAME_25+48(%r1); \
+ ld %r24,FRAME_24+48(%r1); \
+ ld %r23,FRAME_23+48(%r1); \
+ ld %r22,FRAME_22+48(%r1); \
+ ld %r21,FRAME_21+48(%r1); \
+ ld %r20,FRAME_20+48(%r1); \
+ ld %r19,FRAME_19+48(%r1); \
+ ld %r18,FRAME_18+48(%r1); \
+ ld %r17,FRAME_17+48(%r1); \
+ ld %r16,FRAME_16+48(%r1); \
+ ld %r15,FRAME_15+48(%r1); \
+ ld %r14,FRAME_14+48(%r1); \
+ ld %r13,FRAME_13+48(%r1); \
+ ld %r12,FRAME_12+48(%r1); \
+ ld %r11,FRAME_11+48(%r1); \
+ ld %r10,FRAME_10+48(%r1); \
+ ld %r9, FRAME_9+48(%r1); \
+ ld %r8, FRAME_8+48(%r1); \
+ ld %r7, FRAME_7+48(%r1); \
+ ld %r6, FRAME_6+48(%r1); \
+ ld %r5, FRAME_5+48(%r1); \
+ ld %r4, FRAME_4+48(%r1); \
+ ld %r3, FRAME_3+48(%r1); \
+ ld %r2, FRAME_2+48(%r1); \
+ ld %r0, FRAME_0+48(%r1); \
+ ld %r1, FRAME_1+48(%r1); \
+/* Can't touch %r1 from here on */ \
+ mtsprg2 %r2; /* save r2 & r3 */ \
+ mtsprg3 %r3; \
+/* Disable translation, machine check and recoverability: */ \
+ mfmsr %r2; \
+ andi. %r2,%r2,~(PSL_DR|PSL_IR|PSL_EE|PSL_ME|PSL_RI)@l; \
+ mtmsr %r2; \
+ isync; \
+/* Decide whether we return to user mode: */ \
+ GET_CPUINFO(%r2); \
+ ld %r3,(savearea+CPUSAVE_SRR1)(%r2); \
+ mtcr %r3; \
+ bf 17,1f; /* branch if PSL_PR is false */ \
+/* Restore user SRs */ \
+ GET_CPUINFO(%r3); \
+ std %r27,(savearea+CPUSAVE_R27)(%r3); \
+ std %r28,(savearea+CPUSAVE_R28)(%r3); \
+ std %r29,(savearea+CPUSAVE_R29)(%r3); \
+ std %r30,(savearea+CPUSAVE_R30)(%r3); \
+ std %r31,(savearea+CPUSAVE_R31)(%r3); \
+ mflr %r27; /* preserve LR */ \
+ RESTORE_USER_SRS(); /* uses r28-r31 */ \
+ mtlr %r27; \
+ ld %r31,(savearea+CPUSAVE_R31)(%r3); \
+ ld %r30,(savearea+CPUSAVE_R30)(%r3); \
+ ld %r29,(savearea+CPUSAVE_R29)(%r3); \
+ ld %r28,(savearea+CPUSAVE_R28)(%r3); \
+ ld %r27,(savearea+CPUSAVE_R27)(%r3); \
+1: mfsprg1 %r2; /* restore cr */ \
+ mtcr %r2; \
+ GET_CPUINFO(%r2); \
+ ld %r3,(savearea+CPUSAVE_SRR0)(%r2); /* restore srr0 */ \
+ mtsrr0 %r3; \
+ ld %r3,(savearea+CPUSAVE_SRR1)(%r2); /* restore srr1 */ \
+ mtsrr1 %r3; \
+ mfsprg2 %r2; /* restore r2 & r3 */ \
+ mfsprg3 %r3
+
+#ifdef SMP
+/*
+ * Processor reset exception handler. These are typically
+ * the first instructions the processor executes after a
+ * software reset. We do this in two bits so that we are
+ * not still hanging around in the trap handling region
+ * once the MMU is turned on.
+ */
+ .globl CNAME(rstcode), CNAME(rstsize)
+CNAME(rstcode):
+ /* Explicitly set MSR[SF] */
+ mfmsr %r9
+ li %r8,1
+ insrdi %r9,%r8,1,0
+ mtmsrd %r9
+ isync
+
+ ba cpu_reset
+CNAME(rstsize) = . - CNAME(rstcode)
+
+cpu_reset:
+ lis %r1,(tmpstk+TMPSTKSZ-48)@ha /* get new SP */
+ addi %r1,%r1,(tmpstk+TMPSTKSZ-48)@l
+
+ lis %r3,tocbase@ha
+ ld %r2,tocbase@l(%r3)
+ lis %r3,1@l
+ bl CNAME(.cpudep_ap_early_bootstrap) /* Set PCPU */
+ nop
+ bl CNAME(.pmap_cpu_bootstrap) /* Turn on virtual memory */
+ nop
+ bl CNAME(.cpudep_ap_bootstrap) /* Set up PCPU and stack */
+ nop
+ mr %r1,%r3 /* Use new stack */
+ bl CNAME(.machdep_ap_bootstrap) /* And away! */
+ nop
+
+ /* Should not be reached */
+9:
+ b 9b
+#endif
+
+/*
+ * This code gets copied to all the trap vectors
+ * (except ISI/DSI, ALI, and the interrupts)
+ */
+
+ .globl CNAME(trapcode),CNAME(trapsize)
+CNAME(trapcode):
+ mtsprg1 %r1 /* save SP */
+ mflr %r1 /* Save the old LR in r1 */
+ mtsprg2 %r1 /* And then in SPRG2 */
+ li %r1, 0xA0 /* How to get the vector from LR */
+ bla generictrap /* LR & SPRG3 is exception # */
+CNAME(trapsize) = .-CNAME(trapcode)
+
+/*
+ * For ALI: has to save DSISR and DAR
+ */
+ .globl CNAME(alitrap),CNAME(alisize)
+CNAME(alitrap):
+ mtsprg1 %r1 /* save SP */
+ GET_CPUINFO(%r1)
+ std %r27,(PC_TEMPSAVE+CPUSAVE_R27)(%r1) /* free r27-r31 */
+ std %r28,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
+ std %r29,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
+ std %r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
+ std %r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
+ mfdar %r30
+ mfdsisr %r31
+ std %r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1)
+ std %r31,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1)
+ mfsprg1 %r1 /* restore SP, in case of branch */
+ mflr %r28 /* save LR */
+ mfcr %r29 /* save CR */
+
+ /* Put our exception vector in SPRG3 */
+ li %r31, EXC_ALI
+ mtsprg3 %r31
+
+ /* Test whether we already had PR set */
+ mfsrr1 %r31
+ mtcr %r31
+ bla s_trap
+CNAME(alisize) = .-CNAME(alitrap)
+
+/*
+ * Similar to the above for DSI
+ * Has to handle BAT spills
+ * and standard pagetable spills
+ */
+ .globl CNAME(dsitrap),CNAME(dsisize)
+CNAME(dsitrap):
+ mtsprg1 %r1 /* save SP */
+ GET_CPUINFO(%r1)
+ std %r27,(PC_DISISAVE+CPUSAVE_R27)(%r1) /* free r27-r31 */
+ std %r28,(PC_DISISAVE+CPUSAVE_R28)(%r1)
+ std %r29,(PC_DISISAVE+CPUSAVE_R29)(%r1)
+ std %r30,(PC_DISISAVE+CPUSAVE_R30)(%r1)
+ std %r31,(PC_DISISAVE+CPUSAVE_R31)(%r1)
+ mfsprg1 %r1 /* restore SP */
+ mfcr %r29 /* save CR */
+ mfxer %r30 /* save XER */
+ mtsprg2 %r30 /* in SPRG2 */
+ mfsrr1 %r31 /* test kernel mode */
+ mtcr %r31
+ mflr %r28 /* save LR (SP already saved) */
+ bla disitrap
+CNAME(dsisize) = .-CNAME(dsitrap)
+
+/*
+ * Preamble code for DSI/ISI traps
+ */
+disitrap:
+ /* Write the trap vector to SPRG3 by computing LR & 0xff00 */
+ mflr %r1
+ andi. %r1,%r1,0xff00
+ mtsprg3 %r1
+
+ GET_CPUINFO(%r1)
+ ld %r31,(PC_DISISAVE+CPUSAVE_R27)(%r1)
+ std %r31,(PC_TEMPSAVE+CPUSAVE_R27)(%r1)
+ ld %r30,(PC_DISISAVE+CPUSAVE_R28)(%r1)
+ std %r30,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
+ ld %r31,(PC_DISISAVE+CPUSAVE_R29)(%r1)
+ std %r31,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
+ ld %r30,(PC_DISISAVE+CPUSAVE_R30)(%r1)
+ std %r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
+ ld %r31,(PC_DISISAVE+CPUSAVE_R31)(%r1)
+ std %r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
+ mfdar %r30
+ mfdsisr %r31
+ std %r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1)
+ std %r31,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1)
+
+#ifdef KDB
+ /* Try and detect a kernel stack overflow */
+ mfsrr1 %r31
+ mtcr %r31
+ bt 17,realtrap /* branch is user mode */
+ mfsprg1 %r31 /* get old SP */
+ sub. %r30,%r31,%r30 /* SP - DAR */
+ bge 1f
+ neg %r30,%r30 /* modulo value */
+1: cmpldi %cr0,%r30,4096 /* is DAR within a page of SP? */
+ bge %cr0,realtrap /* no, too far away. */
+
+ /* Now convert this DSI into a DDB trap. */
+ GET_CPUINFO(%r1)
+ ld %r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1) /* get DAR */
+ std %r30,(PC_DBSAVE +CPUSAVE_AIM_DAR)(%r1) /* save DAR */
+ ld %r30,(PC_TEMPSAVE+CPUSAVE_AIM_DSISR)(%r1) /* get DSISR */
+ std %r30,(PC_DBSAVE +CPUSAVE_AIM_DSISR)(%r1) /* save DSISR */
+ ld %r31,(PC_DISISAVE+CPUSAVE_R27)(%r1) /* get r27 */
+ std %r31,(PC_DBSAVE +CPUSAVE_R27)(%r1) /* save r27 */
+ ld %r30,(PC_DISISAVE+CPUSAVE_R28)(%r1) /* get r28 */
+ std %r30,(PC_DBSAVE +CPUSAVE_R28)(%r1) /* save r28 */
+ ld %r31,(PC_DISISAVE+CPUSAVE_R29)(%r1) /* get r29 */
+ std %r31,(PC_DBSAVE +CPUSAVE_R29)(%r1) /* save r29 */
+ ld %r30,(PC_DISISAVE+CPUSAVE_R30)(%r1) /* get r30 */
+ std %r30,(PC_DBSAVE +CPUSAVE_R30)(%r1) /* save r30 */
+ ld %r31,(PC_DISISAVE+CPUSAVE_R31)(%r1) /* get r31 */
+ std %r31,(PC_DBSAVE +CPUSAVE_R31)(%r1) /* save r31 */
+ b dbtrap
+#endif
+
+ /* XXX need stack probe here */
+realtrap:
+/* Test whether we already had PR set */
+ mfsrr1 %r1
+ mtcr %r1
+ mfsprg1 %r1 /* restore SP (might have been
+ overwritten) */
+ bf 17,k_trap /* branch if PSL_PR is false */
+ GET_CPUINFO(%r1)
+ ld %r1,PC_CURPCB(%r1)
+ mr %r27,%r28 /* Save LR, r29 */
+ mtsprg2 %r29
+ RESTORE_KERN_SRS() /* enable kernel mapping */
+ mfsprg2 %r29
+ mr %r28,%r27
+ ba s_trap
+
+/*
+ * generictrap does some standard setup for trap handling to minimize
+ * the code that need be installed in the actual vectors. It expects
+ * the following conditions.
+ *
+ * R1 - Trap vector = LR & (0xff00 | R1)
+ * SPRG1 - Original R1 contents
+ * SPRG2 - Original LR
+ */
+
+generictrap:
+ /* Save R1 for computing the exception vector */
+ mtsprg3 %r1
+
+ /* Save interesting registers */
+ GET_CPUINFO(%r1)
+ std %r27,(PC_TEMPSAVE+CPUSAVE_R27)(%r1) /* free r27-r31 */
+ std %r28,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
+ std %r29,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
+ std %r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
+ std %r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
+ mfdar %r30
+ std %r30,(PC_TEMPSAVE+CPUSAVE_AIM_DAR)(%r1)
+ mfsprg1 %r1 /* restore SP, in case of branch */
+ mfsprg2 %r28 /* save LR */
+ mfcr %r29 /* save CR */
+
+ /* Compute the exception vector from the link register */
+ mfsprg3 %r31
+ ori %r31,%r31,0xff00
+ mflr %r30
+ and %r30,%r30,%r31
+ mtsprg3 %r30
+
+ /* Test whether we already had PR set */
+ mfsrr1 %r31
+ mtcr %r31
+
+s_trap:
+ bf 17,k_trap /* branch if PSL_PR is false */
+ GET_CPUINFO(%r1)
+u_trap:
+ ld %r1,PC_CURPCB(%r1)
+ mr %r27,%r28 /* Save LR, r29 */
+ mtsprg2 %r29
+ RESTORE_KERN_SRS() /* enable kernel mapping */
+ mfsprg2 %r29
+ mr %r28,%r27
+
+/*
+ * Now the common trap catching code.
+ */
+k_trap:
+ FRAME_SETUP(PC_TEMPSAVE)
+/* Call C interrupt dispatcher: */
+trapagain:
+ lis %r3,tocbase@ha
+ ld %r2,tocbase@l(%r3)
+ addi %r3,%r1,48
+ bl CNAME(.powerpc_interrupt)
+ nop
+
+ .globl CNAME(trapexit) /* backtrace code sentinel */
+CNAME(trapexit):
+/* Disable interrupts: */
+ mfmsr %r3
+ andi. %r3,%r3,~PSL_EE@l
+ mtmsr %r3
+/* Test AST pending: */
+ ld %r5,FRAME_SRR1+48(%r1)
+ mtcr %r5
+ bf 17,1f /* branch if PSL_PR is false */
+
+ GET_CPUINFO(%r3) /* get per-CPU pointer */
+ ld %r4, PC_CURTHREAD(%r3) /* deref to get curthread */
+ lwz %r4, TD_FLAGS(%r4) /* get thread flags value */
+ lis %r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@h
+ ori %r5,%r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@l
+ and. %r4,%r4,%r5
+ beq 1f
+ mfmsr %r3 /* re-enable interrupts */
+ ori %r3,%r3,PSL_EE@l
+ mtmsr %r3
+ isync
+ lis %r3,tocbase@ha
+ ld %r2,tocbase@l(%r3)
+ addi %r3,%r1,48
+ bl CNAME(.ast)
+ nop
+ .globl CNAME(asttrapexit) /* backtrace code sentinel #2 */
+CNAME(asttrapexit):
+ b trapexit /* test ast ret value ? */
+1:
+ FRAME_LEAVE(PC_TEMPSAVE)
+ rfid
+
+#if defined(KDB)
+/*
+ * Deliberate entry to dbtrap
+ */
+ASENTRY(breakpoint)
+ mtsprg1 %r1
+ mfmsr %r3
+ mtsrr1 %r3
+ andi. %r3,%r3,~(PSL_EE|PSL_ME)@l
+ mtmsr %r3 /* disable interrupts */
+ isync
+ GET_CPUINFO(%r3)
+ std %r27,(PC_DBSAVE+CPUSAVE_R27)(%r3)
+ std %r28,(PC_DBSAVE+CPUSAVE_R28)(%r3)
+ std %r29,(PC_DBSAVE+CPUSAVE_R29)(%r3)
+ std %r30,(PC_DBSAVE+CPUSAVE_R30)(%r3)
+ std %r31,(PC_DBSAVE+CPUSAVE_R31)(%r3)
+ mflr %r28
+ li %r29,EXC_BPT
+ mtlr %r29
+ mfcr %r29
+ mtsrr0 %r28
+
+/*
+ * Now the kdb trap catching code.
+ */
+dbtrap:
+ /* Write the trap vector to SPRG3 by computing LR & 0xff00 */
+ mflr %r1
+ andi. %r1,%r1,0xff00
+ mtsprg3 %r1
+
+ lis %r1,(tmpstk+TMPSTKSZ-48)@ha /* get new SP */
+ addi %r1,%r1,(tmpstk+TMPSTKSZ-48)@l
+
+ FRAME_SETUP(PC_DBSAVE)
+/* Call C trap code: */
+ lis %r3,tocbase@ha
+ ld %r2,tocbase@l(%r3)
+ addi %r3,%r1,48
+ bl CNAME(.db_trap_glue)
+ nop
+ or. %r3,%r3,%r3
+ bne dbleave
+/* This wasn't for KDB, so switch to real trap: */
+ ld %r3,FRAME_EXC+48(%r1) /* save exception */
+ GET_CPUINFO(%r4)
+ std %r3,(PC_DBSAVE+CPUSAVE_R31)(%r4)
+ FRAME_LEAVE(PC_DBSAVE)
+ mtsprg1 %r1 /* prepare for entrance to realtrap */
+ GET_CPUINFO(%r1)
+ std %r27,(PC_TEMPSAVE+CPUSAVE_R27)(%r1)
+ std %r28,(PC_TEMPSAVE+CPUSAVE_R28)(%r1)
+ std %r29,(PC_TEMPSAVE+CPUSAVE_R29)(%r1)
+ std %r30,(PC_TEMPSAVE+CPUSAVE_R30)(%r1)
+ std %r31,(PC_TEMPSAVE+CPUSAVE_R31)(%r1)
+ mflr %r28
+ mfcr %r29
+ ld %r31,(PC_DBSAVE+CPUSAVE_R31)(%r1)
+ mtsprg3 %r31 /* SPRG3 was clobbered by FRAME_LEAVE */
+ mfsprg1 %r1
+ b realtrap
+dbleave:
+ FRAME_LEAVE(PC_DBSAVE)
+ rfid
+
+/*
+ * In case of KDB we want a separate trap catcher for it
+ */
+ .globl CNAME(dblow),CNAME(dbsize)
+CNAME(dblow):
+ mtsprg1 %r1 /* save SP */
+ mtsprg2 %r29 /* save r29 */
+ mfcr %r29 /* save CR in r29 */
+ mfsrr1 %r1
+ mtcr %r1
+ bf 17,1f /* branch if privileged */
+
+ /* Unprivileged case */
+ mtcr %r29 /* put the condition register back */
+ mfsprg2 %r29 /* ... and r29 */
+ mflr %r1 /* save LR */
+ mtsprg2 %r1 /* And then in SPRG2 */
+ li %r1, 0 /* How to get the vector from LR */
+
+ bla generictrap /* and we look like a generic trap */
+1:
+ /* Privileged, so drop to KDB */
+ GET_CPUINFO(%r1)
+ std %r27,(PC_DBSAVE+CPUSAVE_R27)(%r1) /* free r27 */
+ std %r28,(PC_DBSAVE+CPUSAVE_R28)(%r1) /* free r28 */
+ mfsprg2 %r28 /* r29 holds cr... */
+ std %r28,(PC_DBSAVE+CPUSAVE_R29)(%r1) /* free r29 */
+ std %r30,(PC_DBSAVE+CPUSAVE_R30)(%r1) /* free r30 */
+ std %r31,(PC_DBSAVE+CPUSAVE_R31)(%r1) /* free r31 */
+ mflr %r28 /* save LR */
+ bla dbtrap
+CNAME(dbsize) = .-CNAME(dblow)
+#endif /* KDB */
diff --git a/sys/powerpc/aim/vm_machdep.c b/sys/powerpc/aim/vm_machdep.c
index 121e901..6ee03fb 100644
--- a/sys/powerpc/aim/vm_machdep.c
+++ b/sys/powerpc/aim/vm_machdep.c
@@ -81,7 +81,6 @@
#include <sys/kernel.h>
#include <sys/mbuf.h>
#include <sys/sf_buf.h>
-#include <sys/syscall.h>
#include <sys/sysctl.h>
#include <sys/sysent.h>
#include <sys/unistd.h>
@@ -131,6 +130,10 @@ static u_int sf_buf_alloc_want;
*/
static struct mtx sf_buf_lock;
+#ifdef __powerpc64__
+extern uintptr_t tocbase;
+#endif
+
/*
* Finish a fork operation, with process p2 nearly set up.
@@ -147,7 +150,8 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
KASSERT(td1 == curthread || td1 == &thread0,
("cpu_fork: p1 not curproc and not proc0"));
- CTR3(KTR_PROC, "cpu_fork: called td1=%08x p2=%08x flags=%x", (u_int)td1, (u_int)p2, flags);
+ CTR3(KTR_PROC, "cpu_fork: called td1=%p p2=%p flags=%x",
+ td1, p2, flags);
if ((flags & RFPROC) == 0)
return;
@@ -155,7 +159,7 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
p1 = td1->td_proc;
pcb = (struct pcb *)((td2->td_kstack +
- td2->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb)) & ~0x2fU);
+ td2->td_kstack_pages * PAGE_SIZE - sizeof(struct pcb)) & ~0x2fUL);
td2->td_pcb = pcb;
/* Copy the pcb */
@@ -178,13 +182,22 @@ cpu_fork(struct thread *td1, struct proc *p2, struct thread *td2, int flags)
cf = (struct callframe *)tf - 1;
memset(cf, 0, sizeof(struct callframe));
+ #ifdef __powerpc64__
+ cf->cf_toc = tocbase;
+ #endif
cf->cf_func = (register_t)fork_return;
cf->cf_arg0 = (register_t)td2;
cf->cf_arg1 = (register_t)tf;
pcb->pcb_sp = (register_t)cf;
+ #ifdef __powerpc64__
+ pcb->pcb_lr = ((register_t *)fork_trampoline)[0];
+ pcb->pcb_toc = ((register_t *)fork_trampoline)[1];
+ #else
pcb->pcb_lr = (register_t)fork_trampoline;
- pcb->pcb_cpu.aim.usr = kernel_pmap->pm_sr[USER_SR];
+ #endif
+ pcb->pcb_cpu.aim.usr_vsid = 0;
+ pcb->pcb_cpu.aim.usr_esid = 0;
/* Setup to release spin count in fork_exit(). */
td2->td_md.md_spinlock_count = 1;
@@ -209,8 +222,8 @@ cpu_set_fork_handler(td, func, arg)
{
struct callframe *cf;
- CTR4(KTR_PROC, "%s called with td=%08x func=%08x arg=%08x",
- __func__, (u_int)td, (u_int)func, (u_int)arg);
+ CTR4(KTR_PROC, "%s called with td=%p func=%p arg=%p",
+ __func__, td, func, arg);
cf = (struct callframe *)td->td_pcb->pcb_sp;
@@ -384,7 +397,9 @@ is_physical_memory(addr)
}
/*
- * Threading functions
+ * CPU threading functions related to the VM layer. These could be used
+ * to map the SLB bits required for the kernel stack instead of forcing a
+ * fixed-size KVA.
*/
void
OpenPOWER on IntegriCloud