Follow up r223485, which made AIM use the ABI thread pointer instead of

PCPU fields for curthread, by doing the same to Book-E. This closes
some potential races switching between CPUs. As a side effect, it turns out
the AIM and Book-E swtch.S implementations were the same to within a few
registers, so move that to powerpc/powerpc.

MFC after: 3 months

9 files changed, 44 insertions, 172 deletions

diff --git a/sys/powerpc/booke/locore.S b/sys/powerpc/booke/locore.S
index d7ebac9..514afa6 100644
--- a/sys/powerpc/booke/locore.S
+++ b/sys/powerpc/booke/locore.S
@@ -738,8 +738,7 @@ ENTRY(icache_enable)
 setfault:
 	mflr	%r0
 	mfsprg0	%r4
-	lwz	%r4, PC_CURTHREAD(%r4)
-	lwz	%r4, TD_PCB(%r4)
+	lwz	%r4, TD_PCB(%r2)
 	stw	%r3, PCB_ONFAULT(%r4)
 	mfcr	%r10
 	mfctr	%r11
diff --git a/sys/powerpc/booke/machdep.c b/sys/powerpc/booke/machdep.c
index 9318012..69a9ef6 100644
--- a/sys/powerpc/booke/machdep.c
+++ b/sys/powerpc/booke/machdep.c
@@ -409,6 +409,11 @@ booke_init(uint32_t arg1, uint32_t arg2)
 	pc = &__pcpu[0];
 	pcpu_init(pc, 0, sizeof(struct pcpu));
 	pc->pc_curthread = &thread0;
+#ifdef __powerpc64__
+	__asm __volatile("mr 13,%0" :: "r"(pc->pc_curthread));
+#else
+	__asm __volatile("mr 2,%0" :: "r"(pc->pc_curthread));
+#endif
 	__asm __volatile("mtsprg 0, %0" :: "r"(pc));
 
 	/* Initialize system mutexes. */
diff --git a/sys/powerpc/booke/mp_cpudep.c b/sys/powerpc/booke/mp_cpudep.c
index 55e33d8..5f6bece 100644
--- a/sys/powerpc/booke/mp_cpudep.c
+++ b/sys/powerpc/booke/mp_cpudep.c
@@ -71,6 +71,11 @@ cpudep_ap_bootstrap()
 
 	/* Assign pcpu fields, return ptr to this AP's idle thread kstack */
 	pcpup->pc_curthread = pcpup->pc_idlethread;
+#ifdef __powerpc64__
+	__asm __volatile("mr 13,%0" :: "r"(pcpup->pc_curthread));
+#else
+	__asm __volatile("mr 2,%0" :: "r"(pcpup->pc_curthread));
+#endif
 	pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb;
 	sp = pcpup->pc_curpcb->pcb_sp;
 
diff --git a/sys/powerpc/booke/pmap.c b/sys/powerpc/booke/pmap.c
index 6d943c9..a271b99 100644
--- a/sys/powerpc/booke/pmap.c
+++ b/sys/powerpc/booke/pmap.c
@@ -100,8 +100,6 @@ __FBSDID("$FreeBSD$");
 
 #define TODO			panic("%s: not implemented", __func__);
 
-extern struct mtx sched_lock;
-
 extern int dumpsys_minidump;
 
 extern unsigned char _etext[];
@@ -1906,7 +1904,7 @@ mmu_booke_activate(mmu_t mmu, struct thread *td)
 
 	KASSERT((pmap != kernel_pmap), ("mmu_booke_activate: kernel_pmap!"));
 
-	mtx_lock_spin(&sched_lock);
+	sched_pin();
 
 	cpuid = PCPU_GET(cpuid);
 	CPU_SET_ATOMIC(cpuid, &pmap->pm_active);
@@ -1919,7 +1917,7 @@ mmu_booke_activate(mmu_t mmu, struct thread *td)
 	mtspr(SPR_PID0, pmap->pm_tid[cpuid]);
 	__asm __volatile("isync");
 
-	mtx_unlock_spin(&sched_lock);
+	sched_unpin();
 
 	CTR3(KTR_PMAP, "%s: e (tid = %d for '%s')", __func__,
 	    pmap->pm_tid[PCPU_GET(cpuid)], td->td_proc->p_comm);
diff --git a/sys/powerpc/booke/swtch.S b/sys/powerpc/booke/swtch.S
deleted file mode 100644
index 88cbf29..0000000
--- a/sys/powerpc/booke/swtch.S
+++ /dev/null
@@ -1,159 +0,0 @@
-/*-
- * Copyright (C) 2001 Benno Rice
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- *
- * THIS SOFTWARE IS PROVIDED BY Benno Rice ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- *
- *	from: $NetBSD: locore.S,v 1.24 2000/05/31 05:09:17 thorpej Exp $
- *	from: $FreeBSD$
- *
- * $FreeBSD$
- */
-/*-
- * Copyright (C) 1995, 1996 Wolfgang Solfrank.
- * Copyright (C) 1995, 1996 TooLs GmbH.
- * All rights reserved.
- *
- * Redistribution and use in source and binary forms, with or without
- * modification, are permitted provided that the following conditions
- * are met:
- * 1. Redistributions of source code must retain the above copyright
- *    notice, this list of conditions and the following disclaimer.
- * 2. Redistributions in binary form must reproduce the above copyright
- *    notice, this list of conditions and the following disclaimer in the
- *    documentation and/or other materials provided with the distribution.
- * 3. All advertising materials mentioning features or use of this software
- *    must display the following acknowledgement:
- *	This product includes software developed by TooLs GmbH.
- * 4. The name of TooLs GmbH may not be used to endorse or promote products
- *    derived from this software without specific prior written permission.
- *
- * THIS SOFTWARE IS PROVIDED BY TOOLS GMBH ``AS IS'' AND ANY EXPRESS OR
- * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
- * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
- * IN NO EVENT SHALL TOOLS GMBH BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
- * SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
- * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS;
- * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY,
- * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR
- * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF
- * ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
- */
-
-#include "assym.s"
-
-#include <sys/syscall.h>
-
-#include <machine/trap.h>
-#include <machine/param.h>
-#include <machine/asm.h>
-#include <machine/spr.h>
-
-/*
- * void cpu_throw(struct thread *old, struct thread *new)
- */
-ENTRY(cpu_throw)
-	mr	%r15, %r4
-	b	cpu_switchin
-
-/*
- * void cpu_switch(struct thread *old, struct thread *new, struct mutex *mtx);)
- *
- * Switch to a new thread saving the current state in the old thread.
- */
-ENTRY(cpu_switch)
-	stw	%r5, TD_LOCK(%r3)
-
-	lwz	%r5, TD_PCB(%r3)	/* Get the old thread's PCB ptr */
-
-	mr	%r12, %r2	
-	stmw	%r12, PCB_CONTEXT(%r5)	/* Save the non-volatile GP regs.
-					   These can now be used for scratch */
-
-	mfcr	%r16			/* Save the condition register */
-	stw	%r16, PCB_CR(%r5)
-	mflr	%r16			/* Save the link register */
-	stw	%r16, PCB_LR(%r5)
-	mfctr	%r16
-	stw	%r16, PCB_BOOKE_CTR(%r5)
-	mfxer	%r16
-	stw	%r16, PCB_BOOKE_XER(%r5)
-	mfspr	%r16, SPR_DBCR0
-	stw	%r16, PCB_BOOKE_DBCR0(%r5)
-
-	stw	%r1, PCB_SP(%r5)	/* Save the stack pointer */
-
-	mr	%r14, %r3		/* Copy the old thread ptr... */
-	mr	%r15, %r4		/* and the new thread ptr in scratch */
-
-	bl	pmap_deactivate		/* Deactivate the current pmap */
-
-cpu_switchin:
-	mr	%r3, %r15		/* Get new thread ptr */
-	bl	pmap_activate		/* Activate the new address space */
-
-	mfsprg	%r7, 0			/* Get the pcpu pointer */
-	stw	%r15, PC_CURTHREAD(%r7)	/* Store new current thread */
-	lwz	%r17, TD_PCB(%r15)	/* Store new current PCB */
-	stw	%r17, PC_CURPCB(%r7)
-
-	mr	%r3, %r17		/* Recover PCB ptr */
-	lmw	%r12, PCB_CONTEXT(%r3)	/* Load the non-volatile GP regs */
-	mr	%r2, %r12
-	lwz	%r5, PCB_CR(%r3)	/* Load the condition register */
-	mtcr	%r5
-	lwz	%r5, PCB_LR(%r3)	/* Load the link register */
-	mtlr	%r5
-	lwz	%r5, PCB_BOOKE_CTR(%r3)
-	mtctr	%r5
-	lwz	%r5, PCB_BOOKE_XER(%r3)
-	mtxer	%r5
-	lwz	%r5, PCB_BOOKE_DBCR0(%r3)
-	mtspr	SPR_DBCR0, %r5
-
-	lwz	%r1, PCB_SP(%r3)	/* Load the stack pointer */
-	blr
-
-/*
- * savectx(pcb)
- * Update pcb, saving current processor state
- */
-ENTRY(savectx)
-	mr	%r12, %r2
-	stmw	%r12, PCB_CONTEXT(%r3)	/* Save the non-volatile GP regs */
-	mfcr	%r4			/* Save the condition register */
-	stw	%r4, PCB_CONTEXT(%r3)
-	blr
-
-/*
- * fork_trampoline()
- * Set up the return from cpu_fork()
- */
-ENTRY(fork_trampoline)
-	lwz	%r3, CF_FUNC(%r1)
-	lwz	%r4, CF_ARG0(%r1)
-	lwz	%r5, CF_ARG1(%r1)
-	bl	fork_exit
-	addi	%r1, %r1, CF_SIZE-FSP	/* Allow 8 bytes in front of
-					   trapframe to simulate FRAME_SETUP
-					   does when allocating space for
-					   a frame pointer/saved LR */
-	b	trapexit
diff --git a/sys/powerpc/booke/trap_subr.S b/sys/powerpc/booke/trap_subr.S
index 33976ca..7576449 100644
--- a/sys/powerpc/booke/trap_subr.S
+++ b/sys/powerpc/booke/trap_subr.S
@@ -219,7 +219,8 @@
 	lwz	%r30, (savearea+CPUSAVE_SRR0)(%r2);			\
 	lwz	%r31, (savearea+CPUSAVE_SRR1)(%r2);			\
 	stw	%r30, FRAME_SRR0+8(%r1);				\
-	stw	%r31, FRAME_SRR1+8(%r1)
+	stw	%r31, FRAME_SRR1+8(%r1);				\
+	lwz	%r2,PC_CURTHREAD(%r2)	/* set curthread pointer */
 
 /*
  *
@@ -734,7 +735,8 @@ interrupt_vector_top:
 INTERRUPT(int_debug)
 	STANDARD_CRIT_PROLOG(SPR_SPRG2, PC_BOOKE_CRITSAVE, SPR_CSRR0, SPR_CSRR1)
 	FRAME_SETUP(SPR_SPRG2, PC_BOOKE_CRITSAVE, EXC_DEBUG)
-	lwz	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r2);
+	GET_CPUINFO(%r3)
+	lwz	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0)(%r3)
 	lis	%r4, interrupt_vector_base@ha
 	addi	%r4, %r4, interrupt_vector_base@l
 	cmplw	cr0, %r3, %r4
@@ -748,9 +750,10 @@ INTERRUPT(int_debug)
 	rlwinm	%r3, %r3, 0, 23, 21
 	stw	%r3, FRAME_SRR1+8(%r1);
 	/* Restore srr0 and srr1 as they could have been clobbered. */
-	lwz	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0+8)(%r2);
+	GET_CPUINFO(%r4)
+	lwz	%r3, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR0+8)(%r4);
 	mtspr	SPR_SRR0, %r3
-	lwz	%r4, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR1+8)(%r2);
+	lwz	%r4, (PC_BOOKE_CRITSAVE+CPUSAVE_SRR1+8)(%r4);
 	mtspr	SPR_SRR1, %r4
 	b	9f
 1:
diff --git a/sys/powerpc/include/pcpu.h b/sys/powerpc/include/pcpu.h
index 7ab184e..8a55967 100644
--- a/sys/powerpc/include/pcpu.h
+++ b/sys/powerpc/include/pcpu.h
@@ -135,7 +135,6 @@ struct pmap;
 
 #define pcpup	((struct pcpu *) powerpc_get_pcpup())
 
-#ifdef AIM /* Book-E not yet adapted */
 static __inline __pure2 struct thread *
 __curthread(void)
 {
@@ -148,7 +147,6 @@ __curthread(void)
 	return (td);
 }
 #define curthread (__curthread())
-#endif
 
 #define	PCPU_GET(member)	(pcpup->pc_ ## member)
 
diff --git a/sys/powerpc/aim/swtch32.S b/sys/powerpc/powerpc/swtch32.S
index fa87aba..358586f 100644
--- a/sys/powerpc/aim/swtch32.S
+++ b/sys/powerpc/powerpc/swtch32.S
@@ -64,6 +64,7 @@
 #include <machine/trap.h>
 #include <machine/param.h>
 #include <machine/asm.h>
+#include <machine/spr.h>
 
 /*
  * void cpu_throw(struct thread *old, struct thread *new)
@@ -88,6 +89,14 @@ ENTRY(cpu_switch)
 	stw	%r16,PCB_CR(%r6)
 	mflr	%r16			/* Save the link register */
 	stw	%r16,PCB_LR(%r6)
+#ifdef BOOKE
+	mfctr	%r16
+	stw	%r16,PCB_BOOKE_CTR(%r6)
+	mfxer	%r16
+	stw	%r16,PCB_BOOKE_XER(%r6)
+	mfspr	%r16,SPR_DBCR0
+	stw	%r16,PCB_BOOKE_DBCR0(%r6)
+#endif
 	stw	%r1,PCB_SP(%r6)		/* Save the stack pointer */
 
 	mr	%r14,%r3		/* Copy the old thread ptr... */
@@ -95,6 +104,7 @@ ENTRY(cpu_switch)
 	mr	%r16,%r5		/* and the new lock */
 	mr	%r17,%r6		/* and the PCB */
 	
+#ifdef AIM
 	lwz	%r7,PCB_FLAGS(%r17)
 	/* Save FPU context if needed */
 	andi.	%r7, %r7, PCB_FPU
@@ -110,6 +120,7 @@ ENTRY(cpu_switch)
 	bl	save_vec
 	
 .L2:
+#endif
 	mr	%r3,%r14		/* restore old thread ptr */
 	bl	pmap_deactivate		/* Deactivate the current pmap */
 
@@ -136,6 +147,7 @@ blocked_loop:
 	mr	%r3,%r2			/* Get new thread ptr */
 	bl	pmap_activate		/* Activate the new address space */
 
+#ifdef AIM
 	lwz	%r6, PCB_FLAGS(%r17)
 	/* Restore FPU context if needed */
 	andi.	%r6, %r6, PCB_FPU
@@ -151,18 +163,29 @@ blocked_loop:
 	mr	%r3,%r2			/* Pass curthread to enable_vec */
 	bl	enable_vec
 
-	/* thread to restore is in r3 */
 .L4:
+#endif
+	/* thread to restore is in r3 */
 	mr	%r3,%r17		/* Recover PCB ptr */
 	lmw	%r12,PCB_CONTEXT(%r3)	/* Load the non-volatile GP regs */
 	lwz	%r5,PCB_CR(%r3)		/* Load the condition register */
 	mtcr	%r5
 	lwz	%r5,PCB_LR(%r3)		/* Load the link register */
 	mtlr	%r5
+#ifdef AIM
 	lwz	%r5,PCB_AIM_USR_VSID(%r3) /* Load the USER_SR segment reg */
 	isync
 	mtsr	USER_SR,%r5
 	isync
+#endif
+#ifdef BOOKE
+	lwz	%r5,PCB_BOOKE_CTR(%r3)
+	mtctr	%r5
+	lwz	%r5,PCB_BOOKE_XER(%r3)
+	mtctr	%r5
+	lwz	%r5,PCB_BOOKE_DBCR0(%r3)
+	mtspr	SPR_DBCR0,%r5
+#endif
 	lwz	%r1,PCB_SP(%r3)		/* Load the stack pointer */
 	/*
 	 * Perform a dummy stwcx. to clear any reservations we may have
diff --git a/sys/powerpc/aim/swtch64.S b/sys/powerpc/powerpc/swtch64.S
index ab6f532..ab6f532 100644
--- a/sys/powerpc/aim/swtch64.S
+++ b/sys/powerpc/powerpc/swtch64.S