Use the ABI-mandated thread pointer register (r2 for ppc32, r13 for ppc64)

instead of a PCPU field for curthread. This averts a race on SMP systems
with a high interrupt rate where the thread looking up the value of
curthread could be preempted and migrated between obtaining the PCPU
pointer and reading the value of pc_curthread, resulting in curthread being
observed to be the current thread on the thread's original CPU. This played
merry havoc with the system, in particular with mutexes. Many thanks to
jhb for helping me work this one out.

Note that Book-E is in principle susceptible to the same problem, but has
not been modified yet due to lack of Book-E hardware.

MFC after:	2 weeks

10 files changed, 54 insertions, 49 deletions

diff --git a/sys/powerpc/aim/copyinout.c b/sys/powerpc/aim/copyinout.c
index 3592691..69ba431 100644
--- a/sys/powerpc/aim/copyinout.c
+++ b/sys/powerpc/aim/copyinout.c
@@ -142,7 +142,7 @@ copyout(const void *kaddr, void *udaddr, size_t len)
 	char		*up, *p;
 	size_t		l;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 
 	if (setfault(env)) {
@@ -183,7 +183,7 @@ copyin(const void *udaddr, void *kaddr, size_t len)
 	char		*kp, *p;
 	size_t		l;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 
 	if (setfault(env)) {
@@ -225,7 +225,7 @@ copyinstr(const void *udaddr, void *kaddr, size_t len, size_t *done)
 	size_t		l;
 	int		rv, c;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 
 	if (setfault(env)) {
@@ -267,7 +267,7 @@ subyte(void *addr, int byte)
 	faultbuf	env;
 	char		*p;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 	p = (char *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
 
@@ -293,7 +293,7 @@ suword32(void *addr, int word)
 	faultbuf	env;
 	int		*p;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 	p = (int *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
 
@@ -319,7 +319,7 @@ suword(void *addr, long word)
 	faultbuf	env;
 	long		*p;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 	p = (long *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
 
@@ -359,7 +359,7 @@ fubyte(const void *addr)
 	u_char		*p;
 	int		val;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 	p = (u_char *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
 
@@ -385,7 +385,7 @@ fuword32(const void *addr)
 	faultbuf	env;
 	int32_t		*p, val;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 	p = (int32_t *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
 
@@ -411,7 +411,7 @@ fuword(const void *addr)
 	faultbuf	env;
 	long		*p, val;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 	p = (long *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
 
@@ -444,7 +444,7 @@ casuword32(volatile uint32_t *addr, uint32_t old, uint32_t new)
 	faultbuf env;
 	uint32_t *p, val;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 	p = (uint32_t *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
 
@@ -489,7 +489,7 @@ casuword(volatile u_long *addr, u_long old, u_long new)
 	faultbuf env;
 	u_long *p, val;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	pm = &td->td_proc->p_vmspace->vm_pmap;
 	p = (u_long *)(USER_ADDR + ((uintptr_t)addr & ~SEGMENT_MASK));
 
diff --git a/sys/powerpc/aim/locore32.S b/sys/powerpc/aim/locore32.S
index 35ea99b..d0a3778 100644
--- a/sys/powerpc/aim/locore32.S
+++ b/sys/powerpc/aim/locore32.S
@@ -181,8 +181,7 @@ setfault:
 	mflr	0
 	mfcr	12
 	mfsprg	4,0
-	lwz	4,PC_CURTHREAD(4)
-	lwz	4,TD_PCB(4)
+	lwz	4,TD_PCB(2)	/* curthread = r2 */
 	stw	3,PCB_ONFAULT(4)
 	stw	0,0(3)
 	stw	1,4(3)
diff --git a/sys/powerpc/aim/locore64.S b/sys/powerpc/aim/locore64.S
index c200b4c..5af5e3a 100644
--- a/sys/powerpc/aim/locore64.S
+++ b/sys/powerpc/aim/locore64.S
@@ -203,8 +203,7 @@ ASENTRY(setfault)
 	mflr	0
 	mfcr	12
 	mfsprg	4,0
-	ld	4,PC_CURTHREAD(4)
-	ld	4,TD_PCB(4)
+	ld	4,TD_PCB(13)	/* curthread = r13 */
 	std	3,PCB_ONFAULT(4)
 	std	0,0(3)
 	std	1,8(3)
diff --git a/sys/powerpc/aim/machdep.c b/sys/powerpc/aim/machdep.c
index 3ccae91..c368c68 100644
--- a/sys/powerpc/aim/machdep.c
+++ b/sys/powerpc/aim/machdep.c
@@ -303,7 +303,7 @@ powerpc_init(vm_offset_t startkernel, vm_offset_t endkernel,
 	 */
 	pc = __pcpu;
 	pcpu_init(pc, 0, sizeof(struct pcpu));
-	pc->pc_curthread = &thread0;
+	curthread_reg = pc->pc_curthread = &thread0;
 	pc->pc_cpuid = 0;
 
 	__asm __volatile("mtsprg 0, %0" :: "r"(pc));
@@ -745,7 +745,7 @@ kcopy(const void *src, void *dst, size_t len)
 	faultbuf	env, *oldfault;
 	int		rv;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	oldfault = td->td_pcb->pcb_onfault;
 	if ((rv = setfault(env)) != 0) {
 		td->td_pcb->pcb_onfault = oldfault;
diff --git a/sys/powerpc/aim/mp_cpudep.c b/sys/powerpc/aim/mp_cpudep.c
index d617fde..68fe517 100644
--- a/sys/powerpc/aim/mp_cpudep.c
+++ b/sys/powerpc/aim/mp_cpudep.c
@@ -88,7 +88,7 @@ cpudep_ap_bootstrap(void)
 	msr = PSL_KERNSET & ~PSL_EE;
 	mtmsr(msr);
 
-	pcpup->pc_curthread = pcpup->pc_idlethread;
+	curthread_reg = pcpup->pc_curthread = pcpup->pc_idlethread;
 	pcpup->pc_curpcb = pcpup->pc_curthread->td_pcb;
 	sp = pcpup->pc_curpcb->pcb_sp;
 
diff --git a/sys/powerpc/aim/swtch32.S b/sys/powerpc/aim/swtch32.S
index cd141aa..a44b861 100644
--- a/sys/powerpc/aim/swtch32.S
+++ b/sys/powerpc/aim/swtch32.S
@@ -69,7 +69,7 @@
  * void cpu_throw(struct thread *old, struct thread *new)
  */
 ENTRY(cpu_throw)
-	mr	%r15, %r4
+	mr	%r2, %r4
 	b	cpu_switchin
 
 /*
@@ -89,10 +89,9 @@ ENTRY(cpu_switch)
 	mflr	%r16			/* Save the link register */
 	stw	%r16,PCB_LR(%r6)
 	stw	%r1,PCB_SP(%r6)		/* Save the stack pointer */
-	stw	%r2,PCB_TOC(%r6)	/* Save the TOC pointer */
 
 	mr	%r14,%r3		/* Copy the old thread ptr... */
-	mr	%r15,%r4		/* and the new thread ptr in scratch */
+	mr	%r2,%r4			/* and the new thread ptr in curthread */
 	mr	%r16,%r5		/* and the new lock */
 	mr	%r17,%r6		/* and the PCB */
 	
@@ -122,24 +121,24 @@ cpu_switchin:
 	lis	%r6,blocked_lock@ha
 	addi	%r6,%r6,blocked_lock@l
 blocked_loop:
-	lwz	%r7,TD_LOCK(%r15)
+	lwz	%r7,TD_LOCK(%r2)
 	cmpw	%r6,%r7 
 	beq	blocked_loop
 #endif
 
 	mfsprg	%r7,0			/* Get the pcpu pointer */
-	stw	%r15,PC_CURTHREAD(%r7)	/* Store new current thread */
-	lwz	%r17,TD_PCB(%r15)	/* Store new current PCB */
+	stw	%r2,PC_CURTHREAD(%r7)	/* Store new current thread */
+	lwz	%r17,TD_PCB(%r2)	/* Store new current PCB */
 	stw	%r17,PC_CURPCB(%r7)
 
-	mr	%r3,%r15		/* Get new thread ptr */
+	mr	%r3,%r2			/* Get new thread ptr */
 	bl	pmap_activate		/* Activate the new address space */
 
 	lwz	%r6, PCB_FLAGS(%r17)
 	/* Restore FPU context if needed */
 	andi.	%r6, %r6, PCB_FPU
 	beq	.L3
-	mr	%r3,%r15		/* Pass curthread to enable_fpu */
+	mr	%r3,%r2			/* Pass curthread to enable_fpu */
 	bl	enable_fpu
 
 .L3:
@@ -147,7 +146,7 @@ blocked_loop:
 	/* Restore Altivec context if needed */
 	andi.	%r6, %r6, PCB_VEC
 	beq	.L4
-	mr	%r3,%r15		/* Pass curthread to enable_vec */
+	mr	%r3,%r2			/* Pass curthread to enable_vec */
 	bl	enable_vec
 
 	/* thread to restore is in r3 */
@@ -163,7 +162,6 @@ blocked_loop:
 	mtsr	USER_SR,%r5
 	isync
 	lwz	%r1,PCB_SP(%r3)		/* Load the stack pointer */
-	lwz	%r2,PCB_TOC(%r3)	/* Load the TOC pointer */
 	/*
 	 * Perform a dummy stwcx. to clear any reservations we may have
 	 * inherited from the previous thread. It doesn't matter if the
diff --git a/sys/powerpc/aim/swtch64.S b/sys/powerpc/aim/swtch64.S
index 1877e8d..c5cdcbc 100644
--- a/sys/powerpc/aim/swtch64.S
+++ b/sys/powerpc/aim/swtch64.S
@@ -69,7 +69,7 @@
  * void cpu_throw(struct thread *old, struct thread *new)
  */
 ENTRY(cpu_throw)
-	mr	%r15, %r4
+	mr	%r13, %r4
 	b	cpu_switchin
 
 /*
@@ -83,7 +83,6 @@ ENTRY(cpu_switch)
 	ld	%r6,TD_PCB(%r3)		/* Get the old thread's PCB ptr */
 	std	%r12,PCB_CONTEXT(%r6)	/* Save the non-volatile GP regs.
 					   These can now be used for scratch */
-	std	%r13,PCB_CONTEXT+1*8(%r6)	
 	std	%r14,PCB_CONTEXT+2*8(%r6)	
 	std	%r15,PCB_CONTEXT+3*8(%r6)	
 	std	%r16,PCB_CONTEXT+4*8(%r6)	
@@ -111,7 +110,7 @@ ENTRY(cpu_switch)
 	std	%r2,PCB_TOC(%r6)	/* Save the TOC pointer */
 	
 	mr	%r14,%r3		/* Copy the old thread ptr... */
-	mr	%r15,%r4		/* and the new thread ptr in scratch */
+	mr	%r13,%r4		/* and the new thread ptr in curthread*/
 	mr	%r16,%r5		/* and the new lock */
 	mr	%r17,%r6		/* and the PCB */
 	
@@ -148,19 +147,19 @@ cpu_switchin:
 	lis	%r6,blocked_lock@ha
 	addi	%r6,%r6,blocked_lock@l
 blocked_loop:
-	ld	%r7,TD_LOCK(%r15)
+	ld	%r7,TD_LOCK(%r13)
 	cmpd	%r6,%r7 
 	beq	blocked_loop
 #endif
 
 	mfsprg	%r7,0			/* Get the pcpu pointer */
-	std	%r15,PC_CURTHREAD(%r7)	/* Store new current thread */
-	ld	%r17,TD_PCB(%r15)	/* Store new current PCB */
+	std	%r13,PC_CURTHREAD(%r7)	/* Store new current thread */
+	ld	%r17,TD_PCB(%r13)	/* Store new current PCB */
 	std	%r17,PC_CURPCB(%r7)
 
 	stdu	%r1,-48(%r1)
 
-	mr	%r3,%r15		/* Get new thread ptr */
+	mr	%r3,%r13		/* Get new thread ptr */
 	bl	pmap_activate		/* Activate the new address space */
 	nop
 
@@ -168,7 +167,7 @@ blocked_loop:
 	/* Restore FPU context if needed */
 	andi.	%r6, %r6, PCB_FPU
 	beq	.L3
-	mr	%r3,%r15		/* Pass curthread to enable_fpu */
+	mr	%r3,%r13		/* Pass curthread to enable_fpu */
 	bl	enable_fpu
 	nop
 
@@ -177,7 +176,7 @@ blocked_loop:
 	/* Restore Altivec context if needed */
 	andi.	%r6, %r6, PCB_VEC
 	beq	.L4
-	mr	%r3,%r15		/* Pass curthread to enable_vec */
+	mr	%r3,%r13		/* Pass curthread to enable_vec */
 	bl	enable_vec
 	nop
 
@@ -186,7 +185,6 @@ blocked_loop:
 	addi	%r1,%r1,48
 	mr	%r3,%r17		/* Recover PCB ptr */
 	ld	%r12,PCB_CONTEXT(%r3)	/* Load the non-volatile GP regs. */
-	ld	%r13,PCB_CONTEXT+1*8(%r3)	
 	ld	%r14,PCB_CONTEXT+2*8(%r3)	
 	ld	%r15,PCB_CONTEXT+3*8(%r3)	
 	ld	%r16,PCB_CONTEXT+4*8(%r3)	
diff --git a/sys/powerpc/aim/trap.c b/sys/powerpc/aim/trap.c
index 60c0f3b..51dee0e 100644
--- a/sys/powerpc/aim/trap.c
+++ b/sys/powerpc/aim/trap.c
@@ -149,7 +149,7 @@ trap(struct trapframe *frame)
 
 	PCPU_INC(cnt.v_trap);
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	p = td->td_proc;
 
 	type = ucode = frame->exc;
@@ -452,7 +452,7 @@ syscall(struct trapframe *frame)
 	struct syscall_args sa;
 	int error;
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 	td->td_frame = frame;
 
 #ifdef __powerpc64__
@@ -600,7 +600,7 @@ badaddr_read(void *addr, size_t size, int *rptr)
 	/* Get rid of any stale machine checks that have been waiting.  */
 	__asm __volatile ("sync; isync");
 
-	td = PCPU_GET(curthread);
+	td = curthread;
 
 	if (setfault(env)) {
 		td->td_pcb->pcb_onfault = 0;
diff --git a/sys/powerpc/aim/trap_subr32.S b/sys/powerpc/aim/trap_subr32.S
index 80329cd..f00020b 100644
--- a/sys/powerpc/aim/trap_subr32.S
+++ b/sys/powerpc/aim/trap_subr32.S
@@ -155,9 +155,15 @@
 	stw	%r28,FRAME_AIM_DAR+8(1);				\
 	stw	%r29,FRAME_AIM_DSISR+8(1); /* save dsisr/srr0/srr1 */	\
 	stw	%r30,FRAME_SRR0+8(1);					\
-	stw	%r31,FRAME_SRR1+8(1)
+	stw	%r31,FRAME_SRR1+8(1);					\
+	lwz	%r2,PC_CURTHREAD(%r2)	/* set curthread pointer */
 
 #define	FRAME_LEAVE(savearea)						\
+/* Disable exceptions: */						\
+	mfmsr	%r2;							\
+	andi.	%r2,%r2,~PSL_EE@l;					\
+	mtmsr	%r2;							\
+	isync;								\
 /* Now restore regs: */							\
 	lwz	%r2,FRAME_SRR0+8(%r1);					\
 	lwz	%r3,FRAME_SRR1+8(%r1);					\
@@ -209,7 +215,7 @@
 	mtsprg3	%r3;							\
 /* Disable translation, machine check and recoverability: */		\
 	mfmsr	%r2;							\
-	andi.	%r2,%r2,~(PSL_DR|PSL_IR|PSL_EE|PSL_ME|PSL_RI)@l;	\
+	andi.	%r2,%r2,~(PSL_DR|PSL_IR|PSL_ME|PSL_RI)@l;	\
 	mtmsr	%r2;							\
 	isync;								\
 /* Decide whether we return to user mode: */				\
@@ -764,8 +770,8 @@ CNAME(trapexit):
 	bf	17,1f			/* branch if PSL_PR is false */
 
 	GET_CPUINFO(%r3)		/* get per-CPU pointer */
-	lwz	%r4, PC_CURTHREAD(%r3)	/* deref to get curthread */
-	lwz	%r4, TD_FLAGS(%r4)	/* get thread flags value */
+	lwz	%r4, TD_FLAGS(%r2)	/* get thread flags value
+					 * (r2 is curthread) */
 	lis	%r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@h
 	ori	%r5,%r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@l
 	and.	%r4,%r4,%r5
diff --git a/sys/powerpc/aim/trap_subr64.S b/sys/powerpc/aim/trap_subr64.S
index 66c543f..1ec8507 100644
--- a/sys/powerpc/aim/trap_subr64.S
+++ b/sys/powerpc/aim/trap_subr64.S
@@ -179,9 +179,15 @@ restore_kernsrs:
 	std	%r28,FRAME_AIM_DAR+48(1);				\
 	std	%r29,FRAME_AIM_DSISR+48(1); /* save dsisr/srr0/srr1 */	\
 	std	%r30,FRAME_SRR0+48(1);					\
-	std	%r31,FRAME_SRR1+48(1)
+	std	%r31,FRAME_SRR1+48(1);					\
+	ld	%r13,PC_CURTHREAD(%r2)	/* set kernel curthread */
 
 #define	FRAME_LEAVE(savearea)						\
+/* Disable exceptions: */						\
+	mfmsr	%r2;							\
+	andi.	%r2,%r2,~PSL_EE@l;					\
+	mtmsr	%r2;							\
+	isync;								\
 /* Now restore regs: */							\
 	ld	%r2,FRAME_SRR0+48(%r1);					\
 	ld	%r3,FRAME_SRR1+48(%r1);					\
@@ -233,7 +239,7 @@ restore_kernsrs:
 	mtsprg3	%r3;							\
 /* Disable translation, machine check and recoverability: */		\
 	mfmsr	%r2;							\
-	andi.	%r2,%r2,~(PSL_DR|PSL_IR|PSL_EE|PSL_ME|PSL_RI)@l;	\
+	andi.	%r2,%r2,~(PSL_DR|PSL_IR|PSL_ME|PSL_RI)@l;		\
 	mtmsr	%r2;							\
 	isync;								\
 /* Decide whether we return to user mode: */				\
@@ -526,8 +532,7 @@ CNAME(trapexit):
 	bf	17,1f			/* branch if PSL_PR is false */
 
 	GET_CPUINFO(%r3)		/* get per-CPU pointer */
-	ld	%r4, PC_CURTHREAD(%r3)	/* deref to get curthread */
-	lwz	%r4, TD_FLAGS(%r4)	/* get thread flags value */
+	lwz	%r4, TD_FLAGS(%r13)	/* get thread flags value */
 	lis	%r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@h
 	ori	%r5,%r5, (TDF_ASTPENDING|TDF_NEEDRESCHED)@l
 	and.	%r4,%r4,%r5