Eliminate the PC_CURTHREAD symbol and load the current thread's

thread structure pointer atomically from r13 (the pcpu pointer)
for the current CPU/core.
Add a CTASSERT in machdep.c to make sure that pc_curthread is in
fact the first field in struct pcpu.

The only non-atomic operations left were those related to process-
space operations, such as casuword, subyte, suword16, fubyte,
fuword16, copyin, copyout and their variations.

The casuword function has been re-structured more complete than
the others. This way we have an example of a better bundling
without introducing a lot of risk when we get it wrong. The
other functions can be rebundled in separate commits and with
the appropriate testing.

3 files changed, 33 insertions, 35 deletions

diff --git a/sys/ia64/ia64/genassym.c b/sys/ia64/ia64/genassym.c
index 6adaa33..10ba5ab 100644
--- a/sys/ia64/ia64/genassym.c
+++ b/sys/ia64/ia64/genassym.c
@@ -96,7 +96,6 @@ ASSYM(PAGE_SHIFT,	PAGE_SHIFT);
 ASSYM(PAGE_SIZE,	PAGE_SIZE);
 
 ASSYM(PC_CURRENT_PMAP,	offsetof(struct pcpu, pc_md.current_pmap));
-ASSYM(PC_CURTHREAD,	offsetof(struct pcpu, pc_curthread));
 ASSYM(PC_IDLETHREAD,	offsetof(struct pcpu, pc_idlethread));
 
 ASSYM(PCB_CURRENT_PMAP,	offsetof(struct pcb, pcb_current_pmap));
diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c
index fbc35d1..98bcf44 100644
--- a/sys/ia64/ia64/machdep.c
+++ b/sys/ia64/ia64/machdep.c
@@ -98,6 +98,22 @@ __FBSDID("$FreeBSD$");
 #include <machine/unwind.h>
 #include <machine/vmparam.h>
 
+/*
+ * For atomicity reasons, we demand that pc_curthread is the first
+ * field in the struct pcpu. It allows us to read the pointer with
+ * a single atomic instruction:
+ *	ld8 %curthread = [r13]
+ * Otherwise we would first have to calculate the load address and
+ * store the result in a temporary register and that for the load:
+ *	add %temp = %offsetof(struct pcpu), r13
+ *	ld8 %curthread = [%temp]
+ * A context switch inbetween the add and the ld8 could have the
+ * thread migrate to a different core. In that case,  %curthread
+ * would be the thread running on the original core and not actually
+ * the current thread.
+ */
+CTASSERT(offsetof(struct pcpu, pc_curthread) == 0);
+
 static SYSCTL_NODE(_hw, OID_AUTO, freq, CTLFLAG_RD, 0, "");
 static SYSCTL_NODE(_machdep, OID_AUTO, cpu, CTLFLAG_RD, 0, "");
 
diff --git a/sys/ia64/ia64/support.S b/sys/ia64/ia64/support.S
index b803581..46e5972 100644
--- a/sys/ia64/ia64/support.S
+++ b/sys/ia64/ia64/support.S
@@ -73,26 +73,25 @@ END(fusufault)
  */
 ENTRY(casuword, 3)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
+	ld8.acq		r15=[r13]		// r15 = curthread
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	add		r15=TD_PCB,r15
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
 }
 {	.mlx
-	add		r15=TD_PCB,r15
+	ld8		r15=[r15]		// r15 = PCB
 	movl		r14=fusufault
 	;;
 }
 {	.mmi
-	ld8		r15=[r15]		// r15 = PCB
-	;;
 	mov		ar.ccv=in1
 	add		r15=PCB_ONFAULT,r15
+	nop		0
 	;;
 }
 {	.mmi
@@ -123,12 +122,11 @@ END(casuword)
  */
 ENTRY(casuword32, 3)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	ld8.acq		r15=[r13]		// r15 = curthread
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
@@ -177,12 +175,11 @@ END(casuword32)
 
 ENTRY(subyte, 2)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	ld8.acq		r15=[r13]		// r15 = curthread
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
@@ -223,12 +220,11 @@ END(subyte)
 
 ENTRY(suword16, 2)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	ld8.acq		r15=[r13]		// r15 = curthread
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
@@ -269,12 +265,11 @@ END(suword16)
 
 ENTRY(suword32, 2)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	ld8.acq		r15=[r13]		// r15 = curthread
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
@@ -316,12 +311,11 @@ END(suword32)
 ENTRY(suword64, 2)
 XENTRY(suword)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	ld8.acq		r15=[r13]		// r15 = curthread
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
@@ -370,12 +364,11 @@ END(suword64)
 
 ENTRY(fubyte, 1)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	ld8.acq		r15=[r13]		// r15 = curthread
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
@@ -416,12 +409,11 @@ END(fubyte)
 
 ENTRY(fuword16, 2)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	ld8.acq		r15=[r13]		// r15 = curthread
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
@@ -462,12 +454,11 @@ END(fuword16)
 
 ENTRY(fuword32, 2)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	ld8.acq		r15=[r13]		// r15 = curthread
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
@@ -509,12 +500,11 @@ END(fuword32)
 ENTRY(fuword64, 2)
 XENTRY(fuword)
 {	.mlx
-	add		r15=PC_CURTHREAD,r13
 	movl		r14=VM_MAXUSER_ADDRESS
 	;;
 }
 {	.mib
-	ld8		r15=[r15]		// r15 = curthread
+	ld8.acq		r15=[r13]		// r15 = curthread
 	cmp.geu		p6,p0=in0,r14
 (p6)	br.dpnt.few	1f
 	;;
@@ -630,10 +620,8 @@ ENTRY(copyinstr, 4)
 	cmp.geu	p6,p0=in0,loc2			// is in user space.
 	;; 
 (p6)	br.cond.spnt.few copyerr		// if it's not, error out.
+	ld8.acq	r15=[r13]
 	movl	r14=copyerr			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
 	;;
 	add	r15=TD_PCB,r15			// find pcb
 	;;
@@ -743,10 +731,8 @@ ENTRY(copyin, 3)
 	cmp.geu	p6,p0=in0,loc2			// is in user space.
 	;; 
 (p6)	br.cond.spnt.few copyerr		// if it's not, error out.
+	ld8.acq	r15=[r13]
 	movl	r14=copyerr			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
 	;;
 	add	r15=TD_PCB,r15			// find pcb
 	;;
@@ -781,10 +767,8 @@ ENTRY(copyout, 3)
 	cmp.geu	p6,p0=in1,loc2			// is in user space.
 	;; 
 (p6)	br.cond.spnt.few copyerr		// if it's not, error out.
+	ld8.acq	r15=[r13]
 	movl	r14=copyerr			// set up fault handler.
-	add	r15=PC_CURTHREAD,r13		// find curthread
-	;;
-	ld8	r15=[r15]
 	;;
 	add	r15=TD_PCB,r15			// find pcb
 	;;
@@ -806,8 +790,7 @@ ENTRY(copyout, 3)
 END(copyout)
 
 ENTRY(copyerr, 0)
-	add	r14=PC_CURTHREAD,r13 ;;		// find curthread
-	ld8	r14=[r14] ;;
+	ld8.acq	r14=[r13] ;;
 	add	r14=TD_PCB,r14 ;;		// curthread->td_addr
 	ld8	r14=[r14] ;;
 	add	r14=PCB_ONFAULT,r14 ;;		// &curthread->td_pcb->pcb_onfault