From 6da471ce0205104fc4ba5b07ac0e084a5a77c6fd Mon Sep 17 00:00:00 2001 From: marcel Date: Tue, 12 Feb 2013 17:38:35 +0000 Subject: Eliminate the PC_CURTHREAD symbol and load the current thread's thread structure pointer atomically from r13 (the pcpu pointer) for the current CPU/core. Add a CTASSERT in machdep.c to make sure that pc_curthread is in fact the first field in struct pcpu. The only non-atomic operations left were those related to process- space operations, such as casuword, subyte, suword16, fubyte, fuword16, copyin, copyout and their variations. The casuword function has been re-structured more complete than the others. This way we have an example of a better bundling without introducing a lot of risk when we get it wrong. The other functions can be rebundled in separate commits and with the appropriate testing. --- sys/ia64/ia64/genassym.c | 1 - sys/ia64/ia64/machdep.c | 16 +++++++++++++++ sys/ia64/ia64/support.S | 51 ++++++++++++++++-------------------------------- 3 files changed, 33 insertions(+), 35 deletions(-) diff --git a/sys/ia64/ia64/genassym.c b/sys/ia64/ia64/genassym.c index 6adaa33..10ba5ab 100644 --- a/sys/ia64/ia64/genassym.c +++ b/sys/ia64/ia64/genassym.c @@ -96,7 +96,6 @@ ASSYM(PAGE_SHIFT, PAGE_SHIFT); ASSYM(PAGE_SIZE, PAGE_SIZE); ASSYM(PC_CURRENT_PMAP, offsetof(struct pcpu, pc_md.current_pmap)); -ASSYM(PC_CURTHREAD, offsetof(struct pcpu, pc_curthread)); ASSYM(PC_IDLETHREAD, offsetof(struct pcpu, pc_idlethread)); ASSYM(PCB_CURRENT_PMAP, offsetof(struct pcb, pcb_current_pmap)); diff --git a/sys/ia64/ia64/machdep.c b/sys/ia64/ia64/machdep.c index fbc35d1..98bcf44 100644 --- a/sys/ia64/ia64/machdep.c +++ b/sys/ia64/ia64/machdep.c @@ -98,6 +98,22 @@ __FBSDID("$FreeBSD$"); #include #include +/* + * For atomicity reasons, we demand that pc_curthread is the first + * field in the struct pcpu. It allows us to read the pointer with + * a single atomic instruction: + * ld8 %curthread = [r13] + * Otherwise we would first have to calculate the load address and + * store the result in a temporary register and that for the load: + * add %temp = %offsetof(struct pcpu), r13 + * ld8 %curthread = [%temp] + * A context switch inbetween the add and the ld8 could have the + * thread migrate to a different core. In that case, %curthread + * would be the thread running on the original core and not actually + * the current thread. + */ +CTASSERT(offsetof(struct pcpu, pc_curthread) == 0); + static SYSCTL_NODE(_hw, OID_AUTO, freq, CTLFLAG_RD, 0, ""); static SYSCTL_NODE(_machdep, OID_AUTO, cpu, CTLFLAG_RD, 0, ""); diff --git a/sys/ia64/ia64/support.S b/sys/ia64/ia64/support.S index b803581..46e5972 100644 --- a/sys/ia64/ia64/support.S +++ b/sys/ia64/ia64/support.S @@ -73,26 +73,25 @@ END(fusufault) */ ENTRY(casuword, 3) { .mlx - add r15=PC_CURTHREAD,r13 + ld8.acq r15=[r13] // r15 = curthread movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + add r15=TD_PCB,r15 cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; } { .mlx - add r15=TD_PCB,r15 + ld8 r15=[r15] // r15 = PCB movl r14=fusufault ;; } { .mmi - ld8 r15=[r15] // r15 = PCB - ;; mov ar.ccv=in1 add r15=PCB_ONFAULT,r15 + nop 0 ;; } { .mmi @@ -123,12 +122,11 @@ END(casuword) */ ENTRY(casuword32, 3) { .mlx - add r15=PC_CURTHREAD,r13 movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + ld8.acq r15=[r13] // r15 = curthread cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; @@ -177,12 +175,11 @@ END(casuword32) ENTRY(subyte, 2) { .mlx - add r15=PC_CURTHREAD,r13 movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + ld8.acq r15=[r13] // r15 = curthread cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; @@ -223,12 +220,11 @@ END(subyte) ENTRY(suword16, 2) { .mlx - add r15=PC_CURTHREAD,r13 movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + ld8.acq r15=[r13] // r15 = curthread cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; @@ -269,12 +265,11 @@ END(suword16) ENTRY(suword32, 2) { .mlx - add r15=PC_CURTHREAD,r13 movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + ld8.acq r15=[r13] // r15 = curthread cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; @@ -316,12 +311,11 @@ END(suword32) ENTRY(suword64, 2) XENTRY(suword) { .mlx - add r15=PC_CURTHREAD,r13 movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + ld8.acq r15=[r13] // r15 = curthread cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; @@ -370,12 +364,11 @@ END(suword64) ENTRY(fubyte, 1) { .mlx - add r15=PC_CURTHREAD,r13 movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + ld8.acq r15=[r13] // r15 = curthread cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; @@ -416,12 +409,11 @@ END(fubyte) ENTRY(fuword16, 2) { .mlx - add r15=PC_CURTHREAD,r13 movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + ld8.acq r15=[r13] // r15 = curthread cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; @@ -462,12 +454,11 @@ END(fuword16) ENTRY(fuword32, 2) { .mlx - add r15=PC_CURTHREAD,r13 movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + ld8.acq r15=[r13] // r15 = curthread cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; @@ -509,12 +500,11 @@ END(fuword32) ENTRY(fuword64, 2) XENTRY(fuword) { .mlx - add r15=PC_CURTHREAD,r13 movl r14=VM_MAXUSER_ADDRESS ;; } { .mib - ld8 r15=[r15] // r15 = curthread + ld8.acq r15=[r13] // r15 = curthread cmp.geu p6,p0=in0,r14 (p6) br.dpnt.few 1f ;; @@ -630,10 +620,8 @@ ENTRY(copyinstr, 4) cmp.geu p6,p0=in0,loc2 // is in user space. ;; (p6) br.cond.spnt.few copyerr // if it's not, error out. + ld8.acq r15=[r13] movl r14=copyerr // set up fault handler. - add r15=PC_CURTHREAD,r13 // find curthread - ;; - ld8 r15=[r15] ;; add r15=TD_PCB,r15 // find pcb ;; @@ -743,10 +731,8 @@ ENTRY(copyin, 3) cmp.geu p6,p0=in0,loc2 // is in user space. ;; (p6) br.cond.spnt.few copyerr // if it's not, error out. + ld8.acq r15=[r13] movl r14=copyerr // set up fault handler. - add r15=PC_CURTHREAD,r13 // find curthread - ;; - ld8 r15=[r15] ;; add r15=TD_PCB,r15 // find pcb ;; @@ -781,10 +767,8 @@ ENTRY(copyout, 3) cmp.geu p6,p0=in1,loc2 // is in user space. ;; (p6) br.cond.spnt.few copyerr // if it's not, error out. + ld8.acq r15=[r13] movl r14=copyerr // set up fault handler. - add r15=PC_CURTHREAD,r13 // find curthread - ;; - ld8 r15=[r15] ;; add r15=TD_PCB,r15 // find pcb ;; @@ -806,8 +790,7 @@ ENTRY(copyout, 3) END(copyout) ENTRY(copyerr, 0) - add r14=PC_CURTHREAD,r13 ;; // find curthread - ld8 r14=[r14] ;; + ld8.acq r14=[r13] ;; add r14=TD_PCB,r14 ;; // curthread->td_addr ld8 r14=[r14] ;; add r14=PCB_ONFAULT,r14 ;; // &curthread->td_pcb->pcb_onfault -- cgit v1.1