MFC

4 files changed, 25 insertions, 23 deletions

diff --git a/sys/ia64/ia64/exception.S b/sys/ia64/ia64/exception.S
index 7ff771a..729a96c 100644
--- a/sys/ia64/ia64/exception.S
+++ b/sys/ia64/ia64/exception.S
@@ -202,13 +202,13 @@ exception_save_restart:
 {	.mmi
 	st8		[r30]=r19,16		// length
 	st8		[r31]=r0,16		// flags
-	add		r19=16,r19
+	add		r29=16,r19		// Clobber restart token
 	;;
 }
 {	.mmi
 	st8.spill	[r30]=sp,16		// sp
 	st8		[r31]=r20,16		// unat
-	sub		sp=r23,r19
+	sub		sp=r23,r29
 	;;
 }
 {	.mmi
@@ -641,7 +641,7 @@ exception_restore_restart:
 	mov		r30=ar.bspstore
 	;;
 	loadrs					// load user regs
-	nop		0
+	mov		r29=0			// Clobber restart token
 	;;
 }
 {	.mmi
@@ -1094,22 +1094,26 @@ IVT_ENTRY(Data_Nested_TLB, 0x1400)
 }
 {	.mlx
 	mov		r26=ar.bsp
-	movl		r27=kstack
+	movl		r29=kstack
 	;;
 }
-{	.mmi
+{	.mlx
 	mov		r28=sp
-	nop		0
-	addl		r27=KSTACK_PAGES*PAGE_SIZE-16,r0
+	movl		r27=kstack_top
 	;;
 }
 {	.mmi
-	mov		sp=r27
+	add		sp=-16,r27
 	;;
 	mov		r27=ar.bspstore
 	nop		0
 	;;
 }
+	mov		ar.rsc=0
+	dep		r29=r27,r29,0,9
+	;;
+	mov		ar.bspstore=r29
+	;;
 	CALL(trap, 5, r30)
 IVT_END(Data_Nested_TLB)
 
diff --git a/sys/ia64/ia64/syscall.S b/sys/ia64/ia64/syscall.S
index b35523a..ef559e8 100644
--- a/sys/ia64/ia64/syscall.S
+++ b/sys/ia64/ia64/syscall.S
@@ -239,21 +239,22 @@ ENTRY_NOPROFILE(epc_syscall, 8)
 	;;
 }
 {	.mmi
-	mov		ar.bspstore=r15
+	mov		r13=ar.k4
 	add		r30=-SIZEOF_TRAPFRAME,r14
 	mov		r20=sp
 	;;
 }
 {	.mii
-	mov		r13=ar.k4
+	mov		r21=ar.unat
 	dep		r30=0,r30,0,10
 	;;
 	add		sp=-16,r30
 	;;
 }
 {	.mmi
+	mov		ar.bspstore=r15
+	;;
 	mov		ar.rnat=r19
-	mov		r21=ar.unat
 	add		r31=8,r30
 	;;
 }
diff --git a/sys/ia64/include/ia64_cpu.h b/sys/ia64/include/ia64_cpu.h
index 92614fc..bb8284d 100644
--- a/sys/ia64/include/ia64_cpu.h
+++ b/sys/ia64/include/ia64_cpu.h
@@ -508,17 +508,14 @@ ia64_enable_highfp(void)
 	__asm __volatile("rsm psr.dfh;; srlz.d");
 }
 
-static __inline void
-ia64_srlz_d(void)
-{
-	__asm __volatile("srlz.d");
-}
-
-static __inline void
-ia64_srlz_i(void)
-{
-	__asm __volatile("srlz.i;;");
-}
+/*
+ * Avoid inline functions for the following so that they still work
+ * correctly when inlining is not enabled (e.g. -O0). Function calls
+ * need data serialization after setting psr, which results in a
+ * hazard.
+ */
+#define	ia64_srlz_d()	__asm __volatile("srlz.d")
+#define	ia64_srlz_i()	__asm __volatile("srlz.i;;")
 
 #endif /* !LOCORE */
 
diff --git a/sys/ia64/include/pcpu.h b/sys/ia64/include/pcpu.h
index 58403aa..05e2cc1 100644
--- a/sys/ia64/include/pcpu.h
+++ b/sys/ia64/include/pcpu.h
@@ -68,7 +68,7 @@ struct pcpu_md {
 
 struct pcpu;
 
-register struct pcpu *pcpup __asm__("r13");
+register struct pcpu * volatile pcpup __asm__("r13");
 
 static __inline __pure2 struct thread *
 __curthread(void)