* Redo the cmpset inlines to use one less register. This incidentally

  fixes a serious problem with the previous version where an input could
  have been placed in the same register as an output which would stop
  the inline from working properly.

* Redo atomic_{set,clear,add,subtract}_{32,64} as inlines since the code
  sequence is shorter than the call sequence to the code in atomic.s.
  I will remove the functions from atomic.s after a grace period to allow
  people to rebuild kernel modules.

1 files changed, 156 insertions, 22 deletions

diff --git a/sys/alpha/include/atomic.h b/sys/alpha/include/atomic.h
index 5cc2c3a..d6869b2 100644
--- a/sys/alpha/include/atomic.h
+++ b/sys/alpha/include/atomic.h
@@ -44,15 +44,149 @@ void atomic_clear_16(volatile u_int16_t *, u_int16_t);
 void atomic_add_16(volatile u_int16_t *, u_int16_t);
 void atomic_subtract_16(volatile u_int16_t *, u_int16_t);
 
-void atomic_set_32(volatile u_int32_t *, u_int32_t);
-void atomic_clear_32(volatile u_int32_t *, u_int32_t);
-void atomic_add_32(volatile u_int32_t *, u_int32_t);
-void atomic_subtract_32(volatile u_int32_t *, u_int32_t);
+static __inline void atomic_set_32(volatile u_int32_t *p, u_int32_t v)
+{
+	u_int32_t temp;
+
+	__asm __volatile (
+		"1:\tldl_l %0, %2\n\t"		/* load old value */
+		"bis %0, %3, %0\n\t"		/* calculate new value */
+		"stl_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 2f\n\t"		/* spin if failed */
+		"mb\n\t"			/* drain to memory */
+		".section .text3,\"ax\"\n"	/* improve branch prediction */
+		"2:\tbr 1b\n"			/* try again */
+		".previous\n"
+		: "=&r" (temp), "=m" (*p)
+		: "m" (*p), "r" (v)
+		: "memory");
+}
+
+static __inline void atomic_clear_32(volatile u_int32_t *p, u_int32_t v)
+{
+	u_int32_t temp;
+
+	__asm __volatile (
+		"1:\tldl_l %0, %2\n\t"		/* load old value */
+		"bic %0, %3, %0\n\t"		/* calculate new value */
+		"stl_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 2f\n\t"		/* spin if failed */
+		"mb\n\t"			/* drain to memory */
+		".section .text3,\"ax\"\n"	/* improve branch prediction */
+		"2:\tbr 1b\n"			/* try again */
+		".previous\n"
+		: "=&r" (temp), "=m" (*p)
+		: "m" (*p), "r" (v)
+		: "memory");
+}
+
+static __inline void atomic_add_32(volatile u_int32_t *p, u_int32_t v)
+{
+	u_int32_t temp;
+
+	__asm __volatile (
+		"1:\tldl_l %0, %2\n\t"		/* load old value */
+		"addl %0, %3, %0\n\t"		/* calculate new value */
+		"stl_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 2f\n\t"		/* spin if failed */
+		"mb\n\t"			/* drain to memory */
+		".section .text3,\"ax\"\n"	/* improve branch prediction */
+		"2:\tbr 1b\n"			/* try again */
+		".previous\n"
+		: "=&r" (temp), "=m" (*p)
+		: "m" (*p), "r" (v)
+		: "memory");
+}
+
+static __inline void atomic_subtract_32(volatile u_int32_t *p, u_int32_t v)
+{
+	u_int32_t temp;
+
+	__asm __volatile (
+		"1:\tldl_l %0, %2\n\t"		/* load old value */
+		"subl %0, %3, %0\n\t"		/* calculate new value */
+		"stl_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 2f\n\t"		/* spin if failed */
+		"mb\n\t"			/* drain to memory */
+		".section .text3,\"ax\"\n"	/* improve branch prediction */
+		"2:\tbr 1b\n"			/* try again */
+		".previous\n"
+		: "=&r" (temp), "=m" (*p)
+		: "m" (*p), "r" (v)
+		: "memory");
+}
 
-void atomic_set_64(volatile u_int64_t *, u_int64_t);
-void atomic_clear_64(volatile u_int64_t *, u_int64_t);
-void atomic_add_64(volatile u_int64_t *, u_int64_t);
-void atomic_subtract_64(volatile u_int64_t *, u_int64_t);
+static __inline void atomic_set_64(volatile u_int64_t *p, u_int64_t v)
+{
+	u_int64_t temp;
+
+	__asm __volatile (
+		"1:\tldq_l %0, %2\n\t"		/* load old value */
+		"bis %0, %3, %0\n\t"		/* calculate new value */
+		"stq_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 2f\n\t"		/* spin if failed */
+		"mb\n\t"			/* drain to memory */
+		".section .text3,\"ax\"\n"	/* improve branch prediction */
+		"2:\tbr 1b\n"			/* try again */
+		".previous\n"
+		: "=&r" (temp), "=m" (*p)
+		: "m" (*p), "r" (v)
+		: "memory");
+}
+
+static __inline void atomic_clear_64(volatile u_int64_t *p, u_int64_t v)
+{
+	u_int64_t temp;
+
+	__asm __volatile (
+		"1:\tldq_l %0, %2\n\t"		/* load old value */
+		"bic %0, %3, %0\n\t"		/* calculate new value */
+		"stq_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 2f\n\t"		/* spin if failed */
+		"mb\n\t"			/* drain to memory */
+		".section .text3,\"ax\"\n"	/* improve branch prediction */
+		"2:\tbr 1b\n"			/* try again */
+		".previous\n"
+		: "=&r" (temp), "=m" (*p)
+		: "m" (*p), "r" (v)
+		: "memory");
+}
+
+static __inline void atomic_add_64(volatile u_int64_t *p, u_int64_t v)
+{
+	u_int64_t temp;
+
+	__asm __volatile (
+		"1:\tldq_l %0, %2\n\t"		/* load old value */
+		"addq %0, %3, %0\n\t"		/* calculate new value */
+		"stq_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 2f\n\t"		/* spin if failed */
+		"mb\n\t"			/* drain to memory */
+		".section .text3,\"ax\"\n"	/* improve branch prediction */
+		"2:\tbr 1b\n"			/* try again */
+		".previous\n"
+		: "=&r" (temp), "=m" (*p)
+		: "m" (*p), "r" (v)
+		: "memory");
+}
+
+static __inline void atomic_subtract_64(volatile u_int64_t *p, u_int64_t v)
+{
+	u_int64_t temp;
+
+	__asm __volatile (
+		"1:\tldq_l %0, %2\n\t"		/* load old value */
+		"subq %0, %3, %0\n\t"		/* calculate new value */
+		"stq_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 2f\n\t"		/* spin if failed */
+		"mb\n\t"			/* drain to memory */
+		".section .text3,\"ax\"\n"	/* improve branch prediction */
+		"2:\tbr 1b\n"			/* try again */
+		".previous\n"
+		: "=&r" (temp), "=m" (*p)
+		: "m" (*p), "r" (v)
+		: "memory");
+}
 
 #define atomic_set_char		atomic_set_8
 #define atomic_clear_char	atomic_clear_8
@@ -82,20 +216,20 @@ void atomic_subtract_64(volatile u_int64_t *, u_int64_t);
 static __inline u_int32_t
 atomic_cmpset_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval)
 {
-	u_int32_t ret, temp;
+	u_int32_t ret;
 
 	__asm __volatile (
-		"1:\tldl_l %1, %5\n\t"		/* load old value */
-		"cmpeq %1, %3, %0\n\t"		/* compare */
+		"1:\tldl_l %0, %4\n\t"		/* load old value */
+		"cmpeq %0, %2, %0\n\t"		/* compare */
 		"beq %0, 2f\n\t"		/* exit if not equal */
-		"mov %4, %1\n\t"		/* value to store */
-		"stl_c %1, %2\n\t"		/* attempt to store */
-		"beq %1, 3f\n\t"		/* if it failed, spin */
+		"mov %3, %0\n\t"		/* value to store */
+		"stl_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 3f\n\t"		/* if it failed, spin */
 		"2:\n"				/* done */
 		".section .text3,\"ax\"\n"	/* improve branch prediction */
 		"3:\tbr 1b\n"			/* try again */
 		".previous\n"
-		: "=&r" (ret), "=r" (temp), "=m" (*p)
+		: "=&r" (ret), "=m" (*p)
 		: "r" (cmpval), "r" (newval), "m" (*p)
 		: "memory");
 
@@ -110,20 +244,20 @@ atomic_cmpset_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval)
 static __inline u_int64_t
 atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval)
 {
-	u_int64_t ret, temp;
+	u_int64_t ret;
 
 	__asm __volatile (
-		"1:\tldq_l %1, %5\n\t"		/* load old value */
-		"cmpeq %1, %3, %0\n\t"		/* compare */
+		"1:\tldq_l %0, %4\n\t"		/* load old value */
+		"cmpeq %0, %2, %0\n\t"		/* compare */
 		"beq %0, 2f\n\t"		/* exit if not equal */
-		"mov %4, %1\n\t"		/* value to store */
-		"stq_c %1, %2\n\t"		/* attempt to store */
-		"beq %1, 3f\n\t"		/* if it failed, spin */
+		"mov %3, %0\n\t"		/* value to store */
+		"stq_c %0, %1\n\t"		/* attempt to store */
+		"beq %0, 3f\n\t"		/* if it failed, spin */
 		"2:\n"				/* done */
 		".section .text3,\"ax\"\n"	/* improve branch prediction */
 		"3:\tbr 1b\n"			/* try again */
 		".previous\n"
-		: "=&r" (ret), "=r" (temp), "=m" (*p)
+		: "=&r" (ret), "=m" (*p)
 		: "r" (cmpval), "r" (newval), "m" (*p)
 		: "memory");