From 1abaaa6dc2fde99f517e7f315a593fd2bbbccd58 Mon Sep 17 00:00:00 2001 From: dfr Date: Tue, 12 Sep 2000 22:45:44 +0000 Subject: * Redo the cmpset inlines to use one less register. This incidentally fixes a serious problem with the previous version where an input could have been placed in the same register as an output which would stop the inline from working properly. * Redo atomic_{set,clear,add,subtract}_{32,64} as inlines since the code sequence is shorter than the call sequence to the code in atomic.s. I will remove the functions from atomic.s after a grace period to allow people to rebuild kernel modules. --- sys/alpha/include/atomic.h | 178 +++++++++++++++++++++++++++++++++++++++------ 1 file changed, 156 insertions(+), 22 deletions(-) (limited to 'sys/alpha') diff --git a/sys/alpha/include/atomic.h b/sys/alpha/include/atomic.h index 5cc2c3a..d6869b2 100644 --- a/sys/alpha/include/atomic.h +++ b/sys/alpha/include/atomic.h @@ -44,15 +44,149 @@ void atomic_clear_16(volatile u_int16_t *, u_int16_t); void atomic_add_16(volatile u_int16_t *, u_int16_t); void atomic_subtract_16(volatile u_int16_t *, u_int16_t); -void atomic_set_32(volatile u_int32_t *, u_int32_t); -void atomic_clear_32(volatile u_int32_t *, u_int32_t); -void atomic_add_32(volatile u_int32_t *, u_int32_t); -void atomic_subtract_32(volatile u_int32_t *, u_int32_t); +static __inline void atomic_set_32(volatile u_int32_t *p, u_int32_t v) +{ + u_int32_t temp; + + __asm __volatile ( + "1:\tldl_l %0, %2\n\t" /* load old value */ + "bis %0, %3, %0\n\t" /* calculate new value */ + "stl_c %0, %1\n\t" /* attempt to store */ + "beq %0, 2f\n\t" /* spin if failed */ + "mb\n\t" /* drain to memory */ + ".section .text3,\"ax\"\n" /* improve branch prediction */ + "2:\tbr 1b\n" /* try again */ + ".previous\n" + : "=&r" (temp), "=m" (*p) + : "m" (*p), "r" (v) + : "memory"); +} + +static __inline void atomic_clear_32(volatile u_int32_t *p, u_int32_t v) +{ + u_int32_t temp; + + __asm __volatile ( + "1:\tldl_l %0, %2\n\t" /* load old value */ + "bic %0, %3, %0\n\t" /* calculate new value */ + "stl_c %0, %1\n\t" /* attempt to store */ + "beq %0, 2f\n\t" /* spin if failed */ + "mb\n\t" /* drain to memory */ + ".section .text3,\"ax\"\n" /* improve branch prediction */ + "2:\tbr 1b\n" /* try again */ + ".previous\n" + : "=&r" (temp), "=m" (*p) + : "m" (*p), "r" (v) + : "memory"); +} + +static __inline void atomic_add_32(volatile u_int32_t *p, u_int32_t v) +{ + u_int32_t temp; + + __asm __volatile ( + "1:\tldl_l %0, %2\n\t" /* load old value */ + "addl %0, %3, %0\n\t" /* calculate new value */ + "stl_c %0, %1\n\t" /* attempt to store */ + "beq %0, 2f\n\t" /* spin if failed */ + "mb\n\t" /* drain to memory */ + ".section .text3,\"ax\"\n" /* improve branch prediction */ + "2:\tbr 1b\n" /* try again */ + ".previous\n" + : "=&r" (temp), "=m" (*p) + : "m" (*p), "r" (v) + : "memory"); +} + +static __inline void atomic_subtract_32(volatile u_int32_t *p, u_int32_t v) +{ + u_int32_t temp; + + __asm __volatile ( + "1:\tldl_l %0, %2\n\t" /* load old value */ + "subl %0, %3, %0\n\t" /* calculate new value */ + "stl_c %0, %1\n\t" /* attempt to store */ + "beq %0, 2f\n\t" /* spin if failed */ + "mb\n\t" /* drain to memory */ + ".section .text3,\"ax\"\n" /* improve branch prediction */ + "2:\tbr 1b\n" /* try again */ + ".previous\n" + : "=&r" (temp), "=m" (*p) + : "m" (*p), "r" (v) + : "memory"); +} -void atomic_set_64(volatile u_int64_t *, u_int64_t); -void atomic_clear_64(volatile u_int64_t *, u_int64_t); -void atomic_add_64(volatile u_int64_t *, u_int64_t); -void atomic_subtract_64(volatile u_int64_t *, u_int64_t); +static __inline void atomic_set_64(volatile u_int64_t *p, u_int64_t v) +{ + u_int64_t temp; + + __asm __volatile ( + "1:\tldq_l %0, %2\n\t" /* load old value */ + "bis %0, %3, %0\n\t" /* calculate new value */ + "stq_c %0, %1\n\t" /* attempt to store */ + "beq %0, 2f\n\t" /* spin if failed */ + "mb\n\t" /* drain to memory */ + ".section .text3,\"ax\"\n" /* improve branch prediction */ + "2:\tbr 1b\n" /* try again */ + ".previous\n" + : "=&r" (temp), "=m" (*p) + : "m" (*p), "r" (v) + : "memory"); +} + +static __inline void atomic_clear_64(volatile u_int64_t *p, u_int64_t v) +{ + u_int64_t temp; + + __asm __volatile ( + "1:\tldq_l %0, %2\n\t" /* load old value */ + "bic %0, %3, %0\n\t" /* calculate new value */ + "stq_c %0, %1\n\t" /* attempt to store */ + "beq %0, 2f\n\t" /* spin if failed */ + "mb\n\t" /* drain to memory */ + ".section .text3,\"ax\"\n" /* improve branch prediction */ + "2:\tbr 1b\n" /* try again */ + ".previous\n" + : "=&r" (temp), "=m" (*p) + : "m" (*p), "r" (v) + : "memory"); +} + +static __inline void atomic_add_64(volatile u_int64_t *p, u_int64_t v) +{ + u_int64_t temp; + + __asm __volatile ( + "1:\tldq_l %0, %2\n\t" /* load old value */ + "addq %0, %3, %0\n\t" /* calculate new value */ + "stq_c %0, %1\n\t" /* attempt to store */ + "beq %0, 2f\n\t" /* spin if failed */ + "mb\n\t" /* drain to memory */ + ".section .text3,\"ax\"\n" /* improve branch prediction */ + "2:\tbr 1b\n" /* try again */ + ".previous\n" + : "=&r" (temp), "=m" (*p) + : "m" (*p), "r" (v) + : "memory"); +} + +static __inline void atomic_subtract_64(volatile u_int64_t *p, u_int64_t v) +{ + u_int64_t temp; + + __asm __volatile ( + "1:\tldq_l %0, %2\n\t" /* load old value */ + "subq %0, %3, %0\n\t" /* calculate new value */ + "stq_c %0, %1\n\t" /* attempt to store */ + "beq %0, 2f\n\t" /* spin if failed */ + "mb\n\t" /* drain to memory */ + ".section .text3,\"ax\"\n" /* improve branch prediction */ + "2:\tbr 1b\n" /* try again */ + ".previous\n" + : "=&r" (temp), "=m" (*p) + : "m" (*p), "r" (v) + : "memory"); +} #define atomic_set_char atomic_set_8 #define atomic_clear_char atomic_clear_8 @@ -82,20 +216,20 @@ void atomic_subtract_64(volatile u_int64_t *, u_int64_t); static __inline u_int32_t atomic_cmpset_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) { - u_int32_t ret, temp; + u_int32_t ret; __asm __volatile ( - "1:\tldl_l %1, %5\n\t" /* load old value */ - "cmpeq %1, %3, %0\n\t" /* compare */ + "1:\tldl_l %0, %4\n\t" /* load old value */ + "cmpeq %0, %2, %0\n\t" /* compare */ "beq %0, 2f\n\t" /* exit if not equal */ - "mov %4, %1\n\t" /* value to store */ - "stl_c %1, %2\n\t" /* attempt to store */ - "beq %1, 3f\n\t" /* if it failed, spin */ + "mov %3, %0\n\t" /* value to store */ + "stl_c %0, %1\n\t" /* attempt to store */ + "beq %0, 3f\n\t" /* if it failed, spin */ "2:\n" /* done */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "3:\tbr 1b\n" /* try again */ ".previous\n" - : "=&r" (ret), "=r" (temp), "=m" (*p) + : "=&r" (ret), "=m" (*p) : "r" (cmpval), "r" (newval), "m" (*p) : "memory"); @@ -110,20 +244,20 @@ atomic_cmpset_32(volatile u_int32_t* p, u_int32_t cmpval, u_int32_t newval) static __inline u_int64_t atomic_cmpset_64(volatile u_int64_t* p, u_int64_t cmpval, u_int64_t newval) { - u_int64_t ret, temp; + u_int64_t ret; __asm __volatile ( - "1:\tldq_l %1, %5\n\t" /* load old value */ - "cmpeq %1, %3, %0\n\t" /* compare */ + "1:\tldq_l %0, %4\n\t" /* load old value */ + "cmpeq %0, %2, %0\n\t" /* compare */ "beq %0, 2f\n\t" /* exit if not equal */ - "mov %4, %1\n\t" /* value to store */ - "stq_c %1, %2\n\t" /* attempt to store */ - "beq %1, 3f\n\t" /* if it failed, spin */ + "mov %3, %0\n\t" /* value to store */ + "stq_c %0, %1\n\t" /* attempt to store */ + "beq %0, 3f\n\t" /* if it failed, spin */ "2:\n" /* done */ ".section .text3,\"ax\"\n" /* improve branch prediction */ "3:\tbr 1b\n" /* try again */ ".previous\n" - : "=&r" (ret), "=r" (temp), "=m" (*p) + : "=&r" (ret), "=m" (*p) : "r" (cmpval), "r" (newval), "m" (*p) : "memory"); -- cgit v1.1