diff options
author | jkim <jkim@FreeBSD.org> | 2013-08-21 22:27:42 +0000 |
---|---|---|
committer | jkim <jkim@FreeBSD.org> | 2013-08-21 22:27:42 +0000 |
commit | fe93666d276267c0645b0156f24003d7706f4ffe (patch) | |
tree | bcd78c8641ce181a96df80765f2d6a86de358c90 /sys/i386/include | |
parent | 18751f3bf890aa5feaa6f7c85ef4298532dd859b (diff) | |
download | FreeBSD-src-fe93666d276267c0645b0156f24003d7706f4ffe.zip FreeBSD-src-fe93666d276267c0645b0156f24003d7706f4ffe.tar.gz |
Reimplement atomic_load_acq_64() and atomic_store_rel_64() for i386. These
functions are now real functions rather than function pointers. Supposedly,
it is faster for modern processors.
Suggested by: bde
Diffstat (limited to 'sys/i386/include')
-rw-r--r-- | sys/i386/include/atomic.h | 192 |
1 files changed, 110 insertions, 82 deletions
diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h index 41b4815..88a7a0c 100644 --- a/sys/i386/include/atomic.h +++ b/sys/i386/include/atomic.h @@ -32,6 +32,11 @@ #error this file needs sys/cdefs.h as a prerequisite #endif +#ifdef _KERNEL +#include <machine/md_var.h> +#include <machine/specialreg.h> +#endif + #define mb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") #define wmb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") #define rmb() __asm __volatile("lock; addl $0,(%%esp)" : : : "memory", "cc") @@ -87,6 +92,9 @@ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) #define ATOMIC_STORE(TYPE) \ void atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) +uint64_t atomic_load_acq_64(volatile uint64_t *); +void atomic_store_rel_64(volatile uint64_t *, uint64_t); + #else /* !KLD_MODULE && __GNUCLIKE_ASM */ /* @@ -124,83 +132,6 @@ atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)\ } \ struct __hack -#if defined(_KERNEL) && !defined(WANT_FUNCTIONS) - -/* I486 does not support SMP or CMPXCHG8B. */ -static __inline uint64_t -atomic_load_acq_64_i386(volatile uint64_t *p) -{ - volatile uint32_t *high, *low; - uint64_t res; - - low = (volatile uint32_t *)p; - high = (volatile uint32_t *)p + 1; - __asm __volatile( - " pushfl ; " - " cli ; " - " movl %1,%%eax ; " - " movl %2,%%edx ; " - " popfl" - : "=&A" (res) /* 0 */ - : "m" (*low), /* 1 */ - "m" (*high) /* 2 */ - : "memory"); - return (res); -} - -static __inline void -atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) -{ - volatile uint32_t *high, *low; - - low = (volatile uint32_t *)p; - high = (volatile uint32_t *)p + 1; - __asm __volatile( - " pushfl ; " - " cli ; " - " movl %%eax,%0 ; " - " movl %%edx,%1 ; " - " popfl" - : "=m" (*low), /* 0 */ - "=m" (*high) /* 1 */ - : "A" (v) /* 2 */ - : "memory"); -} - -static __inline uint64_t -atomic_load_acq_64_i586(volatile uint64_t *p) -{ - uint64_t res; - - __asm __volatile( - " movl %%ebx,%%eax ; " - " movl %%ecx,%%edx ; " - " " MPLOCKED " " - " cmpxchg8b %1" - : "=&A" (res), /* 0 */ - "+m" (*p) /* 1 */ - : : "memory", "cc"); - return (res); -} - -static __inline void -atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) -{ - - __asm __volatile( - " movl %%eax,%%ebx ; " - " movl %%edx,%%ecx ; " - "1: " - " " MPLOCKED " " - " cmpxchg8b %0 ; " - " jne 1b" - : "+m" (*p), /* 0 */ - "+A" (v) /* 1 */ - : : "ebx", "ecx", "memory", "cc"); -} - -#endif /* _KERNEL && !WANT_FUNCTIONS */ - /* * Atomic compare and set, used by the mutex functions * @@ -343,6 +274,108 @@ struct __hack #endif /* _KERNEL && !SMP */ +#ifdef _KERNEL + +#ifdef WANT_FUNCTIONS +uint64_t atomic_load_acq_64_i386(volatile uint64_t *); +uint64_t atomic_load_acq_64_i586(volatile uint64_t *); +void atomic_store_rel_64_i386(volatile uint64_t *, uint64_t); +void atomic_store_rel_64_i586(volatile uint64_t *, uint64_t); +#endif + +/* I486 does not support SMP or CMPXCHG8B. */ +static __inline uint64_t +atomic_load_acq_64_i386(volatile uint64_t *p) +{ + volatile uint32_t *q; + uint64_t res; + + q = (volatile uint32_t *)p; + __asm __volatile( + " pushfl ; " + " cli ; " + " movl %1,%%eax ; " + " movl %2,%%edx ; " + " popfl" + : "=&A" (res) /* 0 */ + : "m" (*q), /* 1 */ + "m" (*(q + 1)) /* 2 */ + : "memory"); + return (res); +} + +static __inline void +atomic_store_rel_64_i386(volatile uint64_t *p, uint64_t v) +{ + volatile uint32_t *q; + + q = (volatile uint32_t *)p; + __asm __volatile( + " pushfl ; " + " cli ; " + " movl %%eax,%0 ; " + " movl %%edx,%1 ; " + " popfl" + : "=m" (*q), /* 0 */ + "=m" (*(q + 1)) /* 1 */ + : "A" (v) /* 2 */ + : "memory"); +} + +static __inline uint64_t +atomic_load_acq_64_i586(volatile uint64_t *p) +{ + uint64_t res; + + __asm __volatile( + " movl %%ebx,%%eax ; " + " movl %%ecx,%%edx ; " + " " MPLOCKED " " + " cmpxchg8b %1" + : "=&A" (res), /* 0 */ + "+m" (*p) /* 1 */ + : : "memory", "cc"); + return (res); +} + +static __inline void +atomic_store_rel_64_i586(volatile uint64_t *p, uint64_t v) +{ + + __asm __volatile( + " movl %%eax,%%ebx ; " + " movl %%edx,%%ecx ; " + "1: " + " " MPLOCKED " " + " cmpxchg8b %0 ; " + " jne 1b" + : "+m" (*p), /* 0 */ + "+A" (v) /* 1 */ + : : "ebx", "ecx", "memory", "cc"); +} + +static __inline uint64_t +atomic_load_acq_64(volatile uint64_t *p) +{ + + if ((cpu_feature & CPUID_CX8) == 0) + return (atomic_load_acq_64_i386(p)); + else + return (atomic_load_acq_64_i586(p)); +} + +static __inline void +atomic_store_rel_64(volatile uint64_t *p, uint64_t v) +{ + + if ((cpu_feature & CPUID_CX8) == 0) + atomic_store_rel_64_i386(p, v); + else + atomic_store_rel_64_i586(p, v); +} + +#endif /* _KERNEL */ + #endif /* KLD_MODULE || !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); @@ -381,11 +414,6 @@ ATOMIC_STORE(long); #ifndef WANT_FUNCTIONS -#ifdef _KERNEL -extern uint64_t (*atomic_load_acq_64)(volatile uint64_t *); -extern void (*atomic_store_rel_64)(volatile uint64_t *, uint64_t); -#endif - static __inline int atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src) { |