diff options
author | kib <kib@FreeBSD.org> | 2015-07-08 18:12:24 +0000 |
---|---|---|
committer | kib <kib@FreeBSD.org> | 2015-07-08 18:12:24 +0000 |
commit | c17f8bfdd50cd481c229672bc267cb3c845ace07 (patch) | |
tree | bcad6726cce83587071eccc899bb9aa9ec819f44 | |
parent | 52af6a67b662366f80816c510628a1be8fd2ba0c (diff) | |
download | FreeBSD-src-c17f8bfdd50cd481c229672bc267cb3c845ace07.zip FreeBSD-src-c17f8bfdd50cd481c229672bc267cb3c845ace07.tar.gz |
Add the atomic_thread_fence() family of functions with intent to
provide a semantic defined by the C11 fences with corresponding
memory_order.
atomic_thread_fence_acq() gives r | r, w, where r and w are read and
write accesses, and | denotes the fence itself.
atomic_thread_fence_rel() is r, w | w.
atomic_thread_fence_acq_rel() is the combination of the acquire and
release in single operation. Note that reads after the acq+rel fence
could be made visible before writes preceeding the fence.
atomic_thread_fence_seq_cst() orders all accesses before/after the
fence, and the fence itself is globally ordered against other
sequentially consistent atomic operations.
Reviewed by: alc
Discussed with: bde
Sponsored by: The FreeBSD Foundation
MFC after: 3 weeks
-rw-r--r-- | sys/amd64/include/atomic.h | 32 | ||||
-rw-r--r-- | sys/arm/include/atomic.h | 28 | ||||
-rw-r--r-- | sys/arm64/include/atomic.h | 28 | ||||
-rw-r--r-- | sys/i386/include/atomic.h | 32 | ||||
-rw-r--r-- | sys/mips/include/atomic.h | 28 | ||||
-rw-r--r-- | sys/powerpc/include/atomic.h | 41 | ||||
-rw-r--r-- | sys/sparc64/include/atomic.h | 29 |
7 files changed, 218 insertions, 0 deletions
diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h index 6d5c6b6..7892d51 100644 --- a/sys/amd64/include/atomic.h +++ b/sys/amd64/include/atomic.h @@ -84,6 +84,10 @@ u_int atomic_fetchadd_int(volatile u_int *p, u_int v); u_long atomic_fetchadd_long(volatile u_long *p, u_long v); int atomic_testandset_int(volatile u_int *p, u_int v); int atomic_testandset_long(volatile u_long *p, u_int v); +void atomic_thread_fence_acq(void); +void atomic_thread_fence_acq_rel(void); +void atomic_thread_fence_rel(void); +void atomic_thread_fence_seq_cst(void); #define ATOMIC_LOAD(TYPE) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -328,6 +332,34 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ } \ struct __hack +static __inline void +atomic_thread_fence_acq(void) +{ + + __compiler_membar(); +} + +static __inline void +atomic_thread_fence_rel(void) +{ + + __compiler_membar(); +} + +static __inline void +atomic_thread_fence_acq_rel(void) +{ + + __compiler_membar(); +} + +static __inline void +atomic_thread_fence_seq_cst(void) +{ + + __storeload_barrier(); +} + #endif /* KLD_MODULE || !__GNUCLIKE_ASM */ ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v); diff --git a/sys/arm/include/atomic.h b/sys/arm/include/atomic.h index 48b4a33..140a3bc 100644 --- a/sys/arm/include/atomic.h +++ b/sys/arm/include/atomic.h @@ -1103,6 +1103,34 @@ atomic_store_long(volatile u_long *dst, u_long src) *dst = src; } +static __inline void +atomic_thread_fence_acq(void) +{ + + dmb(); +} + +static __inline void +atomic_thread_fence_rel(void) +{ + + dmb(); +} + +static __inline void +atomic_thread_fence_acq_rel(void) +{ + + dmb(); +} + +static __inline void +atomic_thread_fence_seq_cst(void) +{ + + dmb(); +} + #define atomic_clear_ptr atomic_clear_32 #define atomic_set_ptr atomic_set_32 #define atomic_cmpset_ptr atomic_cmpset_32 diff --git a/sys/arm64/include/atomic.h b/sys/arm64/include/atomic.h index 99252b8..4c9d718 100644 --- a/sys/arm64/include/atomic.h +++ b/sys/arm64/include/atomic.h @@ -728,6 +728,34 @@ atomic_subtract_rel_64(volatile uint64_t *p, uint64_t val) ); } +static __inline void +atomic_thread_fence_acq(void) +{ + + dmb(ld); +} + +static __inline void +atomic_thread_fence_rel(void) +{ + + dmb(sy); +} + +static __inline void +atomic_thread_fence_acq_rel(void) +{ + + dmb(sy); +} + +static __inline void +atomic_thread_fence_seq_cst(void) +{ + + dmb(sy); +} + #define atomic_add_rel_long atomic_add_rel_64 #define atomic_clear_rel_long atomic_clear_rel_64 #define atomic_cmpset_rel_long atomic_cmpset_rel_64 diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h index 9d365bc..adc86ca 100644 --- a/sys/i386/include/atomic.h +++ b/sys/i386/include/atomic.h @@ -86,6 +86,10 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v) int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src); u_int atomic_fetchadd_int(volatile u_int *p, u_int v); int atomic_testandset_int(volatile u_int *p, u_int v); +void atomic_thread_fence_acq(void); +void atomic_thread_fence_acq_rel(void); +void atomic_thread_fence_rel(void); +void atomic_thread_fence_seq_cst(void); #define ATOMIC_LOAD(TYPE) \ u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p) @@ -310,6 +314,34 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \ } \ struct __hack +static __inline void +atomic_thread_fence_acq(void) +{ + + __compiler_membar(); +} + +static __inline void +atomic_thread_fence_rel(void) +{ + + __compiler_membar(); +} + +static __inline void +atomic_thread_fence_acq_rel(void) +{ + + __compiler_membar(); +} + +static __inline void +atomic_thread_fence_seq_cst(void) +{ + + __storeload_barrier(); +} + #ifdef _KERNEL #ifdef WANT_FUNCTIONS diff --git a/sys/mips/include/atomic.h b/sys/mips/include/atomic.h index b43cc61..d4b951e 100644 --- a/sys/mips/include/atomic.h +++ b/sys/mips/include/atomic.h @@ -496,6 +496,34 @@ atomic_fetchadd_64(__volatile uint64_t *p, uint64_t v) } #endif +static __inline void +atomic_thread_fence_acq(void) +{ + + mips_sync(); +} + +static __inline void +atomic_thread_fence_rel(void) +{ + + mips_sync(); +} + +static __inline void +atomic_thread_fence_acq_rel(void) +{ + + mips_sync(); +} + +static __inline void +atomic_thread_fence_seq_cst(void) +{ + + mips_sync(); +} + /* Operations on chars. */ #define atomic_set_char atomic_set_8 #define atomic_set_acq_char atomic_set_acq_8 diff --git a/sys/powerpc/include/atomic.h b/sys/powerpc/include/atomic.h index 60d6d5c..92fe279 100644 --- a/sys/powerpc/include/atomic.h +++ b/sys/powerpc/include/atomic.h @@ -730,4 +730,45 @@ atomic_swap_64(volatile u_long *p, u_long v) #undef __ATOMIC_REL #undef __ATOMIC_ACQ +static __inline void +atomic_thread_fence_acq(void) +{ + + /* See above comment about lwsync being broken on Book-E. */ +#ifdef __powerpc64__ + __asm __volatile("lwsync" : : : "memory"); +#else + __asm __volatile("sync" : : : "memory"); +#endif +} + +static __inline void +atomic_thread_fence_rel(void) +{ + +#ifdef __powerpc64__ + __asm __volatile("lwsync" : : : "memory"); +#else + __asm __volatile("sync" : : : "memory"); +#endif +} + +static __inline void +atomic_thread_fence_acq_rel(void) +{ + +#ifdef __powerpc64__ + __asm __volatile("lwsync" : : : "memory"); +#else + __asm __volatile("sync" : : : "memory"); +#endif +} + +static __inline void +atomic_thread_fence_seq_cst(void) +{ + + __asm __volatile("sync" : : : "memory"); +} + #endif /* ! _MACHINE_ATOMIC_H_ */ diff --git a/sys/sparc64/include/atomic.h b/sys/sparc64/include/atomic.h index 0ceced8..789ed24 100644 --- a/sys/sparc64/include/atomic.h +++ b/sys/sparc64/include/atomic.h @@ -279,6 +279,35 @@ atomic_store_rel_ ## name(volatile ptype p, vtype v) \ atomic_st_rel((p), (v), sz); \ } +static __inline void +atomic_thread_fence_acq(void) +{ + + __compiler_membar(); +} + +static __inline void +atomic_thread_fence_rel(void) +{ + + __compiler_membar(); +} + +static __inline void +atomic_thread_fence_acq_rel(void) +{ + + __compiler_membar(); +} + +static __inline void +atomic_thread_fence_seq_cst(void) +{ + + membar(LoadLoad | LoadStore | StoreStore | StoreLoad); +} + + ATOMIC_GEN(int, u_int *, u_int, u_int, 32); ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32); |