summaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
Diffstat
-rw-r--r--sys/amd64/include/atomic.h32
-rw-r--r--sys/arm/include/atomic.h28
-rw-r--r--sys/arm64/include/atomic.h28
-rw-r--r--sys/i386/include/atomic.h32
-rw-r--r--sys/mips/include/atomic.h28
-rw-r--r--sys/powerpc/include/atomic.h41
-rw-r--r--sys/sparc64/include/atomic.h29
7 files changed, 218 insertions, 0 deletions
diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h
index 6d5c6b6..7892d51 100644
--- a/sys/amd64/include/atomic.h
+++ b/sys/amd64/include/atomic.h
@@ -84,6 +84,10 @@ u_int atomic_fetchadd_int(volatile u_int *p, u_int v);
u_long atomic_fetchadd_long(volatile u_long *p, u_long v);
int atomic_testandset_int(volatile u_int *p, u_int v);
int atomic_testandset_long(volatile u_long *p, u_int v);
+void atomic_thread_fence_acq(void);
+void atomic_thread_fence_acq_rel(void);
+void atomic_thread_fence_rel(void);
+void atomic_thread_fence_seq_cst(void);
#define ATOMIC_LOAD(TYPE) \
u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p)
@@ -328,6 +332,34 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
} \
struct __hack
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+ __compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+ __compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+ __compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+ __storeload_barrier();
+}
+
#endif /* KLD_MODULE || !__GNUCLIKE_ASM */
ATOMIC_ASM(set, char, "orb %b1,%0", "iq", v);
diff --git a/sys/arm/include/atomic.h b/sys/arm/include/atomic.h
index 48b4a33..140a3bc 100644
--- a/sys/arm/include/atomic.h
+++ b/sys/arm/include/atomic.h
@@ -1103,6 +1103,34 @@ atomic_store_long(volatile u_long *dst, u_long src)
*dst = src;
}
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+ dmb();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+ dmb();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+ dmb();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+ dmb();
+}
+
#define atomic_clear_ptr atomic_clear_32
#define atomic_set_ptr atomic_set_32
#define atomic_cmpset_ptr atomic_cmpset_32
diff --git a/sys/arm64/include/atomic.h b/sys/arm64/include/atomic.h
index 99252b8..4c9d718 100644
--- a/sys/arm64/include/atomic.h
+++ b/sys/arm64/include/atomic.h
@@ -728,6 +728,34 @@ atomic_subtract_rel_64(volatile uint64_t *p, uint64_t val)
);
}
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+ dmb(ld);
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+ dmb(sy);
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+ dmb(sy);
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+ dmb(sy);
+}
+
#define atomic_add_rel_long atomic_add_rel_64
#define atomic_clear_rel_long atomic_clear_rel_64
#define atomic_cmpset_rel_long atomic_cmpset_rel_64
diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h
index 9d365bc..adc86ca 100644
--- a/sys/i386/include/atomic.h
+++ b/sys/i386/include/atomic.h
@@ -86,6 +86,10 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
int atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
u_int atomic_fetchadd_int(volatile u_int *p, u_int v);
int atomic_testandset_int(volatile u_int *p, u_int v);
+void atomic_thread_fence_acq(void);
+void atomic_thread_fence_acq_rel(void);
+void atomic_thread_fence_rel(void);
+void atomic_thread_fence_seq_cst(void);
#define ATOMIC_LOAD(TYPE) \
u_##TYPE atomic_load_acq_##TYPE(volatile u_##TYPE *p)
@@ -310,6 +314,34 @@ atomic_store_rel_##TYPE(volatile u_##TYPE *p, u_##TYPE v) \
} \
struct __hack
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+ __compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+ __compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+ __compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+ __storeload_barrier();
+}
+
#ifdef _KERNEL
#ifdef WANT_FUNCTIONS
diff --git a/sys/mips/include/atomic.h b/sys/mips/include/atomic.h
index b43cc61..d4b951e 100644
--- a/sys/mips/include/atomic.h
+++ b/sys/mips/include/atomic.h
@@ -496,6 +496,34 @@ atomic_fetchadd_64(__volatile uint64_t *p, uint64_t v)
}
#endif
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+ mips_sync();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+ mips_sync();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+ mips_sync();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+ mips_sync();
+}
+
/* Operations on chars. */
#define atomic_set_char atomic_set_8
#define atomic_set_acq_char atomic_set_acq_8
diff --git a/sys/powerpc/include/atomic.h b/sys/powerpc/include/atomic.h
index 60d6d5c..92fe279 100644
--- a/sys/powerpc/include/atomic.h
+++ b/sys/powerpc/include/atomic.h
@@ -730,4 +730,45 @@ atomic_swap_64(volatile u_long *p, u_long v)
#undef __ATOMIC_REL
#undef __ATOMIC_ACQ
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+ /* See above comment about lwsync being broken on Book-E. */
+#ifdef __powerpc64__
+ __asm __volatile("lwsync" : : : "memory");
+#else
+ __asm __volatile("sync" : : : "memory");
+#endif
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+#ifdef __powerpc64__
+ __asm __volatile("lwsync" : : : "memory");
+#else
+ __asm __volatile("sync" : : : "memory");
+#endif
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+#ifdef __powerpc64__
+ __asm __volatile("lwsync" : : : "memory");
+#else
+ __asm __volatile("sync" : : : "memory");
+#endif
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+ __asm __volatile("sync" : : : "memory");
+}
+
#endif /* ! _MACHINE_ATOMIC_H_ */
diff --git a/sys/sparc64/include/atomic.h b/sys/sparc64/include/atomic.h
index 0ceced8..789ed24 100644
--- a/sys/sparc64/include/atomic.h
+++ b/sys/sparc64/include/atomic.h
@@ -279,6 +279,35 @@ atomic_store_rel_ ## name(volatile ptype p, vtype v) \
atomic_st_rel((p), (v), sz); \
}
+static __inline void
+atomic_thread_fence_acq(void)
+{
+
+ __compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_rel(void)
+{
+
+ __compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_acq_rel(void)
+{
+
+ __compiler_membar();
+}
+
+static __inline void
+atomic_thread_fence_seq_cst(void)
+{
+
+ membar(LoadLoad | LoadStore | StoreStore | StoreLoad);
+}
+
+
ATOMIC_GEN(int, u_int *, u_int, u_int, 32);
ATOMIC_GEN(32, uint32_t *, uint32_t, uint32_t, 32);
OpenPOWER on IntegriCloud