s390/bitops,atomic: add missing memory barriers

When reworking the bitops and atomic ops I missed that those instructions
that got atomic behaviour only perform a "specific-operand-serialization"
instead of a full "serialization".
The compare-and-swap instruction used before performs a full serialization
before and after the instruction is executed, which means it has full
memory barrier semantics.
In order to give the new bitops and atomic ops functions also full memory
barrier semantics add a "bcr 14,0" before and after each of those new
instructions which performs full serialization as well.

This restores memory barrier semantics for bitops and atomic ops functions
which return values, like e.g. atomic_add_return(), but not for functions
which do not return a value, like e.g. atomic_add().
This is consistent to other architectures and what common code requires.

Cc: stable@vger.kernel.org # v3.13+
Signed-off-by: Heiko Carstens <heiko.carstens@de.ibm.com>
Signed-off-by: Martin Schwidefsky <schwidefsky@de.ibm.com>

2 files changed, 65 insertions, 46 deletions

diff --git a/arch/s390/include/asm/atomic.h b/arch/s390/include/asm/atomic.h
index fa9aaf7..1d47061 100644
--- a/arch/s390/include/asm/atomic.h
+++ b/arch/s390/include/asm/atomic.h
@@ -15,23 +15,29 @@
 
 #include <linux/compiler.h>
 #include <linux/types.h>
+#include <asm/barrier.h>
 #include <asm/cmpxchg.h>
 
 #define ATOMIC_INIT(i)  { (i) }
 
+#define __ATOMIC_NO_BARRIER	"\n"
+
 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 
 #define __ATOMIC_OR	"lao"
 #define __ATOMIC_AND	"lan"
 #define __ATOMIC_ADD	"laa"
+#define __ATOMIC_BARRIER "bcr	14,0\n"
 
-#define __ATOMIC_LOOP(ptr, op_val, op_string)				\
+#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)		\
 ({									\
 	int old_val;							\
 									\
 	typecheck(atomic_t *, ptr);					\
 	asm volatile(							\
+		__barrier						\
 		op_string "	%0,%2,%1\n"				\
+		__barrier						\
 		: "=d" (old_val), "+Q" ((ptr)->counter)			\
 		: "d" (op_val)						\
 		: "cc", "memory");					\
@@ -43,8 +49,9 @@
 #define __ATOMIC_OR	"or"
 #define __ATOMIC_AND	"nr"
 #define __ATOMIC_ADD	"ar"
+#define __ATOMIC_BARRIER "\n"
 
-#define __ATOMIC_LOOP(ptr, op_val, op_string)				\
+#define __ATOMIC_LOOP(ptr, op_val, op_string, __barrier)		\
 ({									\
 	int old_val, new_val;						\
 									\
@@ -82,7 +89,7 @@ static inline void atomic_set(atomic_t *v, int i)
 
 static inline int atomic_add_return(int i, atomic_t *v)
 {
-	return __ATOMIC_LOOP(v, i, __ATOMIC_ADD) + i;
+	return __ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_BARRIER) + i;
 }
 
 static inline void atomic_add(int i, atomic_t *v)
@@ -94,12 +101,10 @@ static inline void atomic_add(int i, atomic_t *v)
 			: "+Q" (v->counter)
 			: "i" (i)
 			: "cc", "memory");
-	} else {
-		atomic_add_return(i, v);
+		return;
 	}
-#else
-	atomic_add_return(i, v);
 #endif
+	__ATOMIC_LOOP(v, i, __ATOMIC_ADD, __ATOMIC_NO_BARRIER);
 }
 
 #define atomic_add_negative(_i, _v)	(atomic_add_return(_i, _v) < 0)
@@ -115,12 +120,12 @@ static inline void atomic_add(int i, atomic_t *v)
 
 static inline void atomic_clear_mask(unsigned int mask, atomic_t *v)
 {
-	__ATOMIC_LOOP(v, ~mask, __ATOMIC_AND);
+	__ATOMIC_LOOP(v, ~mask, __ATOMIC_AND, __ATOMIC_NO_BARRIER);
 }
 
 static inline void atomic_set_mask(unsigned int mask, atomic_t *v)
 {
-	__ATOMIC_LOOP(v, mask, __ATOMIC_OR);
+	__ATOMIC_LOOP(v, mask, __ATOMIC_OR, __ATOMIC_NO_BARRIER);
 }
 
 #define atomic_xchg(v, new) (xchg(&((v)->counter), new))
@@ -157,19 +162,24 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u)
 
 #ifdef CONFIG_64BIT
 
+#define __ATOMIC64_NO_BARRIER	"\n"
+
 #ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
 
 #define __ATOMIC64_OR	"laog"
 #define __ATOMIC64_AND	"lang"
 #define __ATOMIC64_ADD	"laag"
+#define __ATOMIC64_BARRIER "bcr	14,0\n"
 
-#define __ATOMIC64_LOOP(ptr, op_val, op_string)				\
+#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)		\
 ({									\
 	long long old_val;						\
 									\
 	typecheck(atomic64_t *, ptr);					\
 	asm volatile(							\
+		__barrier						\
 		op_string "	%0,%2,%1\n"				\
+		__barrier						\
 		: "=d" (old_val), "+Q" ((ptr)->counter)			\
 		: "d" (op_val)						\
 		: "cc", "memory");					\
@@ -181,8 +191,9 @@ static inline int __atomic_add_unless(atomic_t *v, int a, int u)
 #define __ATOMIC64_OR	"ogr"
 #define __ATOMIC64_AND	"ngr"
 #define __ATOMIC64_ADD	"agr"
+#define __ATOMIC64_BARRIER "\n"
 
-#define __ATOMIC64_LOOP(ptr, op_val, op_string)				\
+#define __ATOMIC64_LOOP(ptr, op_val, op_string, __barrier)		\
 ({									\
 	long long old_val, new_val;					\
 									\
@@ -220,17 +231,32 @@ static inline void atomic64_set(atomic64_t *v, long long i)
 
 static inline long long atomic64_add_return(long long i, atomic64_t *v)
 {
-	return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD) + i;
+	return __ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_BARRIER) + i;
+}
+
+static inline void atomic64_add(long long i, atomic64_t *v)
+{
+#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
+	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
+		asm volatile(
+			"agsi	%0,%1\n"
+			: "+Q" (v->counter)
+			: "i" (i)
+			: "cc", "memory");
+		return;
+	}
+#endif
+	__ATOMIC64_LOOP(v, i, __ATOMIC64_ADD, __ATOMIC64_NO_BARRIER);
 }
 
 static inline void atomic64_clear_mask(unsigned long mask, atomic64_t *v)
 {
-	__ATOMIC64_LOOP(v, ~mask, __ATOMIC64_AND);
+	__ATOMIC64_LOOP(v, ~mask, __ATOMIC64_AND, __ATOMIC64_NO_BARRIER);
 }
 
 static inline void atomic64_set_mask(unsigned long mask, atomic64_t *v)
 {
-	__ATOMIC64_LOOP(v, mask, __ATOMIC64_OR);
+	__ATOMIC64_LOOP(v, mask, __ATOMIC64_OR, __ATOMIC64_NO_BARRIER);
 }
 
 #define atomic64_xchg(v, new) (xchg(&((v)->counter), new))
@@ -334,25 +360,13 @@ static inline void atomic64_clear_mask(unsigned long long mask, atomic64_t *v)
 	} while (atomic64_cmpxchg(v, old, new) != old);
 }
 
-#endif /* CONFIG_64BIT */
-
 static inline void atomic64_add(long long i, atomic64_t *v)
 {
-#ifdef CONFIG_HAVE_MARCH_Z196_FEATURES
-	if (__builtin_constant_p(i) && (i > -129) && (i < 128)) {
-		asm volatile(
-			"agsi	%0,%1\n"
-			: "+Q" (v->counter)
-			: "i" (i)
-			: "cc", "memory");
-	} else {
-		atomic64_add_return(i, v);
-	}
-#else
 	atomic64_add_return(i, v);
-#endif
 }
 
+#endif /* CONFIG_64BIT */
+
 static inline int atomic64_add_unless(atomic64_t *v, long long i, long long u)
 {
 	long long c, old;
diff --git a/arch/s390/include/asm/bitops.h b/arch/s390/include/asm/bitops.h
index ec5ef89..5205424 100644
--- a/arch/s390/include/asm/bitops.h
+++ b/arch/s390/include/asm/bitops.h
@@ -47,14 +47,18 @@
 
 #include <linux/typecheck.h>
 #include <linux/compiler.h>
+#include <asm/barrier.h>
+
+#define __BITOPS_NO_BARRIER	"\n"
 
 #ifndef CONFIG_64BIT
 
 #define __BITOPS_OR		"or"
 #define __BITOPS_AND		"nr"
 #define __BITOPS_XOR		"xr"
+#define __BITOPS_BARRIER	"\n"
 
-#define __BITOPS_LOOP(__addr, __val, __op_string)		\
+#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\
 ({								\
 	unsigned long __old, __new;				\
 								\
@@ -67,7 +71,7 @@
 		"	jl	0b"				\
 		: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
 		: "d" (__val)					\
-		: "cc");					\
+		: "cc", "memory");				\
 	__old;							\
 })
 
@@ -78,17 +82,20 @@
 #define __BITOPS_OR		"laog"
 #define __BITOPS_AND		"lang"
 #define __BITOPS_XOR		"laxg"
+#define __BITOPS_BARRIER	"bcr	14,0\n"
 
-#define __BITOPS_LOOP(__addr, __val, __op_string)		\
+#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\
 ({								\
 	unsigned long __old;					\
 								\
 	typecheck(unsigned long *, (__addr));			\
 	asm volatile(						\
+		__barrier					\
 		__op_string "	%0,%2,%1\n"			\
+		__barrier					\
 		: "=d" (__old),	"+Q" (*(__addr))		\
 		: "d" (__val)					\
-		: "cc");					\
+		: "cc", "memory");				\
 	__old;							\
 })
 
@@ -97,8 +104,9 @@
 #define __BITOPS_OR		"ogr"
 #define __BITOPS_AND		"ngr"
 #define __BITOPS_XOR		"xgr"
+#define __BITOPS_BARRIER	"\n"
 
-#define __BITOPS_LOOP(__addr, __val, __op_string)		\
+#define __BITOPS_LOOP(__addr, __val, __op_string, __barrier)	\
 ({								\
 	unsigned long __old, __new;				\
 								\
@@ -111,7 +119,7 @@
 		"	jl	0b"				\
 		: "=&d" (__old), "=&d" (__new), "+Q" (*(__addr))\
 		: "d" (__val)					\
-		: "cc");					\
+		: "cc", "memory");				\
 	__old;							\
 })
 
@@ -149,12 +157,12 @@ static inline void set_bit(unsigned long nr, volatile unsigned long *ptr)
 			"oi	%0,%b1\n"
 			: "+Q" (*caddr)
 			: "i" (1 << (nr & 7))
-			: "cc");
+			: "cc", "memory");
 		return;
 	}
 #endif
 	mask = 1UL << (nr & (BITS_PER_LONG - 1));
-	__BITOPS_LOOP(addr, mask, __BITOPS_OR);
+	__BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_NO_BARRIER);
 }
 
 static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
@@ -170,12 +178,12 @@ static inline void clear_bit(unsigned long nr, volatile unsigned long *ptr)
 			"ni	%0,%b1\n"
 			: "+Q" (*caddr)
 			: "i" (~(1 << (nr & 7)))
-			: "cc");
+			: "cc", "memory");
 		return;
 	}
 #endif
 	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
-	__BITOPS_LOOP(addr, mask, __BITOPS_AND);
+	__BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_NO_BARRIER);
 }
 
 static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
@@ -191,12 +199,12 @@ static inline void change_bit(unsigned long nr, volatile unsigned long *ptr)
 			"xi	%0,%b1\n"
 			: "+Q" (*caddr)
 			: "i" (1 << (nr & 7))
-			: "cc");
+			: "cc", "memory");
 		return;
 	}
 #endif
 	mask = 1UL << (nr & (BITS_PER_LONG - 1));
-	__BITOPS_LOOP(addr, mask, __BITOPS_XOR);
+	__BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_NO_BARRIER);
 }
 
 static inline int
@@ -206,8 +214,7 @@ test_and_set_bit(unsigned long nr, volatile unsigned long *ptr)
 	unsigned long old, mask;
 
 	mask = 1UL << (nr & (BITS_PER_LONG - 1));
-	old = __BITOPS_LOOP(addr, mask, __BITOPS_OR);
-	barrier();
+	old = __BITOPS_LOOP(addr, mask, __BITOPS_OR, __BITOPS_BARRIER);
 	return (old & mask) != 0;
 }
 
@@ -218,8 +225,7 @@ test_and_clear_bit(unsigned long nr, volatile unsigned long *ptr)
 	unsigned long old, mask;
 
 	mask = ~(1UL << (nr & (BITS_PER_LONG - 1)));
-	old = __BITOPS_LOOP(addr, mask, __BITOPS_AND);
-	barrier();
+	old = __BITOPS_LOOP(addr, mask, __BITOPS_AND, __BITOPS_BARRIER);
 	return (old & ~mask) != 0;
 }
 
@@ -230,8 +236,7 @@ test_and_change_bit(unsigned long nr, volatile unsigned long *ptr)
 	unsigned long old, mask;
 
 	mask = 1UL << (nr & (BITS_PER_LONG - 1));
-	old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR);
-	barrier();
+	old = __BITOPS_LOOP(addr, mask, __BITOPS_XOR, __BITOPS_BARRIER);
 	return (old & mask) != 0;
 }