Implement atomic_swap() and atomic_testandset().

Reviewed by:	arch, bde, jilles, kib

3 files changed, 166 insertions, 51 deletions

diff --git a/share/man/man9/atomic.9 b/share/man/man9/atomic.9
index 0baac45..a2c0a67 100644
--- a/share/man/man9/atomic.9
+++ b/share/man/man9/atomic.9
@@ -23,7 +23,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd September 27, 2005
+.Dd August 20, 2013
 .Dt ATOMIC 9
 .Os
 .Sh NAME
@@ -62,6 +62,10 @@
 .Fn atomic_subtract_[acq_|rel_]<type> "volatile <type> *p" "<type> v"
 .Ft void
 .Fn atomic_store_rel_<type> "volatile <type> *p" "<type> v"
+.Ft <type>
+.Fn atomic_swap_<type> "volatile <type> *p" "<type> v"
+.Ft int
+.Fn atomic_testandset_<type> "volatile <type> *p" "u_int v"
 .Sh DESCRIPTION
 Each of the atomic operations is guaranteed to be atomic in the presence of
 interrupts.
@@ -184,9 +188,9 @@ This section describes the semantics of each operation using a C like notation.
 .Bd -literal -compact
 if (*dst == old) {
 	*dst = new;
-	return 1;
+	return (1);
 } else
-	return 0;
+	return (0);
 .Ed
 .El
 .Pp
@@ -203,7 +207,7 @@ and
 .Bd -literal -compact
 tmp = *p;
 *p += v;
-return tmp;
+return (tmp);
 .Ed
 .El
 .Pp
@@ -216,9 +220,9 @@ and
 .Dq Li 32
 and do not have any variants with memory barriers at this time.
 .Bl -hang
-.It Fn atomic_load addr
+.It Fn atomic_load p
 .Bd -literal -compact
-return (*addr)
+return (*p);
 .Ed
 .El
 .Pp
@@ -226,11 +230,11 @@ The
 .Fn atomic_load
 functions are only provided with acquire memory barriers.
 .Bl -hang
-.It Fn atomic_readandclear addr
+.It Fn atomic_readandclear p
 .Bd -literal -compact
-temp = *addr;
-*addr = 0;
-return (temp);
+tmp = *p;
+*p = 0;
+return (tmp);
 .Ed
 .El
 .Pp
@@ -243,8 +247,7 @@ functions are not implemented for the types
 .Dq Li 8 ,
 and
 .Dq Li 16
-and do
-not have any variants with memory barriers at this time.
+and do not have any variants with memory barriers at this time.
 .Bl -hang
 .It Fn atomic_set p v
 .Bd -literal -compact
@@ -264,6 +267,44 @@ The
 .Fn atomic_store
 functions are only provided with release memory barriers.
 .Pp
+.Bl -hang
+.It Fn atomic_swap p v
+.Bd -literal -compact
+tmp = *p;
+*p = v;
+return (tmp);
+.Ed
+.El
+.Pp
+The
+.Fn atomic_swap
+functions are not implemented for the types
+.Dq Li char ,
+.Dq Li short ,
+.Dq Li ptr ,
+.Dq Li 8 ,
+and
+.Dq Li 16
+and do not have any variants with memory barriers at this time.
+.Bl -hang
+.It Fn atomic_testandset p v
+.Bd -literal -compact
+bit = 1 << (v % (sizeof(*p) * NBBY));
+tmp = (*p & bit) != 0;
+*p |= bit;
+return (tmp);
+.Ed
+.El
+.Pp
+The
+.Fn atomic_testandset
+functions are only implemented for the types
+.Dq Li int ,
+.Dq Li long
+and
+.Dq Li 32
+and do not have any variants with memory barriers at this time.
+.Pp
 The type
 .Dq Li 64
 is currently not implemented for any of the atomic operations on the
@@ -275,15 +316,17 @@ architectures.
 .Sh RETURN VALUES
 The
 .Fn atomic_cmpset
-function
-returns the result of the compare operation.
+function returns the result of the compare operation.
 The
 .Fn atomic_fetchadd ,
 .Fn atomic_load ,
+.Fn atomic_readandclear ,
 and
-.Fn atomic_readandclear
-functions
-return the value at the specified address.
+.Fn atomic_swap
+functions return the value at the specified address.
+The
+.Fn atomic_testandset
+function returns the result of the test operation.
 .Sh EXAMPLES
 This example uses the
 .Fn atomic_cmpset_acq_ptr
@@ -354,3 +397,9 @@ The
 .Fn atomic_fetchadd
 operations were added in
 .Fx 6.0 .
+The
+.Fn atomic_swap
+and
+.Fn atomic_testandset
+operations were added in
+.Fx 10.0 .
diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h
index 7d8d839..4a15220 100644
--- a/sys/amd64/include/atomic.h
+++ b/sys/amd64/include/atomic.h
@@ -54,12 +54,14 @@
  * atomic_clear_int(P, V)	(*(u_int *)(P) &= ~(V))
  * atomic_add_int(P, V)		(*(u_int *)(P) += (V))
  * atomic_subtract_int(P, V)	(*(u_int *)(P) -= (V))
+ * atomic_swap_int(P, V)	(return (*(u_int *)(P)); *(u_int *)(P) = (V);)
  * atomic_readandclear_int(P)	(return (*(u_int *)(P)); *(u_int *)(P) = 0;)
  *
  * atomic_set_long(P, V)	(*(u_long *)(P) |= (V))
  * atomic_clear_long(P, V)	(*(u_long *)(P) &= ~(V))
  * atomic_add_long(P, V)	(*(u_long *)(P) += (V))
  * atomic_subtract_long(P, V)	(*(u_long *)(P) -= (V))
+ * atomic_swap_long(P, V)	(return (*(u_long *)(P)); *(u_long *)(P) = (V);)
  * atomic_readandclear_long(P)	(return (*(u_long *)(P)); *(u_long *)(P) = 0;)
  */
 
@@ -80,6 +82,8 @@ int	atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
 int	atomic_cmpset_long(volatile u_long *dst, u_long expect, u_long src);
 u_int	atomic_fetchadd_int(volatile u_int *p, u_int v);
 u_long	atomic_fetchadd_long(volatile u_long *p, u_long v);
+int	atomic_testandset_int(volatile u_int *p, u_int v);
+int	atomic_testandset_long(volatile u_long *p, u_int v);
 
 #define	ATOMIC_LOAD(TYPE, LOP)					\
 u_##TYPE	atomic_load_acq_##TYPE(volatile u_##TYPE *p)
@@ -205,6 +209,40 @@ atomic_fetchadd_long(volatile u_long *p, u_long v)
 	return (v);
 }
 
+static __inline int
+atomic_testandset_int(volatile u_int *p, u_int v)
+{
+	u_char res;
+
+	__asm __volatile(
+	"	" MPLOCKED "		"
+	"	btsl	%2,%1 ;		"
+	"	setc	%0 ;		"
+	"# atomic_testandset_int"
+	: "=q" (res),			/* 0 */
+	  "+m" (*p)			/* 1 */
+	: "Ir" (v & 0x1f)		/* 2 */
+	: "cc");
+	return (res);
+}
+
+static __inline int
+atomic_testandset_long(volatile u_long *p, u_int v)
+{
+	u_char res;
+
+	__asm __volatile(
+	"	" MPLOCKED "		"
+	"	btsq	%2,%1 ;		"
+	"	setc	%0 ;		"
+	"# atomic_testandset_long"
+	: "=q" (res),			/* 0 */
+	  "+m" (*p)			/* 1 */
+	: "Jr" ((u_long)(v & 0x3f))	/* 2 */
+	: "cc");
+	return (res);
+}
+
 /*
  * We assume that a = b will do atomic loads and stores.  Due to the
  * IA32 memory model, a simple store guarantees release semantics.
@@ -296,43 +334,39 @@ ATOMIC_STORE(long);
 
 #ifndef WANT_FUNCTIONS
 
-/* Read the current value and store a zero in the destination. */
+/* Read the current value and store a new value in the destination. */
 #ifdef __GNUCLIKE_ASM
 
 static __inline u_int
-atomic_readandclear_int(volatile u_int *p)
+atomic_swap_int(volatile u_int *p, u_int v)
 {
-	u_int res;
 
-	res = 0;
 	__asm __volatile(
 	"	xchgl	%1,%0 ;		"
-	"# atomic_readandclear_int"
-	: "+r" (res),			/* 0 */
+	"# atomic_swap_int"
+	: "+r" (v),			/* 0 */
 	  "+m" (*p));			/* 1 */
 
-	return (res);
+	return (v);
 }
 
 static __inline u_long
-atomic_readandclear_long(volatile u_long *p)
+atomic_swap_long(volatile u_long *p, u_long v)
 {
-	u_long res;
 
-	res = 0;
 	__asm __volatile(
 	"	xchgq	%1,%0 ;		"
-	"# atomic_readandclear_long"
-	: "+r" (res),			/* 0 */
+	"# atomic_swap_long"
+	: "+r" (v),			/* 0 */
 	  "+m" (*p));			/* 1 */
 
-	return (res);
+	return (v);
 }
 
 #else /* !__GNUCLIKE_ASM */
 
-u_int	atomic_readandclear_int(volatile u_int *p);
-u_long	atomic_readandclear_long(volatile u_long *p);
+u_int	atomic_swap_int(volatile u_int *p, u_int v);
+u_long	atomic_swap_long(volatile u_long *p, u_long v);
 
 #endif /* __GNUCLIKE_ASM */
 
@@ -376,6 +410,9 @@ u_long	atomic_readandclear_long(volatile u_long *p);
 #define	atomic_cmpset_acq_long		atomic_cmpset_long
 #define	atomic_cmpset_rel_long		atomic_cmpset_long
 
+#define	atomic_readandclear_int(p)	atomic_swap_int(p, 0)
+#define	atomic_readandclear_long(p)	atomic_swap_long(p, 0)
+
 /* Operations on 8-bit bytes. */
 #define	atomic_set_8		atomic_set_char
 #define	atomic_set_acq_8	atomic_set_acq_char
@@ -426,8 +463,10 @@ u_long	atomic_readandclear_long(volatile u_long *p);
 #define	atomic_cmpset_32	atomic_cmpset_int
 #define	atomic_cmpset_acq_32	atomic_cmpset_acq_int
 #define	atomic_cmpset_rel_32	atomic_cmpset_rel_int
+#define	atomic_swap_32		atomic_swap_int
 #define	atomic_readandclear_32	atomic_readandclear_int
 #define	atomic_fetchadd_32	atomic_fetchadd_int
+#define	atomic_testandset_32	atomic_testandset_int
 
 /* Operations on 64-bit quad words. */
 #define	atomic_set_64		atomic_set_long
@@ -447,7 +486,9 @@ u_long	atomic_readandclear_long(volatile u_long *p);
 #define	atomic_cmpset_64	atomic_cmpset_long
 #define	atomic_cmpset_acq_64	atomic_cmpset_acq_long
 #define	atomic_cmpset_rel_64	atomic_cmpset_rel_long
+#define	atomic_swap_64		atomic_swap_long
 #define	atomic_readandclear_64	atomic_readandclear_long
+#define	atomic_testandset_64	atomic_testandset_long
 
 /* Operations on pointers. */
 #define	atomic_set_ptr		atomic_set_long
@@ -467,6 +508,7 @@ u_long	atomic_readandclear_long(volatile u_long *p);
 #define	atomic_cmpset_ptr	atomic_cmpset_long
 #define	atomic_cmpset_acq_ptr	atomic_cmpset_acq_long
 #define	atomic_cmpset_rel_ptr	atomic_cmpset_rel_long
+#define	atomic_swap_ptr		atomic_swap_long
 #define	atomic_readandclear_ptr	atomic_readandclear_long
 
 #endif /* !WANT_FUNCTIONS */
diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h
index f782295..0d1738c 100644
--- a/sys/i386/include/atomic.h
+++ b/sys/i386/include/atomic.h
@@ -54,12 +54,14 @@
  * atomic_clear_int(P, V)	(*(u_int *)(P) &= ~(V))
  * atomic_add_int(P, V)		(*(u_int *)(P) += (V))
  * atomic_subtract_int(P, V)	(*(u_int *)(P) -= (V))
+ * atomic_swap_int(P, V)	(return (*(u_int *)(P)); *(u_int *)(P) = (V);)
  * atomic_readandclear_int(P)	(return (*(u_int *)(P)); *(u_int *)(P) = 0;)
  *
  * atomic_set_long(P, V)	(*(u_long *)(P) |= (V))
  * atomic_clear_long(P, V)	(*(u_long *)(P) &= ~(V))
  * atomic_add_long(P, V)	(*(u_long *)(P) += (V))
  * atomic_subtract_long(P, V)	(*(u_long *)(P) -= (V))
+ * atomic_swap_long(P, V)	(return (*(u_long *)(P)); *(u_long *)(P) = (V);)
  * atomic_readandclear_long(P)	(return (*(u_long *)(P)); *(u_long *)(P) = 0;)
  */
 
@@ -78,6 +80,7 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
 
 int	atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
 u_int	atomic_fetchadd_int(volatile u_int *p, u_int v);
+int	atomic_testandset_int(volatile u_int *p, u_int v);
 
 #define	ATOMIC_LOAD(TYPE, LOP)					\
 u_##TYPE	atomic_load_acq_##TYPE(volatile u_##TYPE *p)
@@ -275,6 +278,23 @@ atomic_fetchadd_int(volatile u_int *p, u_int v)
 	return (v);
 }
 
+static __inline int
+atomic_testandset_int(volatile u_int *p, u_int v)
+{
+	u_char res;
+
+	__asm __volatile(
+	"	" MPLOCKED "		"
+	"	btsl	%2,%1 ;		"
+	"	setc	%0 ;		"
+	"# atomic_testandset_int"
+	: "=q" (res),			/* 0 */
+	  "+m" (*p)			/* 1 */
+	: "Ir" (v & 0x1f)		/* 2 */
+	: "cc");
+	return (res);
+}
+
 /*
  * We assume that a = b will do atomic loads and stores.  Due to the
  * IA32 memory model, a simple store guarantees release semantics.
@@ -386,43 +406,40 @@ atomic_fetchadd_long(volatile u_long *p, u_long v)
 	return (atomic_fetchadd_int((volatile u_int *)p, (u_int)v));
 }
 
-/* Read the current value and store a zero in the destination. */
+static __inline int
+atomic_testandset_long(volatile u_long *p, u_int v)
+{
+
+	return (atomic_testandset_int((volatile u_int *)p, v));
+}
+
+/* Read the current value and store a new value in the destination. */
 #ifdef __GNUCLIKE_ASM
 
 static __inline u_int
-atomic_readandclear_int(volatile u_int *p)
+atomic_swap_int(volatile u_int *p, u_int v)
 {
-	u_int res;
 
-	res = 0;
 	__asm __volatile(
 	"	xchgl	%1,%0 ;		"
-	"# atomic_readandclear_int"
-	: "+r" (res),			/* 0 */
+	"# atomic_swap_int"
+	: "+r" (v),			/* 0 */
 	  "+m" (*p));			/* 1 */
 
-	return (res);
+	return (v);
 }
 
 static __inline u_long
-atomic_readandclear_long(volatile u_long *p)
+atomic_swap_long(volatile u_long *p, u_long v)
 {
-	u_long res;
 
-	res = 0;
-	__asm __volatile(
-	"	xchgl	%1,%0 ;		"
-	"# atomic_readandclear_long"
-	: "+r" (res),			/* 0 */
-	  "+m" (*p));			/* 1 */
-
-	return (res);
+	return (atomic_swap_int((volatile u_int *)p, (u_int)v));
 }
 
 #else /* !__GNUCLIKE_ASM */
 
-u_int	atomic_readandclear_int(volatile u_int *p);
-u_long	atomic_readandclear_long(volatile u_long *p);
+u_int	atomic_swap_int(volatile u_int *p, u_int v);
+u_long	atomic_swap_long(volatile u_long *p, u_long v);
 
 #endif /* __GNUCLIKE_ASM */
 
@@ -466,6 +483,9 @@ u_long	atomic_readandclear_long(volatile u_long *p);
 #define	atomic_cmpset_acq_long		atomic_cmpset_long
 #define	atomic_cmpset_rel_long		atomic_cmpset_long
 
+#define	atomic_readandclear_int(p)	atomic_swap_int(p, 0)
+#define	atomic_readandclear_long(p)	atomic_swap_long(p, 0)
+
 /* Operations on 8-bit bytes. */
 #define	atomic_set_8		atomic_set_char
 #define	atomic_set_acq_8	atomic_set_acq_char
@@ -516,8 +536,10 @@ u_long	atomic_readandclear_long(volatile u_long *p);
 #define	atomic_cmpset_32	atomic_cmpset_int
 #define	atomic_cmpset_acq_32	atomic_cmpset_acq_int
 #define	atomic_cmpset_rel_32	atomic_cmpset_rel_int
+#define	atomic_swap_32		atomic_swap_int
 #define	atomic_readandclear_32	atomic_readandclear_int
 #define	atomic_fetchadd_32	atomic_fetchadd_int
+#define	atomic_testandset_32	atomic_testandset_int
 
 /* Operations on pointers. */
 #define	atomic_set_ptr(p, v) \
@@ -556,6 +578,8 @@ u_long	atomic_readandclear_long(volatile u_long *p);
 #define	atomic_cmpset_rel_ptr(dst, old, new) \
 	atomic_cmpset_rel_int((volatile u_int *)(dst), (u_int)(old), \
 	    (u_int)(new))
+#define	atomic_swap_ptr(p, v) \
+	atomic_swap_int((volatile u_int *)(p), (u_int)(v))
 #define	atomic_readandclear_ptr(p) \
 	atomic_readandclear_int((volatile u_int *)(p))