atomic: Add testandclear on i386/amd64

Reviewed by:	kib
Sponsored by:	Microsoft OSTC
Differential Revision:	https://reviews.freebsd.org/D6381

3 files changed, 83 insertions, 1 deletions

diff --git a/share/man/man9/atomic.9 b/share/man/man9/atomic.9
index cee30e6..e085a78 100644
--- a/share/man/man9/atomic.9
+++ b/share/man/man9/atomic.9
@@ -23,7 +23,7 @@
 .\"
 .\" $FreeBSD$
 .\"
-.Dd August 14, 2015
+.Dd May 12, 2016
 .Dt ATOMIC 9
 .Os
 .Sh NAME
@@ -65,6 +65,8 @@
 .Ft <type>
 .Fn atomic_swap_<type> "volatile <type> *p" "<type> v"
 .Ft int
+.Fn atomic_testandclear_<type> "volatile <type> *p" "u_int v"
+.Ft int
 .Fn atomic_testandset_<type> "volatile <type> *p" "u_int v"
 .Sh DESCRIPTION
 Each of the atomic operations is guaranteed to be atomic across multiple
@@ -309,6 +311,15 @@ and
 .Dq Li 16
 and do not have any variants with memory barriers at this time.
 .Bl -hang
+.It Fn atomic_testandclear p v
+.Bd -literal -compact
+bit = 1 << (v % (sizeof(*p) * NBBY));
+tmp = (*p & bit) != 0;
+*p &= ~bit;
+return (tmp);
+.Ed
+.El
+.Bl -hang
 .It Fn atomic_testandset p v
 .Bd -literal -compact
 bit = 1 << (v % (sizeof(*p) * NBBY));
@@ -320,6 +331,8 @@ return (tmp);
 .Pp
 The
 .Fn atomic_testandset
+and
+.Fn atomic_testandclear
 functions are only implemented for the types
 .Dq Li int ,
 .Dq Li long
@@ -348,6 +361,8 @@ and
 functions return the value at the specified address.
 The
 .Fn atomic_testandset
+and
+.Fn atomic_testandclear
 function returns the result of the test operation.
 .Sh EXAMPLES
 This example uses the
@@ -425,3 +440,6 @@ and
 .Fn atomic_testandset
 operations were added in
 .Fx 10.0 .
+.Fn atomic_testandclear
+operation was added in
+.Fx 11.0 .
diff --git a/sys/amd64/include/atomic.h b/sys/amd64/include/atomic.h
index dd71e92..c9526f4 100644
--- a/sys/amd64/include/atomic.h
+++ b/sys/amd64/include/atomic.h
@@ -103,6 +103,8 @@ u_int	atomic_fetchadd_int(volatile u_int *p, u_int v);
 u_long	atomic_fetchadd_long(volatile u_long *p, u_long v);
 int	atomic_testandset_int(volatile u_int *p, u_int v);
 int	atomic_testandset_long(volatile u_long *p, u_int v);
+int	atomic_testandclear_int(volatile u_int *p, u_int v);
+int	atomic_testandclear_long(volatile u_long *p, u_int v);
 void	atomic_thread_fence_acq(void);
 void	atomic_thread_fence_acq_rel(void);
 void	atomic_thread_fence_rel(void);
@@ -264,6 +266,40 @@ atomic_testandset_long(volatile u_long *p, u_int v)
 	return (res);
 }
 
+static __inline int
+atomic_testandclear_int(volatile u_int *p, u_int v)
+{
+	u_char res;
+
+	__asm __volatile(
+	"	" MPLOCKED "		"
+	"	btrl	%2,%1 ;		"
+	"	setc	%0 ;		"
+	"# atomic_testandclear_int"
+	: "=q" (res),			/* 0 */
+	  "+m" (*p)			/* 1 */
+	: "Ir" (v & 0x1f)		/* 2 */
+	: "cc");
+	return (res);
+}
+
+static __inline int
+atomic_testandclear_long(volatile u_long *p, u_int v)
+{
+	u_char res;
+
+	__asm __volatile(
+	"	" MPLOCKED "		"
+	"	btrq	%2,%1 ;		"
+	"	setc	%0 ;		"
+	"# atomic_testandclear_long"
+	: "=q" (res),			/* 0 */
+	  "+m" (*p)			/* 1 */
+	: "Jr" ((u_long)(v & 0x3f))	/* 2 */
+	: "cc");
+	return (res);
+}
+
 /*
  * We assume that a = b will do atomic loads and stores.  Due to the
  * IA32 memory model, a simple store guarantees release semantics.
@@ -537,6 +573,7 @@ u_long	atomic_swap_long(volatile u_long *p, u_long v);
 #define	atomic_readandclear_32	atomic_readandclear_int
 #define	atomic_fetchadd_32	atomic_fetchadd_int
 #define	atomic_testandset_32	atomic_testandset_int
+#define	atomic_testandclear_32	atomic_testandclear_int
 
 /* Operations on 64-bit quad words. */
 #define	atomic_set_64		atomic_set_long
@@ -560,6 +597,7 @@ u_long	atomic_swap_long(volatile u_long *p, u_long v);
 #define	atomic_readandclear_64	atomic_readandclear_long
 #define	atomic_fetchadd_64	atomic_fetchadd_long
 #define	atomic_testandset_64	atomic_testandset_long
+#define	atomic_testandclear_64	atomic_testandclear_long
 
 /* Operations on pointers. */
 #define	atomic_set_ptr		atomic_set_long
diff --git a/sys/i386/include/atomic.h b/sys/i386/include/atomic.h
index 3242d76..90fb950 100644
--- a/sys/i386/include/atomic.h
+++ b/sys/i386/include/atomic.h
@@ -108,6 +108,7 @@ void atomic_##NAME##_barr_##TYPE(volatile u_##TYPE *p, u_##TYPE v)
 int	atomic_cmpset_int(volatile u_int *dst, u_int expect, u_int src);
 u_int	atomic_fetchadd_int(volatile u_int *p, u_int v);
 int	atomic_testandset_int(volatile u_int *p, u_int v);
+int	atomic_testandclear_int(volatile u_int *p, u_int v);
 void	atomic_thread_fence_acq(void);
 void	atomic_thread_fence_acq_rel(void);
 void	atomic_thread_fence_rel(void);
@@ -250,6 +251,23 @@ atomic_testandset_int(volatile u_int *p, u_int v)
 	return (res);
 }
 
+static __inline int
+atomic_testandclear_int(volatile u_int *p, u_int v)
+{
+	u_char res;
+
+	__asm __volatile(
+	"	" MPLOCKED "		"
+	"	btrl	%2,%1 ;		"
+	"	setc	%0 ;		"
+	"# atomic_testandclear_int"
+	: "=q" (res),			/* 0 */
+	  "+m" (*p)			/* 1 */
+	: "Ir" (v & 0x1f)		/* 2 */
+	: "cc");
+	return (res);
+}
+
 /*
  * We assume that a = b will do atomic loads and stores.  Due to the
  * IA32 memory model, a simple store guarantees release semantics.
@@ -602,6 +620,13 @@ atomic_testandset_long(volatile u_long *p, u_int v)
 	return (atomic_testandset_int((volatile u_int *)p, v));
 }
 
+static __inline int
+atomic_testandclear_long(volatile u_long *p, u_int v)
+{
+
+	return (atomic_testandclear_int((volatile u_int *)p, v));
+}
+
 /* Read the current value and store a new value in the destination. */
 #ifdef __GNUCLIKE_ASM
 
@@ -728,6 +753,7 @@ u_long	atomic_swap_long(volatile u_long *p, u_long v);
 #define	atomic_readandclear_32	atomic_readandclear_int
 #define	atomic_fetchadd_32	atomic_fetchadd_int
 #define	atomic_testandset_32	atomic_testandset_int
+#define	atomic_testandclear_32	atomic_testandclear_int
 
 /* Operations on pointers. */
 #define	atomic_set_ptr(p, v) \