MFC r313275,r313280,r313282,r313335:

mtx: move lockstat handling out of inline primitives

Lockstat requires checking if it is enabled and if so, calling a 6 argument
function. Further, determining whether to call it on unlock requires
pre-reading the lock value.

This is problematic in at least 3 ways:
- more branches in the hot path than necessary
- additional cacheline ping pong under contention
- bigger code

Instead, check first if lockstat handling is necessary and if so, just fall
back to regular locking routines. For this purpose a new macro is introduced
(LOCKSTAT_PROFILE_ENABLED).

LOCK_PROFILING uninlines all primitives. Fold in the current inline lock
variant into the _mtx_lock_flags to retain the support. With this change
the inline variants are not used when LOCK_PROFILING is defined and thus
can ignore its existence.

This results in:
text        data     bss     dec     hex filename
22259667    1303208 4994976 28557851        1b3c21b kernel.orig
21797315    1303208 4994976 28095499        1acb40b kernel.patched

i.e. about 3% reduction in text size.

A remaining action is to remove spurious arguments for internal kernel
consumers.

==

sx: move lockstat handling out of inline primitives

See r313275 for details.

==

rwlock: move lockstat handling out of inline primitives

See r313275 for details.

One difference here is that recursion handling was removed from the fallback
routine. As it is it was never supposed to see a recursed lock in the first
place. Future changes will move it out of inline variants, but right now
there is no easy to way to test if the lock is recursed without reading
additional words.

==

locks: fix recursion support after recent changes

When a relevant lockstat probe is enabled the fallback primitive is called with
a constant signifying a free lock. This works fine for typical cases but breaks
with recursion, since it checks if the passed value is that of the executing
thread.

Read the value if necessary.

5 files changed, 33 insertions, 35 deletions

diff --git a/sys/sys/lockstat.h b/sys/sys/lockstat.h
index e550353..958e950 100644
--- a/sys/sys/lockstat.h
+++ b/sys/sys/lockstat.h
@@ -107,6 +107,10 @@ extern int lockstat_enabled;
 	LOCKSTAT_RECORD1(probe, lp, a);					\
 } while (0)
 
+#ifndef LOCK_PROFILING
+#define	LOCKSTAT_PROFILE_ENABLED(probe)	SDT_PROBE_ENABLED(lockstat, , , probe)
+#endif
+
 struct lock_object;
 uint64_t lockstat_nsecs(struct lock_object *);
 
@@ -130,6 +134,10 @@ uint64_t lockstat_nsecs(struct lock_object *);
 #define	LOCKSTAT_PROFILE_RELEASE_RWLOCK(probe, lp, a)  			\
 	LOCKSTAT_PROFILE_RELEASE_LOCK(probe, lp)
 
+#ifndef LOCK_PROFILING
+#define	LOCKSTAT_PROFILE_ENABLED(probe)	0
+#endif
+
 #endif /* !KDTRACE_HOOKS */
 #endif /* _KERNEL */
 #endif /* _SYS_LOCKSTAT_H */
diff --git a/sys/sys/mutex.h b/sys/sys/mutex.h
index 841bd76..42f26fd 100644
--- a/sys/sys/mutex.h
+++ b/sys/sys/mutex.h
@@ -171,10 +171,8 @@ void	thread_lock_flags_(struct thread *, int, const char *, int);
 #define _mtx_obtain_lock(mp, tid)					\
 	atomic_cmpset_acq_ptr(&(mp)->mtx_lock, MTX_UNOWNED, (tid))
 
-#define _mtx_obtain_lock_fetch(mp, vp, tid) ({				\
-	*vp = MTX_UNOWNED;						\
-	atomic_fcmpset_acq_ptr(&(mp)->mtx_lock, vp, (tid));		\
-})
+#define _mtx_obtain_lock_fetch(mp, vp, tid)				\
+	atomic_fcmpset_acq_ptr(&(mp)->mtx_lock, vp, (tid))
 
 /* Try to release mtx_lock if it is unrecursed and uncontested. */
 #define _mtx_release_lock(mp, tid)					\
@@ -193,13 +191,11 @@ void	thread_lock_flags_(struct thread *, int, const char *, int);
 /* Lock a normal mutex. */
 #define __mtx_lock(mp, tid, opts, file, line) do {			\
 	uintptr_t _tid = (uintptr_t)(tid);				\
-	uintptr_t _v;							\
+	uintptr_t _v = MTX_UNOWNED;					\
 									\
-	if (!_mtx_obtain_lock_fetch((mp), &_v, _tid))			\
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__acquire) ||\
+	    !_mtx_obtain_lock_fetch((mp), &_v, _tid)))			\
 		_mtx_lock_sleep((mp), _v, _tid, (opts), (file), (line));\
-	else								\
-		LOCKSTAT_PROFILE_OBTAIN_LOCK_SUCCESS(adaptive__acquire,	\
-		    mp, 0, 0, file, line);				\
 } while (0)
 
 /*
@@ -211,7 +207,7 @@ void	thread_lock_flags_(struct thread *, int, const char *, int);
 #ifdef SMP
 #define __mtx_lock_spin(mp, tid, opts, file, line) do {			\
 	uintptr_t _tid = (uintptr_t)(tid);				\
-	uintptr_t _v;							\
+	uintptr_t _v = MTX_UNOWNED;					\
 									\
 	spinlock_enter();						\
 	if (!_mtx_obtain_lock_fetch((mp), &_v, _tid)) 			\
@@ -267,9 +263,8 @@ void	thread_lock_flags_(struct thread *, int, const char *, int);
 #define __mtx_unlock(mp, tid, opts, file, line) do {			\
 	uintptr_t _tid = (uintptr_t)(tid);				\
 									\
-	if ((mp)->mtx_recurse == 0)					\
-		LOCKSTAT_PROFILE_RELEASE_LOCK(adaptive__release, mp);	\
-	if (!_mtx_release_lock((mp), _tid))				\
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(adaptive__release) ||\
+	    !_mtx_release_lock((mp), _tid)))				\
 		_mtx_unlock_sleep((mp), (opts), (file), (line));	\
 } while (0)
 
diff --git a/sys/sys/rwlock.h b/sys/sys/rwlock.h
index 816037e..d6dd4f0 100644
--- a/sys/sys/rwlock.h
+++ b/sys/sys/rwlock.h
@@ -84,10 +84,8 @@
 #define	_rw_write_lock(rw, tid)						\
 	atomic_cmpset_acq_ptr(&(rw)->rw_lock, RW_UNLOCKED, (tid))
 
-#define	_rw_write_lock_fetch(rw, vp, tid) ({				\
-	*vp = RW_UNLOCKED;						\
-	atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid));		\
-})
+#define	_rw_write_lock_fetch(rw, vp, tid)				\
+	atomic_fcmpset_acq_ptr(&(rw)->rw_lock, vp, (tid))
 
 /* Release a write lock quickly if there are no waiters. */
 #define	_rw_write_unlock(rw, tid)					\
@@ -102,13 +100,11 @@
 /* Acquire a write lock. */
 #define	__rw_wlock(rw, tid, file, line) do {				\
 	uintptr_t _tid = (uintptr_t)(tid);				\
-	uintptr_t _v;							\
+	uintptr_t _v = RW_UNLOCKED;					\
 									\
-	if (!_rw_write_lock_fetch((rw), &_v, _tid))			\
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__acquire) ||	\
+	    !_rw_write_lock_fetch((rw), &_v, _tid)))			\
 		_rw_wlock_hard((rw), _v, _tid, (file), (line));		\
-	else 								\
-		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(rw__acquire, rw,	\
-		    0, 0, file, line, LOCKSTAT_WRITER);			\
 } while (0)
 
 /* Release a write lock. */
@@ -118,9 +114,8 @@
 	if ((rw)->rw_recurse)						\
 		(rw)->rw_recurse--;					\
 	else {								\
-		LOCKSTAT_PROFILE_RELEASE_RWLOCK(rw__release, rw,	\
-		    LOCKSTAT_WRITER);					\
-		if (!_rw_write_unlock((rw), _tid))			\
+		if (__predict_false(LOCKSTAT_PROFILE_ENABLED(rw__release) ||\
+		    !_rw_write_unlock((rw), _tid)))			\
 			_rw_wunlock_hard((rw), _tid, (file), (line));	\
 	}								\
 } while (0)
diff --git a/sys/sys/sdt.h b/sys/sys/sdt.h
index 25423d7..4259883 100644
--- a/sys/sys/sdt.h
+++ b/sys/sys/sdt.h
@@ -160,6 +160,9 @@ SET_DECLARE(sdt_argtypes_set, struct sdt_argtype);
 #define SDT_PROBE_DECLARE(prov, mod, func, name)				\
 	extern struct sdt_probe sdt_##prov##_##mod##_##func##_##name[1]
 
+#define SDT_PROBE_ENABLED(prov, mod, func, name)				\
+	__predict_false((sdt_##prov##_##mod##_##func##_##name->id))
+
 #define SDT_PROBE(prov, mod, func, name, arg0, arg1, arg2, arg3, arg4)	do {	\
 	if (__predict_false(sdt_##prov##_##mod##_##func##_##name->id))		\
 		(*sdt_probe_func)(sdt_##prov##_##mod##_##func##_##name->id,	\
diff --git a/sys/sys/sx.h b/sys/sys/sx.h
index a676d2a..50b0a24 100644
--- a/sys/sys/sx.h
+++ b/sys/sys/sx.h
@@ -145,21 +145,19 @@ struct sx_args {
  * deferred to 'tougher' functions.
  */
 
+#if	(LOCK_DEBUG == 0) && !defined(SX_NOINLINE)
 /* Acquire an exclusive lock. */
 static __inline int
 __sx_xlock(struct sx *sx, struct thread *td, int opts, const char *file,
     int line)
 {
 	uintptr_t tid = (uintptr_t)td;
-	uintptr_t v;
+	uintptr_t v = SX_LOCK_UNLOCKED;
 	int error = 0;
 
-	v = SX_LOCK_UNLOCKED;
-	if (!atomic_fcmpset_acq_ptr(&sx->sx_lock, &v, tid))
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__acquire) ||
+	    !atomic_fcmpset_acq_ptr(&sx->sx_lock, &v, tid)))
 		error = _sx_xlock_hard(sx, v, tid, opts, file, line);
-	else 
-		LOCKSTAT_PROFILE_OBTAIN_RWLOCK_SUCCESS(sx__acquire, sx,
-		    0, 0, file, line, LOCKSTAT_WRITER);
 
 	return (error);
 }
@@ -170,12 +168,11 @@ __sx_xunlock(struct sx *sx, struct thread *td, const char *file, int line)
 {
 	uintptr_t tid = (uintptr_t)td;
 
-	if (sx->sx_recurse == 0)
-		LOCKSTAT_PROFILE_RELEASE_RWLOCK(sx__release, sx,
-		    LOCKSTAT_WRITER);
-	if (!atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED))
+	if (__predict_false(LOCKSTAT_PROFILE_ENABLED(sx__release) ||
+	    !atomic_cmpset_rel_ptr(&sx->sx_lock, tid, SX_LOCK_UNLOCKED)))
 		_sx_xunlock_hard(sx, tid, file, line);
 }
+#endif
 
 /*
  * Public interface for lock operations.