Implement mwakeup, mwakeup_one, cv_signal_drop and cv_broadcast_drop.

These take an additional mutex argument, which is dropped before any
processes are made runnable.  This can avoid contention on the mutex
if the processes would immediately acquire it, and is done in such a
way that wakeups will not be lost.

Reviewed by:	jhb

4 files changed, 58 insertions, 6 deletions

diff --git a/sys/kern/kern_condvar.c b/sys/kern/kern_condvar.c
index 07ed3b7..cf7dbff 100644
--- a/sys/kern/kern_condvar.c
+++ b/sys/kern/kern_condvar.c
@@ -492,6 +492,26 @@ cv_signal(struct cv *cvp)
 }
 
 /*
+ * Signal a condition variable, dropping the passed in mutex before waking
+ * a waiting process.
+ */
+void
+cv_signal_drop(struct cv *cvp, struct mtx *mp)
+{
+
+	KASSERT(cvp != NULL, ("%s: cvp NULL", __FUNCTION__));
+	KASSERT(mp != NULL, ("%s: mp NULL", __FUNCTION__));
+	mtx_assert(mp, MA_OWNED | MA_NOTRECURSED);
+	mtx_lock_spin(&sched_lock);
+	mtx_unlock_flags(mp, MTX_NOSWITCH);
+	if (!TAILQ_EMPTY(&cvp->cv_waitq)) {
+		CV_SIGNAL_VALIDATE(cvp);
+		cv_wakeup(cvp);
+	}
+	mtx_unlock_spin(&sched_lock);
+}
+
+/*
  * Broadcast a signal to a condition variable.  Wakes up all waiting processes.
  * Should be called with the same mutex as was passed to cv_wait held.
  */
@@ -508,6 +528,25 @@ cv_broadcast(struct cv *cvp)
 }
 
 /*
+ * Broadcast a signal to a condition variable, dropping the passed in mutex
+ * before waking any waiting processes.
+ */
+void
+cv_broadcast_drop(struct cv *cvp, struct mtx *mp)
+{
+
+	KASSERT(cvp != NULL, ("%s: cvp NULL", __FUNCTION__));
+	KASSERT(mp != NULL, ("%s: mp NULL", __FUNCTION__));
+	mtx_assert(mp, MA_OWNED | MA_NOTRECURSED);
+	mtx_lock_spin(&sched_lock);
+	mtx_unlock_flags(mp, MTX_NOSWITCH);
+	CV_SIGNAL_VALIDATE(cvp);
+	while (!TAILQ_EMPTY(&cvp->cv_waitq))
+		cv_wakeup(cvp);
+	mtx_unlock_spin(&sched_lock);
+}
+
+/*
  * Remove a process from the wait queue of its condition variable.  This may be
  * called externally.
  */
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 530b67e..7d55f67 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -709,16 +709,21 @@ unsleep(p)
 }
 
 /*
- * Make all processes sleeping on the specified identifier runnable.
+ * Make all processes sleeping on the specified identifier runnable.  If
+ * non-NULL, the specified mutex is dropped before any processes are made
+ * runnable.
  */
 void
-wakeup(ident)
+mwakeup(ident, mtx)
 	register void *ident;
+	register struct mtx *mtx;
 {
 	register struct slpquehead *qp;
 	register struct proc *p;
 
 	mtx_lock_spin(&sched_lock);
+	if (mtx != NULL)
+		mtx_unlock_flags(mtx, MTX_NOSWITCH);
 	qp = &slpque[LOOKUP(ident)];
 restart:
 	TAILQ_FOREACH(p, qp, p_slpq) {
@@ -751,16 +756,20 @@ restart:
 /*
  * Make a process sleeping on the specified identifier runnable.
  * May wake more than one process if a target process is currently
- * swapped out.
+ * swapped out.  If non-NULL, the specified mutex is dropped before
+ * a process is made runnable.
  */
 void
-wakeup_one(ident)
+mwakeup_one(ident, mtx)
 	register void *ident;
+	register struct mtx *mtx;
 {
 	register struct slpquehead *qp;
 	register struct proc *p;
 
 	mtx_lock_spin(&sched_lock);
+	if (mtx != NULL)
+		mtx_unlock_flags(mtx, MTX_NOSWITCH);
 	qp = &slpque[LOOKUP(ident)];
 
 	TAILQ_FOREACH(p, qp, p_slpq) {
diff --git a/sys/sys/condvar.h b/sys/sys/condvar.h
index 5460024..903d635 100644
--- a/sys/sys/condvar.h
+++ b/sys/sys/condvar.h
@@ -59,7 +59,9 @@ int	cv_timedwait(struct cv *cvp, struct mtx *mp, int timo);
 int	cv_timedwait_sig(struct cv *cvp, struct mtx *mp, int timo);
 
 void	cv_signal(struct cv *cvp);
+void	cv_signal_drop(struct cv *cvp, struct mtx *mp);
 void	cv_broadcast(struct cv *cvp);
+void	cv_broadcast_drop(struct cv *cvp, struct mtx *mp);
 
 void	cv_waitq_remove(struct proc *p);
 
diff --git a/sys/sys/systm.h b/sys/sys/systm.h
index aa95807..c238f3f 100644
--- a/sys/sys/systm.h
+++ b/sys/sys/systm.h
@@ -259,8 +259,10 @@ int	msleep __P((void *chan, struct mtx *mtx, int pri, const char *wmesg,
 int	asleep __P((void *chan, int pri, const char *wmesg, int timo));
 #define await(pri, timo)		mawait(NULL, pri, timo)
 int	mawait  __P((struct mtx *mtx, int pri, int timo));
-void	wakeup __P((void *chan));
-void	wakeup_one __P((void *chan));
+void	mwakeup __P((void *chan, struct mtx *mtx));
+#define	wakeup(chan)			mwakeup(chan, NULL)
+void	mwakeup_one __P((void *chan, struct mtx *mtx));
+#define	wakeup_one(chan)		mwakeup_one(chan, NULL)
 
 /*
  * Common `dev_t' stuff are declared here to avoid #include poisoning