Add second sleep queue so that sx and lockmgr can have separate sleep

queues for shared and exclusive acquisitions

Submitted by: Attilio Rao
Approved by: jhb

5 files changed, 71 insertions, 45 deletions

diff --git a/sys/kern/kern_condvar.c b/sys/kern/kern_condvar.c
index 85a52b1..e7410e6 100644
--- a/sys/kern/kern_condvar.c
+++ b/sys/kern/kern_condvar.c
@@ -149,7 +149,8 @@ cv_wait_unlock(struct cv *cvp, struct mtx *mp)
 	DROP_GIANT();
 	mtx_unlock(mp);
 
-	sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR);
+	sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR, 
+		   0);
 	sleepq_wait(cvp);
 
 	PICKUP_GIANT();
@@ -197,7 +198,7 @@ cv_wait_sig(struct cv *cvp, struct mtx *mp)
 	mtx_unlock(mp);
 
 	sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR |
-	    SLEEPQ_INTERRUPTIBLE);
+	    SLEEPQ_INTERRUPTIBLE, 0);
 	rval = sleepq_wait_sig(cvp);
 
 #ifdef KTRACE
@@ -250,7 +251,8 @@ cv_timedwait(struct cv *cvp, struct mtx *mp, int timo)
 	DROP_GIANT();
 	mtx_unlock(mp);
 
-	sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR);
+	sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR, 
+		   0);
 	sleepq_set_timeout(cvp, timo);
 	rval = sleepq_timedwait(cvp);
 
@@ -308,7 +310,7 @@ cv_timedwait_sig(struct cv *cvp, struct mtx *mp, int timo)
 	mtx_unlock(mp);
 
 	sleepq_add(cvp, &mp->mtx_object, cvp->cv_description, SLEEPQ_CONDVAR |
-	    SLEEPQ_INTERRUPTIBLE);
+	    SLEEPQ_INTERRUPTIBLE, 0);
 	sleepq_set_timeout(cvp, timo);
 	rval = sleepq_timedwait_sig(cvp);
 
@@ -337,7 +339,7 @@ cv_signal(struct cv *cvp)
 	sleepq_lock(cvp);
 	if (cvp->cv_waiters > 0) {
 		cvp->cv_waiters--;
-		sleepq_signal(cvp, SLEEPQ_CONDVAR, -1);
+		sleepq_signal(cvp, SLEEPQ_CONDVAR, -1, 0);
 	} else
 		sleepq_release(cvp);
 }
@@ -353,7 +355,7 @@ cv_broadcastpri(struct cv *cvp, int pri)
 	sleepq_lock(cvp);
 	if (cvp->cv_waiters > 0) {
 		cvp->cv_waiters = 0;
-		sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri);
+		sleepq_broadcast(cvp, SLEEPQ_CONDVAR, pri, 0);
 	} else
 		sleepq_release(cvp);
 }
diff --git a/sys/kern/kern_synch.c b/sys/kern/kern_synch.c
index 818d7e3..27fd835 100644
--- a/sys/kern/kern_synch.c
+++ b/sys/kern/kern_synch.c
@@ -189,7 +189,7 @@ msleep(ident, mtx, priority, wmesg, timo)
 	 * return from cursig().
 	 */
 	sleepq_add(ident, ident == &lbolt ? NULL : &mtx->mtx_object, wmesg,
-	    flags);
+	    flags, 0);
 	if (timo)
 		sleepq_set_timeout(ident, timo);
 
@@ -267,7 +267,7 @@ msleep_spin(ident, mtx, wmesg, timo)
 	/*
 	 * We put ourselves on the sleep queue and start our timeout.
 	 */
-	sleepq_add(ident, &mtx->mtx_object, wmesg, SLEEPQ_MSLEEP);
+	sleepq_add(ident, &mtx->mtx_object, wmesg, SLEEPQ_MSLEEP, 0);
 	if (timo)
 		sleepq_set_timeout(ident, timo);
 
@@ -316,7 +316,7 @@ wakeup(ident)
 {
 
 	sleepq_lock(ident);
-	sleepq_broadcast(ident, SLEEPQ_MSLEEP, -1);
+	sleepq_broadcast(ident, SLEEPQ_MSLEEP, -1, 0);
 }
 
 /*
@@ -330,7 +330,7 @@ wakeup_one(ident)
 {
 
 	sleepq_lock(ident);
-	sleepq_signal(ident, SLEEPQ_MSLEEP, -1);
+	sleepq_signal(ident, SLEEPQ_MSLEEP, -1, 0);
 }
 
 /*
diff --git a/sys/kern/subr_sleepqueue.c b/sys/kern/subr_sleepqueue.c
index 2d79c99..0333665 100644
--- a/sys/kern/subr_sleepqueue.c
+++ b/sys/kern/subr_sleepqueue.c
@@ -94,7 +94,7 @@ __FBSDID("$FreeBSD$");
 #define	SC_SHIFT	8
 #define	SC_HASH(wc)	(((uintptr_t)(wc) >> SC_SHIFT) & SC_MASK)
 #define	SC_LOOKUP(wc)	&sleepq_chains[SC_HASH(wc)]
-
+#define NR_SLEEPQS      2
 /*
  * There two different lists of sleep queues.  Both lists are connected
  * via the sq_hash entries.  The first list is the sleep queue chain list
@@ -114,7 +114,7 @@ __FBSDID("$FreeBSD$");
  *  c - sleep queue chain lock
  */
 struct sleepqueue {
-	TAILQ_HEAD(, thread) sq_blocked;	/* (c) Blocked threads. */
+	TAILQ_HEAD(, thread) sq_blocked[NR_SLEEPQS];	/* (c) Blocked threads. */
 	LIST_ENTRY(sleepqueue) sq_hash;		/* (c) Chain and free list. */
 	LIST_HEAD(, sleepqueue) sq_free;	/* (c) Free queues. */
 	void	*sq_wchan;			/* (c) Wait channel. */
@@ -148,6 +148,8 @@ static MALLOC_DEFINE(M_SLEEPQUEUE, "sleepqueue", "sleep queues");
 /*
  * Prototypes for non-exported routines.
  */
+static int	sleepq_catch_signals(void *wchan);
+static int	sleepq_check_signals(void);
 static int	sleepq_check_timeout(void);
 static void	sleepq_switch(void *wchan);
 static void	sleepq_timeout(void *arg);
@@ -192,9 +194,11 @@ struct sleepqueue *
 sleepq_alloc(void)
 {
 	struct sleepqueue *sq;
+	int i;
 
 	sq = malloc(sizeof(struct sleepqueue), M_SLEEPQUEUE, M_WAITOK | M_ZERO);
-	TAILQ_INIT(&sq->sq_blocked);
+	for (i = 0; i < NR_SLEEPQS; i++)
+		TAILQ_INIT(&sq->sq_blocked[i]);
 	LIST_INIT(&sq->sq_free);
 	return (sq);
 }
@@ -205,9 +209,11 @@ sleepq_alloc(void)
 void
 sleepq_free(struct sleepqueue *sq)
 {
+	int i;
 
 	MPASS(sq != NULL);
-	MPASS(TAILQ_EMPTY(&sq->sq_blocked));
+	for (i = 0; i < NR_SLEEPQS; i++)
+		MPASS(TAILQ_EMPTY(&sq->sq_blocked[i]));
 	free(sq, M_SLEEPQUEUE);
 }
 
@@ -262,7 +268,8 @@ sleepq_release(void *wchan)
  * woken up.
  */
 void
-sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
+sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags,
+    int queue)
 {
 	struct sleepqueue_chain *sc;
 	struct sleepqueue *sq;
@@ -273,6 +280,7 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
 	mtx_assert(&sc->sc_lock, MA_OWNED);
 	MPASS(td->td_sleepqueue != NULL);
 	MPASS(wchan != NULL);
+	MPASS((queue >= 0) && (queue < NR_SLEEPQS));
 
 	/* If this thread is not allowed to sleep, die a horrible death. */
 	KASSERT(!(td->td_pflags & TDP_NOSLEEPING),
@@ -287,6 +295,16 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
 	 * into the sleep queue already in use by this wait channel.
 	 */
 	if (sq == NULL) {
+#ifdef INVARIANTS
+		int i;
+		for (i = 0; i < NR_SLEEP_QUEUEUS; i++)
+			KASSERT(TAILQ_EMPTY(&sq->sq_blocked[i]),
+				("thread's sleep queue %d is not empty", i));
+	
+		KASSERT(LIST_EMPTY(&sq->sq_free),
+		    ("thread's sleep queue has a non-empty free list"));
+		KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
+#endif
 #ifdef SLEEPQUEUE_PROFILING
 		sc->sc_depth++;
 		if (sc->sc_depth > sc->sc_max_depth) {
@@ -297,11 +315,6 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
 #endif
 		sq = td->td_sleepqueue;
 		LIST_INSERT_HEAD(&sc->sc_queues, sq, sq_hash);
-		KASSERT(TAILQ_EMPTY(&sq->sq_blocked),
-		    ("thread's sleep queue has a non-empty queue"));
-		KASSERT(LIST_EMPTY(&sq->sq_free),
-		    ("thread's sleep queue has a non-empty free list"));
-		KASSERT(sq->sq_wchan == NULL, ("stale sq_wchan pointer"));
 		sq->sq_wchan = wchan;
 #ifdef INVARIANTS
 		sq->sq_lock = lock;
@@ -313,9 +326,10 @@ sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg, int flags)
 		MPASS((flags & SLEEPQ_TYPE) == sq->sq_type);
 		LIST_INSERT_HEAD(&sq->sq_free, td->td_sleepqueue, sq_hash);
 	}
-	TAILQ_INSERT_TAIL(&sq->sq_blocked, td, td_slpq);
+	TAILQ_INSERT_TAIL(&sq->sq_blocked[queue], td, td_slpq);
 	td->td_sleepqueue = NULL;
 	mtx_lock_spin(&sched_lock);
+	td->td_sqqueue = queue;
 	td->td_wchan = wchan;
 	td->td_wmesg = wmesg;
 	if (flags & SLEEPQ_INTERRUPTIBLE) {
@@ -606,12 +620,13 @@ sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri)
 	MPASS(td != NULL);
 	MPASS(sq->sq_wchan != NULL);
 	MPASS(td->td_wchan == sq->sq_wchan);
+	MPASS(td->td_sqqueue < NR_SLEEPQS && td->td_sqqueue >= 0);
 	sc = SC_LOOKUP(sq->sq_wchan);
 	mtx_assert(&sc->sc_lock, MA_OWNED);
 	mtx_assert(&sched_lock, MA_OWNED);
 
 	/* Remove the thread from the queue. */
-	TAILQ_REMOVE(&sq->sq_blocked, td, td_slpq);
+	TAILQ_REMOVE(&sq->sq_blocked[td->td_sqqueue], td, td_slpq);
 
 	/*
 	 * Get a sleep queue for this thread.  If this is the last waiter,
@@ -656,13 +671,14 @@ sleepq_resume_thread(struct sleepqueue *sq, struct thread *td, int pri)
  * Find the highest priority thread sleeping on a wait channel and resume it.
  */
 void
-sleepq_signal(void *wchan, int flags, int pri)
+sleepq_signal(void *wchan, int flags, int pri, int queue)
 {
 	struct sleepqueue *sq;
 	struct thread *td, *besttd;
 
 	CTR2(KTR_PROC, "sleepq_signal(%p, %d)", wchan, flags);
 	KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
+	MPASS((queue >= 0) && (queue < NR_SLEEPQS));
 	sq = sleepq_lookup(wchan);
 	if (sq == NULL) {
 		sleepq_release(wchan);
@@ -678,7 +694,7 @@ sleepq_signal(void *wchan, int flags, int pri)
 	 * the tail of sleep queues.
 	 */
 	besttd = NULL;
-	TAILQ_FOREACH(td, &sq->sq_blocked, td_slpq) {
+	TAILQ_FOREACH(td, &sq->sq_blocked[queue], td_slpq) {
 		if (besttd == NULL || td->td_priority < besttd->td_priority)
 			besttd = td;
 	}
@@ -693,12 +709,13 @@ sleepq_signal(void *wchan, int flags, int pri)
  * Resume all threads sleeping on a specified wait channel.
  */
 void
-sleepq_broadcast(void *wchan, int flags, int pri)
+sleepq_broadcast(void *wchan, int flags, int pri, int queue)
 {
 	struct sleepqueue *sq;
 
 	CTR2(KTR_PROC, "sleepq_broadcast(%p, %d)", wchan, flags);
 	KASSERT(wchan != NULL, ("%s: invalid NULL wait channel", __func__));
+	MPASS((queue >= 0) && (queue < NR_SLEEPQS));
 	sq = sleepq_lookup(wchan);
 	if (sq == NULL) {
 		sleepq_release(wchan);
@@ -709,8 +726,9 @@ sleepq_broadcast(void *wchan, int flags, int pri)
 
 	/* Resume all blocked threads on the sleep queue. */
 	mtx_lock_spin(&sched_lock);
-	while (!TAILQ_EMPTY(&sq->sq_blocked))
-		sleepq_resume_thread(sq, TAILQ_FIRST(&sq->sq_blocked), pri);
+	while (!TAILQ_EMPTY(&sq->sq_blocked[queue]))
+		sleepq_resume_thread(sq, TAILQ_FIRST(&sq->sq_blocked[queue]),
+		    pri);
 	mtx_unlock_spin(&sched_lock);
 	sleepq_release(wchan);
 }
@@ -901,15 +919,19 @@ found:
 	}
 #endif
 	db_printf("Blocked threads:\n");
-	if (TAILQ_EMPTY(&sq->sq_blocked))
-		db_printf("\tempty\n");
-	else
-		TAILQ_FOREACH(td, &sq->sq_blocked, td_slpq) {
-			db_printf("\t%p (tid %d, pid %d, \"%s\")\n", td,
-			    td->td_tid, td->td_proc->p_pid,
-			    td->td_name[0] != '\0' ? td->td_name :
-			    td->td_proc->p_comm);
-		}	
+	for (i = 0; i < NR_SLEEPQS; i++) {
+		db_printf("\nQueue[%d]:\n", i);
+		if (TAILQ_EMPTY(&sq->sq_blocked[i]))
+			db_printf("\tempty\n");
+		else
+			TAILQ_FOREACH(td, &sq->sq_blocked[0],
+				      td_slpq) {
+				db_printf("\t%p (tid %d, pid %d, \"%s\")\n", td,
+					  td->td_tid, td->td_proc->p_pid,
+					  td->td_name[i] != '\0' ? td->td_name :
+					  td->td_proc->p_comm);
+			}
+	}
 }
 
 /* Alias 'show sleepqueue' to 'show sleepq'. */
diff --git a/sys/sys/proc.h b/sys/sys/proc.h
index dd19f40..13e41e0 100644
--- a/sys/sys/proc.h
+++ b/sys/sys/proc.h
@@ -235,6 +235,7 @@ struct thread {
 	int		td_inhibitors;	/* (j) Why can not run. */
 	int		td_pflags;	/* (k) Private thread (TDP_*) flags. */
 	int		td_dupfd;	/* (k) Ret value from fdopen. XXX */
+	int		td_sqqueue;	/* (j) Sleepqueue queue blocked on. */
 	void		*td_wchan;	/* (j) Sleep address. */
 	const char	*td_wmesg;	/* (j) Reason for sleep. */
 	u_char		td_lastcpu;	/* (j) Last cpu we were on. */
diff --git a/sys/sys/sleepqueue.h b/sys/sys/sleepqueue.h
index 0979726..19a231d 100644
--- a/sys/sys/sleepqueue.h
+++ b/sys/sys/sleepqueue.h
@@ -88,19 +88,20 @@ struct thread;
 
 void	init_sleepqueues(void);
 void	sleepq_abort(struct thread *td, int intrval);
-void	sleepq_add(void *, struct lock_object *, const char *, int);
+void	sleepq_add(void *wchan, struct lock_object *lock, const char *wmesg,
+	    int flags, int queue);
 struct sleepqueue *sleepq_alloc(void);
-void	sleepq_broadcast(void *, int, int);
-void	sleepq_free(struct sleepqueue *);
-void	sleepq_lock(void *);
-struct sleepqueue *sleepq_lookup(void *);
-void	sleepq_release(void *);
-void	sleepq_remove(struct thread *, void *);
-void	sleepq_signal(void *, int, int);
+void	sleepq_broadcast(void *wchan, int flags, int pri, int queue);
+void	sleepq_free(struct sleepqueue *sq);
+void	sleepq_lock(void *wchan);
+struct sleepqueue *sleepq_lookup(void *wchan);
+void	sleepq_release(void *wchan);
+void	sleepq_remove(struct thread *td, void *wchan);
+void	sleepq_signal(void *wchan, int flags, int pri, int queue);
 void	sleepq_set_timeout(void *wchan, int timo);
 int	sleepq_timedwait(void *wchan);
 int	sleepq_timedwait_sig(void *wchan);
-void	sleepq_wait(void *);
+void	sleepq_wait(void *wchan);
 int	sleepq_wait_sig(void *wchan);
 
 #endif	/* _KERNEL */