- Fix the kthread_{suspend, resume, suspend_check}() locking.

  In the current code, the locking is completely broken and may lead
  easilly to deadlocks. Fix it by using the proc_mtx, linked to the
  suspending thread, as lock for the operation.  Keep using the
  thread_lock for setting and reading the flag even if it is not entirely
  necessary (atomic ops may do it as well, but this way the code is more
  readable).
- Fix a deadlock within kthread_suspend().
  The suspender should not sleep on a different channel wrt the suspended
  thread, or, otherwise, the awaker should wakeup both. Uniform the
  interface to what the kproc_* counterparts do (sleeping on the same
  channel).
- Change the kthread_suspend_check() prototype.
  kthread_suspend_check() always assumes curthread and must only refer to
  it, so skip the thread pointer as it may be easilly mistaken.
  If curthread is not a kthread, the system will panic.

In collabouration with:	jhb
Tested by:		Giovanni Trematerra
			<giovanni dot trematerra at gmail dot com>
MFC:			2 weeks

1 files changed, 52 insertions, 18 deletions

diff --git a/sys/kern/kern_kthread.c b/sys/kern/kern_kthread.c
index 3c5248e..5809d14 100644
--- a/sys/kern/kern_kthread.c
+++ b/sys/kern/kern_kthread.c
@@ -335,34 +335,55 @@ kthread_exit(void)
 int
 kthread_suspend(struct thread *td, int timo)
 {
-	if ((td->td_pflags & TDP_KTHREAD) == 0) {
+	struct proc *p;
+
+	p = td->td_proc;
+
+	/*
+	 * td_pflags should not be ready by any other thread different by
+	 * curthread, but as long as this flag is invariant during the
+	 * thread lifetime, it is ok to check for it now.
+	 */
+	if ((td->td_pflags & TDP_KTHREAD) == 0)
 		return (EINVAL);
-	}
+
+	/*
+	 * The caller of the primitive should have already checked that the
+	 * thread is up and running, thus not being blocked by other
+	 * conditions.
+	 */
+	PROC_LOCK(p);
 	thread_lock(td);
 	td->td_flags |= TDF_KTH_SUSP;
 	thread_unlock(td);
-	/*
-	 * If it's stopped for some other reason,
-	 * kick it to notice our request 
-	 * or we'll end up timing out
-	 */
-	wakeup(td); /* traditional  place for kernel threads to sleep on */ /* XXX ?? */
-	return (tsleep(&td->td_flags, PPAUSE | PDROP, "suspkt", timo));
+	return (msleep(&td->td_flags, &p->p_mtx, PPAUSE | PDROP, "suspkt",
+	    timo));
 }
 
 /*
- * let the kthread it can keep going again.
+ * Resume a thread previously put asleep with kthread_suspend().
  */
 int
 kthread_resume(struct thread *td)
 {
-	if ((td->td_pflags & TDP_KTHREAD) == 0) {
+	struct proc *p;
+
+	p = td->td_proc;
+
+	/*
+	 * td_pflags should not be ready by any other thread different by
+	 * curthread, but as long as this flag is invariant during the
+	 * thread lifetime, it is ok to check for it now.
+	 */
+	if ((td->td_pflags & TDP_KTHREAD) == 0)
 		return (EINVAL);
-	}
+
+	PROC_LOCK(p);
 	thread_lock(td);
 	td->td_flags &= ~TDF_KTH_SUSP;
 	thread_unlock(td);
-	wakeup(&td->td_name);
+	wakeup(&td->td_flags);
+	PROC_UNLOCK(p);
 	return (0);
 }
 
@@ -371,15 +392,28 @@ kthread_resume(struct thread *td)
  * and notify the caller that is has happened.
  */
 void
-kthread_suspend_check(struct thread *td)
+kthread_suspend_check()
 {
+	struct proc *p;
+	struct thread *td;
+
+	td = curthread;
+	p = td->td_proc;
+
+	if ((td->td_pflags & TDP_KTHREAD) == 0)
+		panic("%s: curthread is not a valid kthread", __func__);
+
+	/*
+	 * As long as the double-lock protection is used when accessing the
+	 * TDF_KTH_SUSP flag, synchronizing the read operation via proc mutex
+	 * is fine.
+	 */
+	PROC_LOCK(p);
 	while (td->td_flags & TDF_KTH_SUSP) {
-		/*
-		 * let the caller know we got the message then sleep
-		 */
 		wakeup(&td->td_flags);
-		tsleep(&td->td_name, PPAUSE, "ktsusp", 0);
+		msleep(&td->td_flags, &p->p_mtx, PPAUSE, "ktsusp", 0);
 	}
+	PROC_UNLOCK(p);
 }
 
 int