Fix problems with cancellation while in critical regions.

  o Cancellation flags were not getting properly set/cleared.
  o Loops waiting for internal locks were not being exited
    correctly by a cancelled thread.
  o Minor spelling (cancelation -> cancellation) and formatting
    corrections (missing tab).

Found by:	tg
Reviewed by:	jasone

6 files changed, 21 insertions, 8 deletions

diff --git a/lib/libkse/thread/thr_cancel.c b/lib/libkse/thread/thr_cancel.c
index bad5533..de7c491 100644
--- a/lib/libkse/thread/thr_cancel.c
+++ b/lib/libkse/thread/thr_cancel.c
@@ -20,8 +20,16 @@ pthread_cancel(pthread_t pthread)
 		/* Protect the scheduling queues: */
 		_thread_kern_sig_defer();
 
-		/* Check if we need to kick it back into the run queue: */
-		if ((pthread->cancelflags & PTHREAD_CANCEL_DISABLE) == 0)
+		if (((pthread->cancelflags & PTHREAD_CANCEL_DISABLE) != 0) ||
+		    (((pthread->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) == 0) &&
+		    ((pthread->cancelflags & PTHREAD_AT_CANCEL_POINT) == 0)))
+			/* Just mark it for cancellation: */
+			pthread->cancelflags |= PTHREAD_CANCELLING;
+		else {
+			/*
+			 * Check if we need to kick it back into the
+			 * run queue:
+			 */
 			switch (pthread->state) {
 			case PS_RUNNING:
 				/* No need to resume: */
@@ -33,7 +41,7 @@ pthread_cancel(pthread_t pthread)
 			case PS_FDW_WAIT:
 			case PS_POLL_WAIT:
 			case PS_SELECT_WAIT:
-			/* Remove these threads from the work queue: */
+				/* Remove these threads from the work queue: */
 				if ((pthread->flags & PTHREAD_FLAGS_IN_WORKQ)
 				    != 0)
 					PTHREAD_WORKQ_REMOVE(pthread);
@@ -75,7 +83,9 @@ pthread_cancel(pthread_t pthread)
 			case PS_STATE_MAX:
 				/* Ignore - only here to silence -Wall: */
 				break;
+			}
 		}
+
 		/* Unprotect the scheduling queues: */
 		_thread_kern_sig_undefer();
 
@@ -96,7 +106,7 @@ pthread_setcancelstate(int state, int *oldstate)
 	case PTHREAD_CANCEL_ENABLE:
 		if (oldstate != NULL)
 			*oldstate = ostate;
-		_thread_run->cancelflags &= PTHREAD_CANCEL_ENABLE;
+		_thread_run->cancelflags &= ~PTHREAD_CANCEL_DISABLE;
 		if ((_thread_run->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0)
 			pthread_testcancel();
 		ret = 0;
@@ -145,7 +155,6 @@ pthread_setcanceltype(int type, int *oldtype)
 void
 pthread_testcancel(void)
 {
-
 	if (((_thread_run->cancelflags & PTHREAD_CANCEL_DISABLE) == 0) &&
 	    ((_thread_run->cancelflags & PTHREAD_CANCELLING) != 0)) {
 		/*
diff --git a/lib/libkse/thread/thr_cond.c b/lib/libkse/thread/thr_cond.c
index 09c5f22..3e215af 100644
--- a/lib/libkse/thread/thr_cond.c
+++ b/lib/libkse/thread/thr_cond.c
@@ -274,6 +274,7 @@ pthread_cond_wait(pthread_cond_t * cond, pthread_mutex_t * mutex)
 		}
 
 		if ((_thread_run->cancelflags & PTHREAD_CANCEL_NEEDED) != 0) {
+			_thread_run->cancelflags &= ~PTHREAD_CANCEL_NEEDED;
 			_thread_exit_cleanup();
 			pthread_exit(PTHREAD_CANCELED);
 		}
@@ -431,6 +432,7 @@ pthread_cond_timedwait(pthread_cond_t * cond, pthread_mutex_t * mutex,
 		}
 
 		if ((_thread_run->cancelflags & PTHREAD_CANCEL_NEEDED) != 0) {
+			_thread_run->cancelflags &= ~PTHREAD_CANCEL_NEEDED;
 			_thread_exit_cleanup();
 			pthread_exit(PTHREAD_CANCELED);
 		}
diff --git a/lib/libkse/thread/thr_join.c b/lib/libkse/thread/thr_join.c
index d149cf1..155dc64 100644
--- a/lib/libkse/thread/thr_join.c
+++ b/lib/libkse/thread/thr_join.c
@@ -98,6 +98,7 @@ pthread_join(pthread_t pthread, void **thread_return)
 		_thread_kern_sig_undefer();
 
 		if ((_thread_run->cancelflags & PTHREAD_CANCEL_NEEDED) != 0) {
+			_thread_run->cancelflags &= ~PTHREAD_CANCEL_NEEDED;
 			_thread_exit_cleanup();
 			pthread_exit(PTHREAD_CANCELED);
 		}
diff --git a/lib/libkse/thread/thr_kern.c b/lib/libkse/thread/thr_kern.c
index f69cb41..b3fbc3a 100644
--- a/lib/libkse/thread/thr_kern.c
+++ b/lib/libkse/thread/thr_kern.c
@@ -111,7 +111,7 @@ __asm__("fnsave %0": :"m"(*fdata));
 		if (((_thread_run->cancelflags & PTHREAD_AT_CANCEL_POINT) == 0) &&
 		    ((_thread_run->cancelflags & PTHREAD_CANCEL_ASYNCHRONOUS) != 0)) {
 			/* 
-			 * Cancelations override signals.
+			 * Cancellations override signals.
 			 *
 			 * Stick a cancellation point at the start of
 			 * each async-cancellable thread's resumption.
diff --git a/lib/libkse/thread/thr_mutex.c b/lib/libkse/thread/thr_mutex.c
index b2a06f2..c625ef2 100644
--- a/lib/libkse/thread/thr_mutex.c
+++ b/lib/libkse/thread/thr_mutex.c
@@ -623,6 +623,7 @@ pthread_mutex_lock(pthread_mutex_t * mutex)
 		_thread_kern_sig_undefer();
 
 		if ((_thread_run->cancelflags & PTHREAD_CANCEL_NEEDED) != 0) {
+			_thread_run->cancelflags &= ~PTHREAD_CANCEL_NEEDED;
 			_thread_exit_cleanup();
 			pthread_exit(PTHREAD_CANCELED);
 		}
diff --git a/lib/libkse/thread/thr_write.c b/lib/libkse/thread/thr_write.c
index 09b09cd..40c4cc5 100644
--- a/lib/libkse/thread/thr_write.c
+++ b/lib/libkse/thread/thr_write.c
@@ -67,8 +67,8 @@ write(int fd, const void *buf, size_t nbytes)
 			/* File is not open for write: */
 			errno = EBADF;
 			_FD_UNLOCK(fd, FD_WRITE);
-		_thread_leave_cancellation_point();
-		return (-1);
+			_thread_leave_cancellation_point();
+			return (-1);
 		}
 
 		/* Check if file operations are to block */