Use cpuset defined in pthread_attr for newly created thread, for now,

we set scheduling parameters and cpu binding fully in userland, and
because default scheduling policy is SCHED_RR (time-sharing), we set
default sched_inherit to PTHREAD_SCHED_INHERIT, this saves a system
call.

3 files changed, 57 insertions, 21 deletions

diff --git a/lib/libthr/thread/thr_create.c b/lib/libthr/thread/thr_create.c
index 3bfd3b3..692ec45 100644
--- a/lib/libthr/thread/thr_create.c
+++ b/lib/libthr/thread/thr_create.c
@@ -29,7 +29,6 @@
 
 #include "namespace.h"
 #include <sys/types.h>
-#include <sys/rtprio.h>
 #include <sys/signalvar.h>
 #include <errno.h>
 #include <stdlib.h>
@@ -53,9 +52,10 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	struct pthread *curthread, *new_thread;
 	struct thr_param param;
 	struct sched_param sched_param;
-	struct rtprio rtp;
 	int ret = 0, locked, create_suspended;
 	sigset_t set, oset;
+	cpuset_t *cpuset = NULL;
+	int cpusetsize = 0;
 
 	_thr_check_init();
 
@@ -76,6 +76,8 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 		new_thread->attr = _pthread_attr_default;
 	else {
 		new_thread->attr = *(*attr);
+		cpuset = new_thread->attr.cpuset;
+		cpusetsize = new_thread->attr.cpusetsize;
 		new_thread->attr.cpuset = NULL;
 		new_thread->attr.cpusetsize = 0;
 	}
@@ -133,7 +135,8 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	_thr_link(curthread, new_thread);
 	/* Return thread pointer eariler so that new thread can use it. */
 	(*thread) = new_thread;
-	if (SHOULD_REPORT_EVENT(curthread, TD_CREATE)) {
+	if (SHOULD_REPORT_EVENT(curthread, TD_CREATE) || cpuset != NULL ||
+	    new_thread->attr.sched_inherit != PTHREAD_INHERIT_SCHED) {
 		THR_THREAD_LOCK(curthread, new_thread);
 		locked = 1;
 	} else
@@ -149,14 +152,7 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 	param.flags = 0;
 	if (new_thread->attr.flags & PTHREAD_SCOPE_SYSTEM)
 		param.flags |= THR_SYSTEM_SCOPE;
-	if (new_thread->attr.sched_inherit == PTHREAD_INHERIT_SCHED)
-		param.rtp = NULL;
-	else {
-		sched_param.sched_priority = new_thread->attr.prio;
-		_schedparam_to_rtp(new_thread->attr.sched_policy,
-			&sched_param, &rtp);
-		param.rtp = &rtp;
-	}
+	param.rtp = NULL;
 
 	/* Schedule the new thread. */
 	if (create_suspended) {
@@ -197,11 +193,43 @@ _pthread_create(pthread_t * thread, const pthread_attr_t * attr,
 		new_thread->tlflags |= TLFLAGS_DETACHED;
 		_thr_ref_delete_unlocked(curthread, new_thread);
 		THREAD_LIST_UNLOCK(curthread);
-		(*thread) = 0;
 	} else if (locked) {
+		if (cpuset != NULL) {
+			if (cpuset_setaffinity(CPU_LEVEL_WHICH, CPU_WHICH_TID,
+				TID(new_thread), cpusetsize, cpuset)) {
+				ret = errno;
+				/* kill the new thread */
+				new_thread->force_exit = 1;
+				THR_THREAD_UNLOCK(curthread, new_thread);
+				goto out;
+			}
+		}
+
+		if (new_thread->attr.sched_inherit != PTHREAD_INHERIT_SCHED) {
+			sched_param.sched_priority = new_thread->attr.prio;
+			if (_thr_setscheduler(TID(new_thread),
+			    new_thread->attr.sched_policy, &sched_param)) {
+				ret = errno;
+				/* kill the new thread */
+				new_thread->force_exit = 1;
+				THR_THREAD_UNLOCK(curthread, new_thread);
+				goto out;
+			}
+		}
+		
 		_thr_report_creation(curthread, new_thread);
 		THR_THREAD_UNLOCK(curthread, new_thread);
+out:
+		if (ret) {
+			THREAD_LIST_LOCK(curthread);
+			new_thread->tlflags |= TLFLAGS_DETACHED;
+			_thr_ref_delete_unlocked(curthread, new_thread);
+			THREAD_LIST_UNLOCK(curthread);
+		}
 	}
+
+	if (ret)
+		(*thread) = 0;
 	return (ret);
 }
 
@@ -224,6 +252,17 @@ create_stack(struct pthread_attr *pattr)
 static void
 thread_start(struct pthread *curthread)
 {
+	/*
+	 * This is used as a serialization point to allow parent
+	 * to report 'new thread' event to debugger or tweak new thread's
+	 * attributes before the new thread does real-world work.
+	 */
+	THR_LOCK(curthread);
+	THR_UNLOCK(curthread);
+
+	if (curthread->force_exit)
+		_pthread_exit(PTHREAD_CANCELED);
+
 	if (curthread->unblock_sigcancel) {
 		sigset_t set;
 
@@ -244,14 +283,6 @@ thread_start(struct pthread *curthread)
 		sigprocmask(SIG_SETMASK, &set, NULL);
 	}
 
-	/*
-	 * This is used as a serialization point to allow parent
-	 * to report 'new thread' event to debugger before the thread
-	 * does real work.
-	 */
-	THR_LOCK(curthread);
-	THR_UNLOCK(curthread);
-
 	/* Run the current thread's start routine with argument: */
 	_pthread_exit(curthread->start_routine(curthread->arg));
 
diff --git a/lib/libthr/thread/thr_init.c b/lib/libthr/thread/thr_init.c
index d76c5ca..100d14a 100644
--- a/lib/libthr/thread/thr_init.c
+++ b/lib/libthr/thread/thr_init.c
@@ -75,7 +75,7 @@ struct pthread_prio	_thr_priorities[3] = {
 
 struct pthread_attr _pthread_attr_default = {
 	.sched_policy = SCHED_OTHER,
-	.sched_inherit = 0,
+	.sched_inherit = PTHREAD_INHERIT_SCHED,
 	.prio = 0,
 	.suspend = THR_CREATE_RUNNING,
 	.flags = PTHREAD_SCOPE_SYSTEM,
diff --git a/lib/libthr/thread/thr_private.h b/lib/libthr/thread/thr_private.h
index ff9f52b..7c1d005 100644
--- a/lib/libthr/thread/thr_private.h
+++ b/lib/libthr/thread/thr_private.h
@@ -384,6 +384,9 @@ struct pthread {
 	/* New thread should unblock SIGCANCEL. */
 	int			unblock_sigcancel;
 
+	/* Force new thread to exit. */
+	int			force_exit;
+
 	/* Thread state: */
 	enum pthread_state 	state;
 
@@ -668,6 +671,8 @@ int	_schedparam_to_rtp(int policy, const struct sched_param *param,
 void	_thread_bp_create(void);
 void	_thread_bp_death(void);
 int	_sched_yield(void);
+int	_pthread_getaffinity_np(pthread_t, size_t, cpuset_t *);
+int	_pthread_setaffinity_np(pthread_t, size_t, const cpuset_t *);
 
 /* #include <fcntl.h> */
 #ifdef  _SYS_FCNTL_H_