- Make runq_steal_from more aggressive. Previously it would examine only

   a single priority queue.  If that queue had a thread or threads which
   could not be migrated we would fail to steal load.  This could cause
   starvation in situations where cores are idle.

Submitted by:	Doug Kilpatrick <dkilpatrick@isilon.com>
Tested by:	pho
Reviewed by:	mav
Sponsored by:	EMC / Isilon Storage Division

1 files changed, 11 insertions, 16 deletions

diff --git a/sys/kern/sched_ule.c b/sys/kern/sched_ule.c
index 7eff4ad..d5162ed 100644
--- a/sys/kern/sched_ule.c
+++ b/sys/kern/sched_ule.c
@@ -1057,32 +1057,27 @@ runq_steal_from(struct runq *rq, int cpu, u_char start)
 	struct rqhead *rqh;
 	struct thread *td, *first;
 	int bit;
-	int pri;
 	int i;
 
 	rqb = &rq->rq_status;
 	bit = start & (RQB_BPW -1);
-	pri = 0;
 	first = NULL;
 again:
 	for (i = RQB_WORD(start); i < RQB_LEN; bit = 0, i++) {
 		if (rqb->rqb_bits[i] == 0)
 			continue;
-		if (bit != 0) {
-			for (pri = bit; pri < RQB_BPW; pri++)
-				if (rqb->rqb_bits[i] & (1ul << pri))
-					break;
-			if (pri >= RQB_BPW)
+		if (bit == 0)
+			bit = RQB_FFS(rqb->rqb_bits[i]);
+		for (; bit < RQB_BPW; bit++) {
+			if ((rqb->rqb_bits[i] & (1ul << bit)) == 0)
 				continue;
-		} else
-			pri = RQB_FFS(rqb->rqb_bits[i]);
-		pri += (i << RQB_L2BPW);
-		rqh = &rq->rq_queues[pri];
-		TAILQ_FOREACH(td, rqh, td_runq) {
-			if (first && THREAD_CAN_MIGRATE(td) &&
-			    THREAD_CAN_SCHED(td, cpu))
-				return (td);
-			first = td;
+			rqh = &rq->rq_queues[bit + (i << RQB_L2BPW)];
+			TAILQ_FOREACH(td, rqh, td_runq) {
+				if (first && THREAD_CAN_MIGRATE(td) &&
+				    THREAD_CAN_SCHED(td, cpu))
+					return (td);
+				first = td;
+			}
 		}
 	}
 	if (start != 0) {