From 82f663277d0db854e8978e5f89fd88f6df75a4a4 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 4 May 2015 22:53:22 +0200 Subject: sched / idle: Move the default idle call code to a separate function Move the code under the "use_default" label in cpuidle_idle_call() into a separate (new) function. This just allows the subsequent changes to be more stratightforward. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano Acked-by: Peter Zijlstra (Intel) --- kernel/sched/idle.c | 43 ++++++++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 19 deletions(-) (limited to 'kernel/sched') diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index fefcb1f..ae7c0be 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -67,6 +67,18 @@ void __weak arch_cpu_idle(void) local_irq_enable(); } +static void default_idle_call(void) +{ + /* + * We can't use the cpuidle framework, let's use the default idle + * routine. + */ + if (current_clr_polling_and_test()) + local_irq_enable(); + else + arch_cpu_idle(); +} + /** * cpuidle_idle_call - the main idle function * @@ -105,8 +117,10 @@ static void cpuidle_idle_call(void) */ rcu_idle_enter(); - if (cpuidle_not_available(drv, dev)) - goto use_default; + if (cpuidle_not_available(drv, dev)) { + default_idle_call(); + goto exit_idle; + } /* * Suspend-to-idle ("freeze") is a system state in which all user space @@ -134,8 +148,10 @@ static void cpuidle_idle_call(void) next_state = cpuidle_select(drv, dev); } /* Fall back to the default arch idle method on errors. */ - if (next_state < 0) - goto use_default; + if (next_state < 0) { + default_idle_call(); + goto exit_idle; + } /* * The idle task must be scheduled, it is pointless to @@ -162,8 +178,10 @@ static void cpuidle_idle_call(void) /* The cpu is no longer idle or about to enter idle. */ idle_set_state(this_rq(), NULL); - if (entered_state == -EBUSY) - goto use_default; + if (entered_state == -EBUSY) { + default_idle_call(); + goto exit_idle; + } /* * Give the governor an opportunity to reflect on the outcome @@ -182,19 +200,6 @@ exit_idle: rcu_idle_exit(); start_critical_timings(); - return; - -use_default: - /* - * We can't use the cpuidle framework, let's use the default - * idle routine. - */ - if (current_clr_polling_and_test()) - local_irq_enable(); - else - arch_cpu_idle(); - - goto exit_idle; } DEFINE_PER_CPU(bool, cpu_dead_idle); -- cgit v1.1 From bcf6ad8a4a3d002e8bc8f6639cdc119168f4e87b Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Mon, 4 May 2015 22:53:35 +0200 Subject: sched / idle: Eliminate the "reflect" check from cpuidle_idle_call() Since cpuidle_reflect() should only be called if the idle state to enter was selected by cpuidle_select(), there is the "reflect" variable in cpuidle_idle_call() whose value is used to determine whether or not that is the case. However, if the entire code run between the conditional setting "reflect" and the call to cpuidle_reflect() is moved to a separate function, it will be possible to call that new function in both branches of the conditional, in which case cpuidle_reflect() will only need to be called from one of them too and the "reflect" variable won't be necessary any more. This eliminates one check made by cpuidle_idle_call() on the majority of its invocations, so change the code as described. Signed-off-by: Rafael J. Wysocki Reviewed-by: Daniel Lezcano Acked-by: Peter Zijlstra (Intel) --- kernel/sched/idle.c | 90 +++++++++++++++++++++++++++-------------------------- 1 file changed, 46 insertions(+), 44 deletions(-) (limited to 'kernel/sched') diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index ae7c0be..9c919b4 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -79,6 +79,46 @@ static void default_idle_call(void) arch_cpu_idle(); } +static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, + int next_state) +{ + int entered_state; + + /* Fall back to the default arch idle method on errors. */ + if (next_state < 0) { + default_idle_call(); + return next_state; + } + + /* + * The idle task must be scheduled, it is pointless to go to idle, just + * update no idle residency and return. + */ + if (current_clr_polling_and_test()) { + dev->last_residency = 0; + local_irq_enable(); + return -EBUSY; + } + + /* Take note of the planned idle state. */ + idle_set_state(this_rq(), &drv->states[next_state]); + + /* + * Enter the idle state previously returned by the governor decision. + * This function will block until an interrupt occurs and will take + * care of re-enabling the local interrupts + */ + entered_state = cpuidle_enter(drv, dev, next_state); + + /* The cpu is no longer idle or about to enter idle. */ + idle_set_state(this_rq(), NULL); + + if (entered_state == -EBUSY) + default_idle_call(); + + return entered_state; +} + /** * cpuidle_idle_call - the main idle function * @@ -93,7 +133,6 @@ static void cpuidle_idle_call(void) struct cpuidle_device *dev = __this_cpu_read(cpuidle_devices); struct cpuidle_driver *drv = cpuidle_get_cpu_driver(dev); int next_state, entered_state; - bool reflect; /* * Check if the idle task must be rescheduled. If it is the @@ -138,56 +177,19 @@ static void cpuidle_idle_call(void) goto exit_idle; } - reflect = false; next_state = cpuidle_find_deepest_state(drv, dev); + call_cpuidle(drv, dev, next_state); } else { - reflect = true; /* * Ask the cpuidle framework to choose a convenient idle state. */ next_state = cpuidle_select(drv, dev); - } - /* Fall back to the default arch idle method on errors. */ - if (next_state < 0) { - default_idle_call(); - goto exit_idle; - } - - /* - * The idle task must be scheduled, it is pointless to - * go to idle, just update no idle residency and get - * out of this function - */ - if (current_clr_polling_and_test()) { - dev->last_residency = 0; - entered_state = next_state; - local_irq_enable(); - goto exit_idle; - } - - /* Take note of the planned idle state. */ - idle_set_state(this_rq(), &drv->states[next_state]); - - /* - * Enter the idle state previously returned by the governor decision. - * This function will block until an interrupt occurs and will take - * care of re-enabling the local interrupts - */ - entered_state = cpuidle_enter(drv, dev, next_state); - - /* The cpu is no longer idle or about to enter idle. */ - idle_set_state(this_rq(), NULL); - - if (entered_state == -EBUSY) { - default_idle_call(); - goto exit_idle; - } - - /* - * Give the governor an opportunity to reflect on the outcome - */ - if (reflect) + entered_state = call_cpuidle(drv, dev, next_state); + /* + * Give the governor an opportunity to reflect on the outcome + */ cpuidle_reflect(dev, entered_state); + } exit_idle: __current_set_polling(); -- cgit v1.1 From faad38492814112e3e7ce94d90123bbe301fff33 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sun, 10 May 2015 01:18:03 +0200 Subject: sched / idle: Call idle_set_state() from cpuidle_enter_state() Introduce a wrapper function around idle_set_state() called sched_idle_set_state() that will pass this_rq() to it as the first argument and make cpuidle_enter_state() call the new function before and after entering the target state. At the same time, remove direct invocations of idle_set_state() from call_cpuidle(). This will allow the invocation of default_idle_call() to be moved from call_cpuidle() to cpuidle_enter_state() safely and call_cpuidle() to be simplified a bit as a result. Signed-off-by: Rafael J. Wysocki Reviewed-by: Preeti U Murthy Tested-by: Preeti U Murthy Tested-by: Sudeep Holla Acked-by: Kevin Hilman --- kernel/sched/idle.c | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) (limited to 'kernel/sched') diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 9c919b4..5d9f549 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -15,6 +15,15 @@ #include "sched.h" +/** + * sched_idle_set_state - Record idle state for the current CPU. + * @idle_state: State to record. + */ +void sched_idle_set_state(struct cpuidle_state *idle_state) +{ + idle_set_state(this_rq(), idle_state); +} + static int __read_mostly cpu_idle_force_poll; void cpu_idle_poll_ctrl(bool enable) @@ -100,9 +109,6 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, return -EBUSY; } - /* Take note of the planned idle state. */ - idle_set_state(this_rq(), &drv->states[next_state]); - /* * Enter the idle state previously returned by the governor decision. * This function will block until an interrupt occurs and will take @@ -110,9 +116,6 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, */ entered_state = cpuidle_enter(drv, dev, next_state); - /* The cpu is no longer idle or about to enter idle. */ - idle_set_state(this_rq(), NULL); - if (entered_state == -EBUSY) default_idle_call(); -- cgit v1.1 From 827a5aefc542b8fb17c00de06118e5cd0e3800f2 Mon Sep 17 00:00:00 2001 From: "Rafael J. Wysocki" Date: Sun, 10 May 2015 01:18:46 +0200 Subject: sched / idle: Call default_idle_call() from cpuidle_enter_state() The check of the cpuidle_enter() return value against -EBUSY made in call_cpuidle() will not be necessary any more if cpuidle_enter_state() calls default_idle_call() directly when it is about to return -EBUSY, so make that happen and eliminate the check. Signed-off-by: Rafael J. Wysocki Reviewed-by: Preeti U Murthy Tested-by: Preeti U Murthy Tested-by: Sudeep Holla Acked-by: Kevin Hilman --- kernel/sched/idle.c | 20 +++++++------------- 1 file changed, 7 insertions(+), 13 deletions(-) (limited to 'kernel/sched') diff --git a/kernel/sched/idle.c b/kernel/sched/idle.c index 5d9f549..594275e 100644 --- a/kernel/sched/idle.c +++ b/kernel/sched/idle.c @@ -76,12 +76,13 @@ void __weak arch_cpu_idle(void) local_irq_enable(); } -static void default_idle_call(void) +/** + * default_idle_call - Default CPU idle routine. + * + * To use when the cpuidle framework cannot be used. + */ +void default_idle_call(void) { - /* - * We can't use the cpuidle framework, let's use the default idle - * routine. - */ if (current_clr_polling_and_test()) local_irq_enable(); else @@ -91,8 +92,6 @@ static void default_idle_call(void) static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, int next_state) { - int entered_state; - /* Fall back to the default arch idle method on errors. */ if (next_state < 0) { default_idle_call(); @@ -114,12 +113,7 @@ static int call_cpuidle(struct cpuidle_driver *drv, struct cpuidle_device *dev, * This function will block until an interrupt occurs and will take * care of re-enabling the local interrupts */ - entered_state = cpuidle_enter(drv, dev, next_state); - - if (entered_state == -EBUSY) - default_idle_call(); - - return entered_state; + return cpuidle_enter(drv, dev, next_state); } /** -- cgit v1.1