sched/fair: Consider missed ticks in NOHZ_FULL in update_cpu_load_nohz()

Usually the tick can be stopped for an idle CPU in NOHZ. However in NOHZ_FULL
mode, a non-idle CPU's tick can also be stopped. However, update_cpu_load_nohz()
does not consider the case a non-idle CPU's tick has been stopped at all.

This patch makes the update_cpu_load_nohz() know if the calling path comes
from NOHZ_FULL or idle NOHZ.

Signed-off-by: Byungchul Park <byungchul.park@lge.com>
Signed-off-by: Peter Zijlstra (Intel) <peterz@infradead.org>
Acked-by: Frederic Weisbecker <fweisbec@gmail.com>
Cc: Linus Torvalds <torvalds@linux-foundation.org>
Cc: Mike Galbraith <efault@gmx.de>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Thomas Gleixner <tglx@linutronix.de>
Link: http://lkml.kernel.org/r/1447115762-19734-3-git-send-email-byungchul.park@lge.com
Signed-off-by: Ingo Molnar <mingo@kernel.org>

diff --git a/include/linux/sched.h b/include/linux/sched.h
index edad7a43edea141b3ada89f7f12edac2add7b68e..f425aac633177e3723fcebafc8b2fe6d4063e465 100644 (file)
--- a/include/linux/sched.h
+++ b/include/linux/sched.h
@@ -177,9 +177,9 @@ extern void get_iowait_load(unsigned long *nr_waiters, unsigned long *load);
 extern void calc_global_load(unsigned long ticks);
 
 #if defined(CONFIG_SMP) && defined(CONFIG_NO_HZ_COMMON)
-extern void update_cpu_load_nohz(void);
+extern void update_cpu_load_nohz(int active);
 #else
-static inline void update_cpu_load_nohz(void) { }
+static inline void update_cpu_load_nohz(int active) { }
 #endif
 
 extern unsigned long get_parent_ip(unsigned long addr);

diff --git a/kernel/sched/fair.c b/kernel/sched/fair.c
index 404006ae1ac956984bb9dc3353b9031a12d23733..309b1d551f25a0ea568ce38b589b9632e6648ec2 100644 (file)
--- a/kernel/sched/fair.c
+++ b/kernel/sched/fair.c
@@ -4397,10 +4397,11 @@ static void update_idle_cpu_load(struct rq *this_rq)
 /*
  * Called from tick_nohz_idle_exit() -- try and fix up the ticks we missed.
  */
-void update_cpu_load_nohz(void)
+void update_cpu_load_nohz(int active)
 {
        struct rq *this_rq = this_rq();
        unsigned long curr_jiffies = READ_ONCE(jiffies);
+       unsigned long load = active ? weighted_cpuload(cpu_of(this_rq)) : 0;
        unsigned long pending_updates;
 
        if (curr_jiffies == this_rq->last_load_update_tick)
@@ -4411,10 +4412,11 @@ void update_cpu_load_nohz(void)
        if (pending_updates) {
                this_rq->last_load_update_tick = curr_jiffies;
                /*
-                * We were idle, this means load 0, the current load might be
-                * !0 due to remote wakeups and the sort.
+                * In the regular NOHZ case, we were idle, this means load 0.
+                * In the NOHZ_FULL case, we were non-idle, we should consider
+                * its weighted load.
                 */
-               __update_cpu_load(this_rq, 0, pending_updates, 0);
+               __update_cpu_load(this_rq, load, pending_updates, active);
        }
        raw_spin_unlock(&this_rq->lock);
 }

diff --git a/kernel/time/tick-sched.c b/kernel/time/tick-sched.c
index 7c7ec45159834a1b25576fbed037c9951f3c076f..515edf3eb62b885a1e8f2007a9ad4dccf2164635 100644 (file)
--- a/kernel/time/tick-sched.c
+++ b/kernel/time/tick-sched.c
@@ -694,11 +694,11 @@ out:
        return tick;
 }
 
-static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now)
+static void tick_nohz_restart_sched_tick(struct tick_sched *ts, ktime_t now, int active)
 {
        /* Update jiffies first */
        tick_do_update_jiffies64(now);
-       update_cpu_load_nohz();
+       update_cpu_load_nohz(active);
 
        calc_load_exit_idle();
        touch_softlockup_watchdog();
@@ -725,7 +725,7 @@ static void tick_nohz_full_update_tick(struct tick_sched *ts)
        if (can_stop_full_tick())
                tick_nohz_stop_sched_tick(ts, ktime_get(), cpu);
        else if (ts->tick_stopped)
-               tick_nohz_restart_sched_tick(ts, ktime_get());
+               tick_nohz_restart_sched_tick(ts, ktime_get(), 1);
 #endif
 }
 
@@ -916,7 +916,7 @@ void tick_nohz_idle_exit(void)
                tick_nohz_stop_idle(ts, now);
 
        if (ts->tick_stopped) {
-               tick_nohz_restart_sched_tick(ts, now);
+               tick_nohz_restart_sched_tick(ts, now, 0);
                tick_nohz_account_idle_ticks(ts);
        }