sched: Use resched IPI to kick off the nohz idle balance

[deliverable/linux.git] / kernel / sched.c

diff --git a/kernel/sched.c b/kernel/sched.c
index 4cdc91cf48f68cd4e3b6d80c25c3ea5fa237092e..9e49af00ae3e41240af20e25e8462734ab4a108b 100644 (file)
--- a/kernel/sched.c
+++ b/kernel/sched.c
@@ -1404,6 +1404,18 @@ void wake_up_idle_cpu(int cpu)
                smp_send_reschedule(cpu);
 }
 
+static inline bool got_nohz_idle_kick(void)
+{
+       return idle_cpu(smp_processor_id()) && this_rq()->nohz_balance_kick;
+}
+
+#else /* CONFIG_NO_HZ */
+
+static inline bool got_nohz_idle_kick(void)
+{
+       return false;
+}
+
 #endif /* CONFIG_NO_HZ */
 
 static u64 sched_avg_period(void)
@@ -2717,7 +2729,7 @@ static void sched_ttwu_pending(void)
 
 void scheduler_ipi(void)
 {
-       if (llist_empty(&this_rq()->wake_list))
+       if (llist_empty(&this_rq()->wake_list) && !got_nohz_idle_kick())
                return;
 
        /*
@@ -2735,6 +2747,12 @@ void scheduler_ipi(void)
         */
        irq_enter();
        sched_ttwu_pending();
+
+       /*
+        * Check if someone kicked us for doing the nohz idle load balance.
+        */
+       if (unlikely(got_nohz_idle_kick() && !need_resched()))
+               raise_softirq_irqoff(SCHED_SOFTIRQ);
        irq_exit();
 }
 
@@ -8288,7 +8306,6 @@ void __init sched_init(void)
                rq_attach_root(rq, &def_root_domain);
 #ifdef CONFIG_NO_HZ
                rq->nohz_balance_kick = 0;
-               init_sched_softirq_csd(&per_cpu(remote_sched_softirq_cb, i));
 #endif
 #endif
                init_rq_hrtick(rq);