Merge branch 'overlayfs-linus' of git://git.kernel.org/pub/scm/linux/kernel/git/mszer...

[deliverable/linux.git] / kernel / sched / core.c

diff --git a/kernel/sched/core.c b/kernel/sched/core.c
index 404c0784b1fc32e7e5cdcf7d17010da638753067..017d5394f5dc9206c21259d61f77936396796448 100644 (file)
--- a/kernel/sched/core.c
+++ b/kernel/sched/core.c
@@ -1768,13 +1768,15 @@ void sched_ttwu_pending(void)
        cookie = lockdep_pin_lock(&rq->lock);
 
        while (llist) {
+               int wake_flags = 0;
+
                p = llist_entry(llist, struct task_struct, wake_entry);
                llist = llist_next(llist);
-               /*
-                * See ttwu_queue(); we only call ttwu_queue_remote() when
-                * its a x-cpu wakeup.
-                */
-               ttwu_do_activate(rq, p, WF_MIGRATED, cookie);
+
+               if (p->sched_remote_wakeup)
+                       wake_flags = WF_MIGRATED;
+
+               ttwu_do_activate(rq, p, wake_flags, cookie);
        }
 
        lockdep_unpin_lock(&rq->lock, cookie);
@@ -1819,10 +1821,12 @@ void scheduler_ipi(void)
        irq_exit();
 }
 
-static void ttwu_queue_remote(struct task_struct *p, int cpu)
+static void ttwu_queue_remote(struct task_struct *p, int cpu, int wake_flags)
 {
        struct rq *rq = cpu_rq(cpu);
 
+       p->sched_remote_wakeup = !!(wake_flags & WF_MIGRATED);
+
        if (llist_add(&p->wake_entry, &cpu_rq(cpu)->wake_list)) {
                if (!set_nr_if_polling(rq->idle))
                        smp_send_reschedule(cpu);
@@ -1869,7 +1873,7 @@ static void ttwu_queue(struct task_struct *p, int cpu, int wake_flags)
 #if defined(CONFIG_SMP)
        if (sched_feat(TTWU_QUEUE) && !cpus_share_cache(smp_processor_id(), cpu)) {
                sched_clock_cpu(cpu); /* sync clocks x-cpu */
-               ttwu_queue_remote(p, cpu);
+               ttwu_queue_remote(p, cpu, wake_flags);
                return;
        }
 #endif
@@ -2249,9 +2253,11 @@ int sysctl_numa_balancing(struct ctl_table *table, int write,
 #endif
 #endif
 
+#ifdef CONFIG_SCHEDSTATS
+
 DEFINE_STATIC_KEY_FALSE(sched_schedstats);
+static bool __initdata __sched_schedstats = false;
 
-#ifdef CONFIG_SCHEDSTATS
 static void set_schedstats(bool enabled)
 {
        if (enabled)
@@ -2274,11 +2280,16 @@ static int __init setup_schedstats(char *str)
        if (!str)
                goto out;
 
+       /*
+        * This code is called before jump labels have been set up, so we can't
+        * change the static branch directly just yet.  Instead set a temporary
+        * variable so init_schedstats() can do it later.
+        */
        if (!strcmp(str, "enable")) {
-               set_schedstats(true);
+               __sched_schedstats = true;
                ret = 1;
        } else if (!strcmp(str, "disable")) {
-               set_schedstats(false);
+               __sched_schedstats = false;
                ret = 1;
        }
 out:
@@ -2289,6 +2300,11 @@ out:
 }
 __setup("schedstats=", setup_schedstats);
 
+static void __init init_schedstats(void)
+{
+       set_schedstats(__sched_schedstats);
+}
+
 #ifdef CONFIG_PROC_SYSCTL
 int sysctl_schedstats(struct ctl_table *table, int write,
                         void __user *buffer, size_t *lenp, loff_t *ppos)
@@ -2309,8 +2325,10 @@ int sysctl_schedstats(struct ctl_table *table, int write,
                set_schedstats(state);
        return err;
 }
-#endif
-#endif
+#endif /* CONFIG_PROC_SYSCTL */
+#else  /* !CONFIG_SCHEDSTATS */
+static inline void init_schedstats(void) {}
+#endif /* CONFIG_SCHEDSTATS */
 
 /*
  * fork()/clone()-time setup:
@@ -3152,7 +3170,8 @@ static noinline void __schedule_bug(struct task_struct *prev)
 static inline void schedule_debug(struct task_struct *prev)
 {
 #ifdef CONFIG_SCHED_STACK_END_CHECK
-       BUG_ON(task_stack_end_corrupted(prev));
+       if (task_stack_end_corrupted(prev))
+               panic("corrupted stack end detected inside scheduler\n");
 #endif
 
        if (unlikely(in_atomic_preempt_off())) {
@@ -7483,6 +7502,8 @@ void __init sched_init(void)
 #endif
        init_sched_fair_class();
 
+       init_schedstats();
+
        scheduler_running = 1;
 }