latencytop: optimize LT_BACKTRACEDEPTH loops a bit

[deliverable/linux.git] / kernel / sched_fair.c

diff --git a/kernel/sched_fair.c b/kernel/sched_fair.c
index b85cac4b5e25627d057773316f534116d3b95905..bedda18f37a5708b58f8eacc9d9d4043d9be889e 100644 (file)
--- a/kernel/sched_fair.c
+++ b/kernel/sched_fair.c
@@ -61,25 +61,15 @@ const_debug unsigned int sysctl_sched_child_runs_first = 1;
  */
 unsigned int __read_mostly sysctl_sched_compat_yield;
 
-/*
- * SCHED_BATCH wake-up granularity.
- * (default: 10 msec * (1 + ilog(ncpus)), units: nanoseconds)
- *
- * This option delays the preemption effects of decoupled workloads
- * and reduces their over-scheduling. Synchronous workloads will still
- * have immediate wakeup/sleep latencies.
- */
-unsigned int sysctl_sched_batch_wakeup_granularity = 10000000UL;
-
 /*
  * SCHED_OTHER wake-up granularity.
- * (default: 5 msec * (1 + ilog(ncpus)), units: nanoseconds)
+ * (default: 10 msec * (1 + ilog(ncpus)), units: nanoseconds)
  *
  * This option delays the preemption effects of decoupled workloads
  * and reduces their over-scheduling. Synchronous workloads will still
  * have immediate wakeup/sleep latencies.
  */
-unsigned int sysctl_sched_wakeup_granularity = 5000000UL;
+unsigned int sysctl_sched_wakeup_granularity = 10000000UL;
 
 const_debug unsigned int sysctl_sched_migration_cost = 500000UL;
 
@@ -302,11 +292,6 @@ static u64 __sched_vslice(unsigned long rq_weight, unsigned long nr_running)
        return vslice;
 }
 
-static u64 sched_vslice(struct cfs_rq *cfs_rq)
-{
-       return __sched_vslice(cfs_rq->load.weight, cfs_rq->nr_running);
-}
-
 static u64 sched_vslice_add(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
        return __sched_vslice(cfs_rq->load.weight + se->load.weight,
@@ -504,15 +489,6 @@ place_entity(struct cfs_rq *cfs_rq, struct sched_entity *se, int initial)
        } else
                vruntime = cfs_rq->min_vruntime;
 
-       if (sched_feat(TREE_AVG)) {
-               struct sched_entity *last = __pick_last_entity(cfs_rq);
-               if (last) {
-                       vruntime += last->vruntime;
-                       vruntime >>= 1;
-               }
-       } else if (sched_feat(APPROX_AVG) && cfs_rq->nr_running)
-               vruntime += sched_vslice(cfs_rq)/2;
-
        /*
         * The 'current' period is already promised to the current tasks,
         * however the extra weight of the new task will slow them down a
@@ -643,20 +619,16 @@ set_next_entity(struct cfs_rq *cfs_rq, struct sched_entity *se)
        se->prev_sum_exec_runtime = se->sum_exec_runtime;
 }
 
+static int
+wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se);
+
 static struct sched_entity *
 pick_next(struct cfs_rq *cfs_rq, struct sched_entity *se)
 {
-       s64 diff, gran;
-
        if (!cfs_rq->next)
                return se;
 
-       diff = cfs_rq->next->vruntime - se->vruntime;
-       if (diff < 0)
-               return se;
-
-       gran = calc_delta_fair(sysctl_sched_wakeup_granularity, &cfs_rq->load);
-       if (diff > gran)
+       if (wakeup_preempt_entity(cfs_rq->next, se) != 0)
                return se;
 
        return cfs_rq->next;
@@ -1115,6 +1087,48 @@ out:
 }
 #endif /* CONFIG_SMP */
 
+static unsigned long wakeup_gran(struct sched_entity *se)
+{
+       unsigned long gran = sysctl_sched_wakeup_granularity;
+
+       /*
+        * More easily preempt - nice tasks, while not making
+        * it harder for + nice tasks.
+        */
+       if (unlikely(se->load.weight > NICE_0_LOAD))
+               gran = calc_delta_fair(gran, &se->load);
+
+       return gran;
+}
+
+/*
+ * Should 'se' preempt 'curr'.
+ *
+ *             |s1
+ *        |s2
+ *   |s3
+ *         g
+ *      |<--->|c
+ *
+ *  w(c, s1) = -1
+ *  w(c, s2) =  0
+ *  w(c, s3) =  1
+ *
+ */
+static int
+wakeup_preempt_entity(struct sched_entity *curr, struct sched_entity *se)
+{
+       s64 gran, vdiff = curr->vruntime - se->vruntime;
+
+       if (vdiff < 0)
+               return -1;
+
+       gran = wakeup_gran(curr);
+       if (vdiff > gran)
+               return 1;
+
+       return 0;
+}
 
 /*
  * Preempt the current task with a newly woken task if needed:
@@ -1124,7 +1138,6 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
        struct task_struct *curr = rq->curr;
        struct cfs_rq *cfs_rq = task_cfs_rq(curr);
        struct sched_entity *se = &curr->se, *pse = &p->se;
-       unsigned long gran;
 
        if (unlikely(rt_prio(p->prio))) {
                update_rq_clock(rq);
@@ -1154,15 +1167,7 @@ static void check_preempt_wakeup(struct rq *rq, struct task_struct *p)
                pse = parent_entity(pse);
        }
 
-       gran = sysctl_sched_wakeup_granularity;
-       /*
-        * More easily preempt - nice tasks, while not making
-        * it harder for + nice tasks.
-        */
-       if (unlikely(se->load.weight > NICE_0_LOAD))
-               gran = calc_delta_fair(gran, &se->load);
-
-       if (pse->vruntime + gran < se->vruntime)
+       if (wakeup_preempt_entity(se, pse) == 1)
                resched_task(curr);
 }