sched, nohz: Change rq->nr_running to always use wrappers

[deliverable/linux.git] / kernel / sched / sched.h

diff --git a/kernel/sched/sched.h b/kernel/sched/sched.h
index b2cbe81308afe79c40944196f4fc398895bc676e..600e2291a75c4092dd6ff400f869c91edcc7903f 100644 (file)
--- a/kernel/sched/sched.h
+++ b/kernel/sched/sched.h
@@ -1206,12 +1206,14 @@ extern void update_idle_cpu_load(struct rq *this_rq);
 
 extern void init_task_runnable_average(struct task_struct *p);
 
-static inline void inc_nr_running(struct rq *rq)
+static inline void add_nr_running(struct rq *rq, unsigned count)
 {
-       rq->nr_running++;
+       unsigned prev_nr = rq->nr_running;
+
+       rq->nr_running = prev_nr + count;
 
 #ifdef CONFIG_NO_HZ_FULL
-       if (rq->nr_running == 2) {
+       if (prev_nr < 2 && rq->nr_running >= 2) {
                if (tick_nohz_full_cpu(rq->cpu)) {
                        /* Order rq->nr_running write against the IPI */
                        smp_wmb();
@@ -1221,9 +1223,9 @@ static inline void inc_nr_running(struct rq *rq)
 #endif
 }
 
-static inline void dec_nr_running(struct rq *rq)
+static inline void sub_nr_running(struct rq *rq, unsigned count)
 {
-       rq->nr_running--;
+       rq->nr_running -= count;
 }
 
 static inline void rq_last_tick_reset(struct rq *rq)