2 * drivers/cpufreq/cpufreq_governor.c
4 * CPUFREQ governors common code
6 * Copyright (C) 2001 Russell King
7 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
8 * (C) 2003 Jun Nakajima <jun.nakajima@intel.com>
9 * (C) 2009 Alexander Clouter <alex@digriz.org.uk>
10 * (c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19 #include <linux/export.h>
20 #include <linux/kernel_stat.h>
21 #include <linux/sched.h>
22 #include <linux/slab.h>
24 #include "cpufreq_governor.h"
26 static DEFINE_PER_CPU(struct cpu_dbs_info
, cpu_dbs
);
28 static DEFINE_MUTEX(gov_dbs_data_mutex
);
30 /* Common sysfs tunables */
32 * store_sampling_rate - update sampling rate effective immediately if needed.
34 * If new rate is smaller than the old, simply updating
35 * dbs.sampling_rate might not be appropriate. For example, if the
36 * original sampling_rate was 1 second and the requested new sampling rate is 10
37 * ms because the user needs immediate reaction from ondemand governor, but not
38 * sure if higher frequency will be required or not, then, the governor may
39 * change the sampling rate too late; up to 1 second later. Thus, if we are
40 * reducing the sampling rate, we need to make the new value effective
43 * This must be called with dbs_data->mutex held, otherwise traversing
44 * policy_dbs_list isn't safe.
46 ssize_t
store_sampling_rate(struct gov_attr_set
*attr_set
, const char *buf
,
49 struct dbs_data
*dbs_data
= to_dbs_data(attr_set
);
50 struct policy_dbs_info
*policy_dbs
;
53 ret
= sscanf(buf
, "%u", &rate
);
57 dbs_data
->sampling_rate
= max(rate
, dbs_data
->min_sampling_rate
);
60 * We are operating under dbs_data->mutex and so the list and its
61 * entries can't be freed concurrently.
63 list_for_each_entry(policy_dbs
, &attr_set
->policy_list
, list
) {
64 mutex_lock(&policy_dbs
->timer_mutex
);
66 * On 32-bit architectures this may race with the
67 * sample_delay_ns read in dbs_update_util_handler(), but that
68 * really doesn't matter. If the read returns a value that's
69 * too big, the sample will be skipped, but the next invocation
70 * of dbs_update_util_handler() (when the update has been
71 * completed) will take a sample.
73 * If this runs in parallel with dbs_work_handler(), we may end
74 * up overwriting the sample_delay_ns value that it has just
75 * written, but it will be corrected next time a sample is
76 * taken, so it shouldn't be significant.
78 gov_update_sample_delay(policy_dbs
, 0);
79 mutex_unlock(&policy_dbs
->timer_mutex
);
84 EXPORT_SYMBOL_GPL(store_sampling_rate
);
87 * gov_update_cpu_data - Update CPU load data.
88 * @dbs_data: Top-level governor data pointer.
90 * Update CPU load data for all CPUs in the domain governed by @dbs_data
91 * (that may be a single policy or a bunch of them if governor tunables are
94 * Call under the @dbs_data mutex.
96 void gov_update_cpu_data(struct dbs_data
*dbs_data
)
98 struct policy_dbs_info
*policy_dbs
;
100 list_for_each_entry(policy_dbs
, &dbs_data
->attr_set
.policy_list
, list
) {
103 for_each_cpu(j
, policy_dbs
->policy
->cpus
) {
104 struct cpu_dbs_info
*j_cdbs
= &per_cpu(cpu_dbs
, j
);
106 j_cdbs
->prev_cpu_idle
= get_cpu_idle_time(j
, &j_cdbs
->prev_cpu_wall
,
107 dbs_data
->io_is_busy
);
108 if (dbs_data
->ignore_nice_load
)
109 j_cdbs
->prev_cpu_nice
= kcpustat_cpu(j
).cpustat
[CPUTIME_NICE
];
113 EXPORT_SYMBOL_GPL(gov_update_cpu_data
);
115 static inline struct gov_attr_set
*to_gov_attr_set(struct kobject
*kobj
)
117 return container_of(kobj
, struct gov_attr_set
, kobj
);
120 static inline struct governor_attr
*to_gov_attr(struct attribute
*attr
)
122 return container_of(attr
, struct governor_attr
, attr
);
125 static ssize_t
governor_show(struct kobject
*kobj
, struct attribute
*attr
,
128 struct governor_attr
*gattr
= to_gov_attr(attr
);
130 return gattr
->show(to_gov_attr_set(kobj
), buf
);
133 static ssize_t
governor_store(struct kobject
*kobj
, struct attribute
*attr
,
134 const char *buf
, size_t count
)
136 struct gov_attr_set
*attr_set
= to_gov_attr_set(kobj
);
137 struct governor_attr
*gattr
= to_gov_attr(attr
);
140 mutex_lock(&attr_set
->update_lock
);
142 if (attr_set
->usage_count
)
143 ret
= gattr
->store(attr_set
, buf
, count
);
145 mutex_unlock(&attr_set
->update_lock
);
151 * Sysfs Ops for accessing governor attributes.
153 * All show/store invocations for governor specific sysfs attributes, will first
154 * call the below show/store callbacks and the attribute specific callback will
155 * be called from within it.
157 static const struct sysfs_ops governor_sysfs_ops
= {
158 .show
= governor_show
,
159 .store
= governor_store
,
162 unsigned int dbs_update(struct cpufreq_policy
*policy
)
164 struct policy_dbs_info
*policy_dbs
= policy
->governor_data
;
165 struct dbs_data
*dbs_data
= policy_dbs
->dbs_data
;
166 unsigned int ignore_nice
= dbs_data
->ignore_nice_load
;
167 unsigned int max_load
= 0;
168 unsigned int sampling_rate
, io_busy
, j
;
171 * Sometimes governors may use an additional multiplier to increase
172 * sample delays temporarily. Apply that multiplier to sampling_rate
173 * so as to keep the wake-up-from-idle detection logic a bit
176 sampling_rate
= dbs_data
->sampling_rate
* policy_dbs
->rate_mult
;
178 * For the purpose of ondemand, waiting for disk IO is an indication
179 * that you're performance critical, and not that the system is actually
180 * idle, so do not add the iowait time to the CPU idle time then.
182 io_busy
= dbs_data
->io_is_busy
;
184 /* Get Absolute Load */
185 for_each_cpu(j
, policy
->cpus
) {
186 struct cpu_dbs_info
*j_cdbs
= &per_cpu(cpu_dbs
, j
);
187 u64 cur_wall_time
, cur_idle_time
;
188 unsigned int idle_time
, wall_time
;
191 cur_idle_time
= get_cpu_idle_time(j
, &cur_wall_time
, io_busy
);
193 wall_time
= cur_wall_time
- j_cdbs
->prev_cpu_wall
;
194 j_cdbs
->prev_cpu_wall
= cur_wall_time
;
196 if (cur_idle_time
<= j_cdbs
->prev_cpu_idle
) {
199 idle_time
= cur_idle_time
- j_cdbs
->prev_cpu_idle
;
200 j_cdbs
->prev_cpu_idle
= cur_idle_time
;
204 u64 cur_nice
= kcpustat_cpu(j
).cpustat
[CPUTIME_NICE
];
206 idle_time
+= cputime_to_usecs(cur_nice
- j_cdbs
->prev_cpu_nice
);
207 j_cdbs
->prev_cpu_nice
= cur_nice
;
210 if (unlikely(!wall_time
|| wall_time
< idle_time
))
214 * If the CPU had gone completely idle, and a task just woke up
215 * on this CPU now, it would be unfair to calculate 'load' the
216 * usual way for this elapsed time-window, because it will show
217 * near-zero load, irrespective of how CPU intensive that task
218 * actually is. This is undesirable for latency-sensitive bursty
221 * To avoid this, we reuse the 'load' from the previous
222 * time-window and give this task a chance to start with a
223 * reasonably high CPU frequency. (However, we shouldn't over-do
224 * this copy, lest we get stuck at a high load (high frequency)
225 * for too long, even when the current system load has actually
226 * dropped down. So we perform the copy only once, upon the
227 * first wake-up from idle.)
229 * Detecting this situation is easy: the governor's utilization
230 * update handler would not have run during CPU-idle periods.
231 * Hence, an unusually large 'wall_time' (as compared to the
232 * sampling rate) indicates this scenario.
234 * prev_load can be zero in two cases and we must recalculate it
236 * - during long idle intervals
237 * - explicitly set to zero
239 if (unlikely(wall_time
> (2 * sampling_rate
) &&
240 j_cdbs
->prev_load
)) {
241 load
= j_cdbs
->prev_load
;
244 * Perform a destructive copy, to ensure that we copy
245 * the previous load only once, upon the first wake-up
248 j_cdbs
->prev_load
= 0;
250 load
= 100 * (wall_time
- idle_time
) / wall_time
;
251 j_cdbs
->prev_load
= load
;
259 EXPORT_SYMBOL_GPL(dbs_update
);
261 static void dbs_work_handler(struct work_struct
*work
)
263 struct policy_dbs_info
*policy_dbs
;
264 struct cpufreq_policy
*policy
;
265 struct dbs_governor
*gov
;
267 policy_dbs
= container_of(work
, struct policy_dbs_info
, work
);
268 policy
= policy_dbs
->policy
;
269 gov
= dbs_governor_of(policy
);
272 * Make sure cpufreq_governor_limits() isn't evaluating load or the
273 * ondemand governor isn't updating the sampling rate in parallel.
275 mutex_lock(&policy_dbs
->timer_mutex
);
276 gov_update_sample_delay(policy_dbs
, gov
->gov_dbs_timer(policy
));
277 mutex_unlock(&policy_dbs
->timer_mutex
);
279 /* Allow the utilization update handler to queue up more work. */
280 atomic_set(&policy_dbs
->work_count
, 0);
282 * If the update below is reordered with respect to the sample delay
283 * modification, the utilization update handler may end up using a stale
284 * sample delay value.
287 policy_dbs
->work_in_progress
= false;
290 static void dbs_irq_work(struct irq_work
*irq_work
)
292 struct policy_dbs_info
*policy_dbs
;
294 policy_dbs
= container_of(irq_work
, struct policy_dbs_info
, irq_work
);
295 schedule_work_on(smp_processor_id(), &policy_dbs
->work
);
298 static void dbs_update_util_handler(struct update_util_data
*data
, u64 time
,
299 unsigned long util
, unsigned long max
)
301 struct cpu_dbs_info
*cdbs
= container_of(data
, struct cpu_dbs_info
, update_util
);
302 struct policy_dbs_info
*policy_dbs
= cdbs
->policy_dbs
;
306 * The work may not be allowed to be queued up right now.
308 * - Work has already been queued up or is in progress.
309 * - It is too early (too little time from the previous sample).
311 if (policy_dbs
->work_in_progress
)
315 * If the reads below are reordered before the check above, the value
316 * of sample_delay_ns used in the computation may be stale.
319 lst
= READ_ONCE(policy_dbs
->last_sample_time
);
320 delta_ns
= time
- lst
;
321 if ((s64
)delta_ns
< policy_dbs
->sample_delay_ns
)
325 * If the policy is not shared, the irq_work may be queued up right away
326 * at this point. Otherwise, we need to ensure that only one of the
327 * CPUs sharing the policy will do that.
329 if (policy_dbs
->is_shared
) {
330 if (!atomic_add_unless(&policy_dbs
->work_count
, 1, 1))
334 * If another CPU updated last_sample_time in the meantime, we
335 * shouldn't be here, so clear the work counter and bail out.
337 if (unlikely(lst
!= READ_ONCE(policy_dbs
->last_sample_time
))) {
338 atomic_set(&policy_dbs
->work_count
, 0);
343 policy_dbs
->last_sample_time
= time
;
344 policy_dbs
->work_in_progress
= true;
345 irq_work_queue(&policy_dbs
->irq_work
);
348 static void gov_set_update_util(struct policy_dbs_info
*policy_dbs
,
349 unsigned int delay_us
)
351 struct cpufreq_policy
*policy
= policy_dbs
->policy
;
354 gov_update_sample_delay(policy_dbs
, delay_us
);
355 policy_dbs
->last_sample_time
= 0;
357 for_each_cpu(cpu
, policy
->cpus
) {
358 struct cpu_dbs_info
*cdbs
= &per_cpu(cpu_dbs
, cpu
);
360 cpufreq_add_update_util_hook(cpu
, &cdbs
->update_util
,
361 dbs_update_util_handler
);
365 static inline void gov_clear_update_util(struct cpufreq_policy
*policy
)
369 for_each_cpu(i
, policy
->cpus
)
370 cpufreq_remove_update_util_hook(i
);
375 static void gov_cancel_work(struct cpufreq_policy
*policy
)
377 struct policy_dbs_info
*policy_dbs
= policy
->governor_data
;
379 gov_clear_update_util(policy_dbs
->policy
);
380 irq_work_sync(&policy_dbs
->irq_work
);
381 cancel_work_sync(&policy_dbs
->work
);
382 atomic_set(&policy_dbs
->work_count
, 0);
383 policy_dbs
->work_in_progress
= false;
386 static struct policy_dbs_info
*alloc_policy_dbs_info(struct cpufreq_policy
*policy
,
387 struct dbs_governor
*gov
)
389 struct policy_dbs_info
*policy_dbs
;
392 /* Allocate memory for per-policy governor data. */
393 policy_dbs
= gov
->alloc();
397 policy_dbs
->policy
= policy
;
398 mutex_init(&policy_dbs
->timer_mutex
);
399 atomic_set(&policy_dbs
->work_count
, 0);
400 init_irq_work(&policy_dbs
->irq_work
, dbs_irq_work
);
401 INIT_WORK(&policy_dbs
->work
, dbs_work_handler
);
403 /* Set policy_dbs for all CPUs, online+offline */
404 for_each_cpu(j
, policy
->related_cpus
) {
405 struct cpu_dbs_info
*j_cdbs
= &per_cpu(cpu_dbs
, j
);
407 j_cdbs
->policy_dbs
= policy_dbs
;
412 static void free_policy_dbs_info(struct policy_dbs_info
*policy_dbs
,
413 struct dbs_governor
*gov
)
417 mutex_destroy(&policy_dbs
->timer_mutex
);
419 for_each_cpu(j
, policy_dbs
->policy
->related_cpus
) {
420 struct cpu_dbs_info
*j_cdbs
= &per_cpu(cpu_dbs
, j
);
422 j_cdbs
->policy_dbs
= NULL
;
423 j_cdbs
->update_util
.func
= NULL
;
425 gov
->free(policy_dbs
);
428 static void gov_attr_set_init(struct gov_attr_set
*attr_set
,
429 struct list_head
*list_node
)
431 INIT_LIST_HEAD(&attr_set
->policy_list
);
432 mutex_init(&attr_set
->update_lock
);
433 attr_set
->usage_count
= 1;
434 list_add(list_node
, &attr_set
->policy_list
);
437 static void gov_attr_set_get(struct gov_attr_set
*attr_set
,
438 struct list_head
*list_node
)
440 mutex_lock(&attr_set
->update_lock
);
441 attr_set
->usage_count
++;
442 list_add(list_node
, &attr_set
->policy_list
);
443 mutex_unlock(&attr_set
->update_lock
);
446 static unsigned int gov_attr_set_put(struct gov_attr_set
*attr_set
,
447 struct list_head
*list_node
)
451 mutex_lock(&attr_set
->update_lock
);
453 count
= --attr_set
->usage_count
;
454 mutex_unlock(&attr_set
->update_lock
);
458 kobject_put(&attr_set
->kobj
);
459 mutex_destroy(&attr_set
->update_lock
);
463 static int cpufreq_governor_init(struct cpufreq_policy
*policy
)
465 struct dbs_governor
*gov
= dbs_governor_of(policy
);
466 struct dbs_data
*dbs_data
;
467 struct policy_dbs_info
*policy_dbs
;
468 unsigned int latency
;
471 /* State should be equivalent to EXIT */
472 if (policy
->governor_data
)
475 policy_dbs
= alloc_policy_dbs_info(policy
, gov
);
479 /* Protect gov->gdbs_data against concurrent updates. */
480 mutex_lock(&gov_dbs_data_mutex
);
482 dbs_data
= gov
->gdbs_data
;
484 if (WARN_ON(have_governor_per_policy())) {
486 goto free_policy_dbs_info
;
488 policy_dbs
->dbs_data
= dbs_data
;
489 policy
->governor_data
= policy_dbs
;
491 gov_attr_set_get(&dbs_data
->attr_set
, &policy_dbs
->list
);
495 dbs_data
= kzalloc(sizeof(*dbs_data
), GFP_KERNEL
);
498 goto free_policy_dbs_info
;
501 gov_attr_set_init(&dbs_data
->attr_set
, &policy_dbs
->list
);
503 ret
= gov
->init(dbs_data
, !policy
->governor
->initialized
);
505 goto free_policy_dbs_info
;
507 /* policy latency is in ns. Convert it to us first */
508 latency
= policy
->cpuinfo
.transition_latency
/ 1000;
512 /* Bring kernel and HW constraints together */
513 dbs_data
->min_sampling_rate
= max(dbs_data
->min_sampling_rate
,
514 MIN_LATENCY_MULTIPLIER
* latency
);
515 dbs_data
->sampling_rate
= max(dbs_data
->min_sampling_rate
,
516 LATENCY_MULTIPLIER
* latency
);
518 if (!have_governor_per_policy())
519 gov
->gdbs_data
= dbs_data
;
521 policy_dbs
->dbs_data
= dbs_data
;
522 policy
->governor_data
= policy_dbs
;
524 gov
->kobj_type
.sysfs_ops
= &governor_sysfs_ops
;
525 ret
= kobject_init_and_add(&dbs_data
->attr_set
.kobj
, &gov
->kobj_type
,
526 get_governor_parent_kobj(policy
),
527 "%s", gov
->gov
.name
);
531 /* Failure, so roll back. */
532 pr_err("cpufreq: Governor initialization failed (dbs_data kobject init error %d)\n", ret
);
534 policy
->governor_data
= NULL
;
536 if (!have_governor_per_policy())
537 gov
->gdbs_data
= NULL
;
538 gov
->exit(dbs_data
, !policy
->governor
->initialized
);
541 free_policy_dbs_info
:
542 free_policy_dbs_info(policy_dbs
, gov
);
545 mutex_unlock(&gov_dbs_data_mutex
);
549 static int cpufreq_governor_exit(struct cpufreq_policy
*policy
)
551 struct dbs_governor
*gov
= dbs_governor_of(policy
);
552 struct policy_dbs_info
*policy_dbs
= policy
->governor_data
;
553 struct dbs_data
*dbs_data
= policy_dbs
->dbs_data
;
556 /* Protect gov->gdbs_data against concurrent updates. */
557 mutex_lock(&gov_dbs_data_mutex
);
559 count
= gov_attr_set_put(&dbs_data
->attr_set
, &policy_dbs
->list
);
561 policy
->governor_data
= NULL
;
564 if (!have_governor_per_policy())
565 gov
->gdbs_data
= NULL
;
567 gov
->exit(dbs_data
, policy
->governor
->initialized
== 1);
571 free_policy_dbs_info(policy_dbs
, gov
);
573 mutex_unlock(&gov_dbs_data_mutex
);
577 static int cpufreq_governor_start(struct cpufreq_policy
*policy
)
579 struct dbs_governor
*gov
= dbs_governor_of(policy
);
580 struct policy_dbs_info
*policy_dbs
= policy
->governor_data
;
581 struct dbs_data
*dbs_data
= policy_dbs
->dbs_data
;
582 unsigned int sampling_rate
, ignore_nice
, j
;
583 unsigned int io_busy
;
588 policy_dbs
->is_shared
= policy_is_shared(policy
);
589 policy_dbs
->rate_mult
= 1;
591 sampling_rate
= dbs_data
->sampling_rate
;
592 ignore_nice
= dbs_data
->ignore_nice_load
;
593 io_busy
= dbs_data
->io_is_busy
;
595 for_each_cpu(j
, policy
->cpus
) {
596 struct cpu_dbs_info
*j_cdbs
= &per_cpu(cpu_dbs
, j
);
597 unsigned int prev_load
;
599 j_cdbs
->prev_cpu_idle
= get_cpu_idle_time(j
, &j_cdbs
->prev_cpu_wall
, io_busy
);
601 prev_load
= j_cdbs
->prev_cpu_wall
- j_cdbs
->prev_cpu_idle
;
602 j_cdbs
->prev_load
= 100 * prev_load
/ (unsigned int)j_cdbs
->prev_cpu_wall
;
605 j_cdbs
->prev_cpu_nice
= kcpustat_cpu(j
).cpustat
[CPUTIME_NICE
];
610 gov_set_update_util(policy_dbs
, sampling_rate
);
614 static int cpufreq_governor_stop(struct cpufreq_policy
*policy
)
616 gov_cancel_work(policy
);
620 static int cpufreq_governor_limits(struct cpufreq_policy
*policy
)
622 struct policy_dbs_info
*policy_dbs
= policy
->governor_data
;
624 mutex_lock(&policy_dbs
->timer_mutex
);
626 if (policy
->max
< policy
->cur
)
627 __cpufreq_driver_target(policy
, policy
->max
, CPUFREQ_RELATION_H
);
628 else if (policy
->min
> policy
->cur
)
629 __cpufreq_driver_target(policy
, policy
->min
, CPUFREQ_RELATION_L
);
631 gov_update_sample_delay(policy_dbs
, 0);
633 mutex_unlock(&policy_dbs
->timer_mutex
);
638 int cpufreq_governor_dbs(struct cpufreq_policy
*policy
, unsigned int event
)
640 if (event
== CPUFREQ_GOV_POLICY_INIT
) {
641 return cpufreq_governor_init(policy
);
642 } else if (policy
->governor_data
) {
644 case CPUFREQ_GOV_POLICY_EXIT
:
645 return cpufreq_governor_exit(policy
);
646 case CPUFREQ_GOV_START
:
647 return cpufreq_governor_start(policy
);
648 case CPUFREQ_GOV_STOP
:
649 return cpufreq_governor_stop(policy
);
650 case CPUFREQ_GOV_LIMITS
:
651 return cpufreq_governor_limits(policy
);
656 EXPORT_SYMBOL_GPL(cpufreq_governor_dbs
);