2 * drivers/cpufreq/cpufreq_governor.c
4 * CPUFREQ governors common code
6 * Copyright (C) 2001 Russell King
7 * (C) 2003 Venkatesh Pallipadi <venkatesh.pallipadi@intel.com>.
8 * (C) 2003 Jun Nakajima <jun.nakajima@intel.com>
9 * (C) 2009 Alexander Clouter <alex@digriz.org.uk>
10 * (c) 2012 Viresh Kumar <viresh.kumar@linaro.org>
12 * This program is free software; you can redistribute it and/or modify
13 * it under the terms of the GNU General Public License version 2 as
14 * published by the Free Software Foundation.
17 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
19 #include <linux/export.h>
20 #include <linux/kernel_stat.h>
21 #include <linux/slab.h>
23 #include "cpufreq_governor.h"
25 static struct attribute_group
*get_sysfs_attr(struct dbs_data
*dbs_data
)
27 if (have_governor_per_policy())
28 return dbs_data
->cdata
->attr_group_gov_pol
;
30 return dbs_data
->cdata
->attr_group_gov_sys
;
33 void dbs_check_cpu(struct dbs_data
*dbs_data
, int cpu
)
35 struct cpu_dbs_info
*cdbs
= dbs_data
->cdata
->get_cpu_cdbs(cpu
);
36 struct od_dbs_tuners
*od_tuners
= dbs_data
->tuners
;
37 struct cs_dbs_tuners
*cs_tuners
= dbs_data
->tuners
;
38 struct cpufreq_policy
*policy
= cdbs
->shared
->policy
;
39 unsigned int sampling_rate
;
40 unsigned int max_load
= 0;
41 unsigned int ignore_nice
;
44 if (dbs_data
->cdata
->governor
== GOV_ONDEMAND
) {
45 struct od_cpu_dbs_info_s
*od_dbs_info
=
46 dbs_data
->cdata
->get_cpu_dbs_info_s(cpu
);
49 * Sometimes, the ondemand governor uses an additional
50 * multiplier to give long delays. So apply this multiplier to
51 * the 'sampling_rate', so as to keep the wake-up-from-idle
52 * detection logic a bit conservative.
54 sampling_rate
= od_tuners
->sampling_rate
;
55 sampling_rate
*= od_dbs_info
->rate_mult
;
57 ignore_nice
= od_tuners
->ignore_nice_load
;
59 sampling_rate
= cs_tuners
->sampling_rate
;
60 ignore_nice
= cs_tuners
->ignore_nice_load
;
63 /* Get Absolute Load */
64 for_each_cpu(j
, policy
->cpus
) {
65 struct cpu_dbs_info
*j_cdbs
;
66 u64 cur_wall_time
, cur_idle_time
;
67 unsigned int idle_time
, wall_time
;
71 j_cdbs
= dbs_data
->cdata
->get_cpu_cdbs(j
);
74 * For the purpose of ondemand, waiting for disk IO is
75 * an indication that you're performance critical, and
76 * not that the system is actually idle. So do not add
77 * the iowait time to the cpu idle time.
79 if (dbs_data
->cdata
->governor
== GOV_ONDEMAND
)
80 io_busy
= od_tuners
->io_is_busy
;
81 cur_idle_time
= get_cpu_idle_time(j
, &cur_wall_time
, io_busy
);
83 wall_time
= (unsigned int)
84 (cur_wall_time
- j_cdbs
->prev_cpu_wall
);
85 j_cdbs
->prev_cpu_wall
= cur_wall_time
;
87 idle_time
= (unsigned int)
88 (cur_idle_time
- j_cdbs
->prev_cpu_idle
);
89 j_cdbs
->prev_cpu_idle
= cur_idle_time
;
93 unsigned long cur_nice_jiffies
;
95 cur_nice
= kcpustat_cpu(j
).cpustat
[CPUTIME_NICE
] -
98 * Assumption: nice time between sampling periods will
99 * be less than 2^32 jiffies for 32 bit sys
101 cur_nice_jiffies
= (unsigned long)
102 cputime64_to_jiffies64(cur_nice
);
104 cdbs
->prev_cpu_nice
=
105 kcpustat_cpu(j
).cpustat
[CPUTIME_NICE
];
106 idle_time
+= jiffies_to_usecs(cur_nice_jiffies
);
109 if (unlikely(!wall_time
|| wall_time
< idle_time
))
113 * If the CPU had gone completely idle, and a task just woke up
114 * on this CPU now, it would be unfair to calculate 'load' the
115 * usual way for this elapsed time-window, because it will show
116 * near-zero load, irrespective of how CPU intensive that task
117 * actually is. This is undesirable for latency-sensitive bursty
120 * To avoid this, we reuse the 'load' from the previous
121 * time-window and give this task a chance to start with a
122 * reasonably high CPU frequency. (However, we shouldn't over-do
123 * this copy, lest we get stuck at a high load (high frequency)
124 * for too long, even when the current system load has actually
125 * dropped down. So we perform the copy only once, upon the
126 * first wake-up from idle.)
128 * Detecting this situation is easy: the governor's deferrable
129 * timer would not have fired during CPU-idle periods. Hence
130 * an unusually large 'wall_time' (as compared to the sampling
131 * rate) indicates this scenario.
133 * prev_load can be zero in two cases and we must recalculate it
135 * - during long idle intervals
136 * - explicitly set to zero
138 if (unlikely(wall_time
> (2 * sampling_rate
) &&
139 j_cdbs
->prev_load
)) {
140 load
= j_cdbs
->prev_load
;
143 * Perform a destructive copy, to ensure that we copy
144 * the previous load only once, upon the first wake-up
147 j_cdbs
->prev_load
= 0;
149 load
= 100 * (wall_time
- idle_time
) / wall_time
;
150 j_cdbs
->prev_load
= load
;
157 dbs_data
->cdata
->gov_check_cpu(cpu
, max_load
);
159 EXPORT_SYMBOL_GPL(dbs_check_cpu
);
161 static inline void __gov_queue_work(int cpu
, struct dbs_data
*dbs_data
,
164 struct cpu_dbs_info
*cdbs
= dbs_data
->cdata
->get_cpu_cdbs(cpu
);
166 mod_delayed_work_on(cpu
, system_wq
, &cdbs
->dwork
, delay
);
169 void gov_queue_work(struct dbs_data
*dbs_data
, struct cpufreq_policy
*policy
,
170 unsigned int delay
, bool all_cpus
)
174 mutex_lock(&cpufreq_governor_lock
);
175 if (!policy
->governor_enabled
)
180 * Use raw_smp_processor_id() to avoid preemptible warnings.
181 * We know that this is only called with all_cpus == false from
182 * works that have been queued with *_work_on() functions and
183 * those works are canceled during CPU_DOWN_PREPARE so they
184 * can't possibly run on any other CPU.
186 __gov_queue_work(raw_smp_processor_id(), dbs_data
, delay
);
188 for_each_cpu(i
, policy
->cpus
)
189 __gov_queue_work(i
, dbs_data
, delay
);
193 mutex_unlock(&cpufreq_governor_lock
);
195 EXPORT_SYMBOL_GPL(gov_queue_work
);
197 static inline void gov_cancel_work(struct dbs_data
*dbs_data
,
198 struct cpufreq_policy
*policy
)
200 struct cpu_dbs_info
*cdbs
;
203 for_each_cpu(i
, policy
->cpus
) {
204 cdbs
= dbs_data
->cdata
->get_cpu_cdbs(i
);
205 cancel_delayed_work_sync(&cdbs
->dwork
);
209 /* Will return if we need to evaluate cpu load again or not */
210 static bool need_load_eval(struct cpu_common_dbs_info
*shared
,
211 unsigned int sampling_rate
)
213 if (policy_is_shared(shared
->policy
)) {
214 ktime_t time_now
= ktime_get();
215 s64 delta_us
= ktime_us_delta(time_now
, shared
->time_stamp
);
217 /* Do nothing if we recently have sampled */
218 if (delta_us
< (s64
)(sampling_rate
/ 2))
221 shared
->time_stamp
= time_now
;
227 static void dbs_timer(struct work_struct
*work
)
229 struct cpu_dbs_info
*cdbs
= container_of(work
, struct cpu_dbs_info
,
231 struct cpu_common_dbs_info
*shared
= cdbs
->shared
;
232 struct cpufreq_policy
*policy
= shared
->policy
;
233 struct dbs_data
*dbs_data
= policy
->governor_data
;
234 unsigned int sampling_rate
, delay
;
235 bool modify_all
= true;
237 mutex_lock(&shared
->timer_mutex
);
239 if (dbs_data
->cdata
->governor
== GOV_CONSERVATIVE
) {
240 struct cs_dbs_tuners
*cs_tuners
= dbs_data
->tuners
;
242 sampling_rate
= cs_tuners
->sampling_rate
;
244 struct od_dbs_tuners
*od_tuners
= dbs_data
->tuners
;
246 sampling_rate
= od_tuners
->sampling_rate
;
249 if (!need_load_eval(cdbs
->shared
, sampling_rate
))
252 delay
= dbs_data
->cdata
->gov_dbs_timer(cdbs
, dbs_data
, modify_all
);
253 gov_queue_work(dbs_data
, policy
, delay
, modify_all
);
255 mutex_unlock(&shared
->timer_mutex
);
258 static void set_sampling_rate(struct dbs_data
*dbs_data
,
259 unsigned int sampling_rate
)
261 if (dbs_data
->cdata
->governor
== GOV_CONSERVATIVE
) {
262 struct cs_dbs_tuners
*cs_tuners
= dbs_data
->tuners
;
263 cs_tuners
->sampling_rate
= sampling_rate
;
265 struct od_dbs_tuners
*od_tuners
= dbs_data
->tuners
;
266 od_tuners
->sampling_rate
= sampling_rate
;
270 static int alloc_common_dbs_info(struct cpufreq_policy
*policy
,
271 struct common_dbs_data
*cdata
)
273 struct cpu_common_dbs_info
*shared
;
276 /* Allocate memory for the common information for policy->cpus */
277 shared
= kzalloc(sizeof(*shared
), GFP_KERNEL
);
281 /* Set shared for all CPUs, online+offline */
282 for_each_cpu(j
, policy
->related_cpus
)
283 cdata
->get_cpu_cdbs(j
)->shared
= shared
;
288 static void free_common_dbs_info(struct cpufreq_policy
*policy
,
289 struct common_dbs_data
*cdata
)
291 struct cpu_dbs_info
*cdbs
= cdata
->get_cpu_cdbs(policy
->cpu
);
292 struct cpu_common_dbs_info
*shared
= cdbs
->shared
;
295 for_each_cpu(j
, policy
->cpus
)
296 cdata
->get_cpu_cdbs(j
)->shared
= NULL
;
301 static int cpufreq_governor_init(struct cpufreq_policy
*policy
,
302 struct dbs_data
*dbs_data
,
303 struct common_dbs_data
*cdata
)
305 unsigned int latency
;
308 /* State should be equivalent to EXIT */
309 if (policy
->governor_data
)
313 if (WARN_ON(have_governor_per_policy()))
316 ret
= alloc_common_dbs_info(policy
, cdata
);
320 dbs_data
->usage_count
++;
321 policy
->governor_data
= dbs_data
;
325 dbs_data
= kzalloc(sizeof(*dbs_data
), GFP_KERNEL
);
329 ret
= alloc_common_dbs_info(policy
, cdata
);
333 dbs_data
->cdata
= cdata
;
334 dbs_data
->usage_count
= 1;
336 ret
= cdata
->init(dbs_data
, !policy
->governor
->initialized
);
338 goto free_common_dbs_info
;
340 /* policy latency is in ns. Convert it to us first */
341 latency
= policy
->cpuinfo
.transition_latency
/ 1000;
345 /* Bring kernel and HW constraints together */
346 dbs_data
->min_sampling_rate
= max(dbs_data
->min_sampling_rate
,
347 MIN_LATENCY_MULTIPLIER
* latency
);
348 set_sampling_rate(dbs_data
, max(dbs_data
->min_sampling_rate
,
349 latency
* LATENCY_MULTIPLIER
));
351 if (!have_governor_per_policy())
352 cdata
->gdbs_data
= dbs_data
;
354 ret
= sysfs_create_group(get_governor_parent_kobj(policy
),
355 get_sysfs_attr(dbs_data
));
357 goto reset_gdbs_data
;
359 policy
->governor_data
= dbs_data
;
364 if (!have_governor_per_policy())
365 cdata
->gdbs_data
= NULL
;
366 cdata
->exit(dbs_data
, !policy
->governor
->initialized
);
367 free_common_dbs_info
:
368 free_common_dbs_info(policy
, cdata
);
374 static int cpufreq_governor_exit(struct cpufreq_policy
*policy
,
375 struct dbs_data
*dbs_data
)
377 struct common_dbs_data
*cdata
= dbs_data
->cdata
;
378 struct cpu_dbs_info
*cdbs
= cdata
->get_cpu_cdbs(policy
->cpu
);
380 /* State should be equivalent to INIT */
381 if (!cdbs
->shared
|| cdbs
->shared
->policy
)
384 policy
->governor_data
= NULL
;
385 if (!--dbs_data
->usage_count
) {
386 sysfs_remove_group(get_governor_parent_kobj(policy
),
387 get_sysfs_attr(dbs_data
));
389 if (!have_governor_per_policy())
390 cdata
->gdbs_data
= NULL
;
392 cdata
->exit(dbs_data
, policy
->governor
->initialized
== 1);
396 free_common_dbs_info(policy
, cdata
);
400 static int cpufreq_governor_start(struct cpufreq_policy
*policy
,
401 struct dbs_data
*dbs_data
)
403 struct common_dbs_data
*cdata
= dbs_data
->cdata
;
404 unsigned int sampling_rate
, ignore_nice
, j
, cpu
= policy
->cpu
;
405 struct cpu_dbs_info
*cdbs
= cdata
->get_cpu_cdbs(cpu
);
406 struct cpu_common_dbs_info
*shared
= cdbs
->shared
;
412 /* State should be equivalent to INIT */
413 if (!shared
|| shared
->policy
)
416 if (cdata
->governor
== GOV_CONSERVATIVE
) {
417 struct cs_dbs_tuners
*cs_tuners
= dbs_data
->tuners
;
419 sampling_rate
= cs_tuners
->sampling_rate
;
420 ignore_nice
= cs_tuners
->ignore_nice_load
;
422 struct od_dbs_tuners
*od_tuners
= dbs_data
->tuners
;
424 sampling_rate
= od_tuners
->sampling_rate
;
425 ignore_nice
= od_tuners
->ignore_nice_load
;
426 io_busy
= od_tuners
->io_is_busy
;
429 shared
->policy
= policy
;
430 shared
->time_stamp
= ktime_get();
431 mutex_init(&shared
->timer_mutex
);
433 for_each_cpu(j
, policy
->cpus
) {
434 struct cpu_dbs_info
*j_cdbs
= cdata
->get_cpu_cdbs(j
);
435 unsigned int prev_load
;
437 j_cdbs
->prev_cpu_idle
=
438 get_cpu_idle_time(j
, &j_cdbs
->prev_cpu_wall
, io_busy
);
440 prev_load
= (unsigned int)(j_cdbs
->prev_cpu_wall
-
441 j_cdbs
->prev_cpu_idle
);
442 j_cdbs
->prev_load
= 100 * prev_load
/
443 (unsigned int)j_cdbs
->prev_cpu_wall
;
446 j_cdbs
->prev_cpu_nice
= kcpustat_cpu(j
).cpustat
[CPUTIME_NICE
];
448 INIT_DEFERRABLE_WORK(&j_cdbs
->dwork
, dbs_timer
);
451 if (cdata
->governor
== GOV_CONSERVATIVE
) {
452 struct cs_cpu_dbs_info_s
*cs_dbs_info
=
453 cdata
->get_cpu_dbs_info_s(cpu
);
455 cs_dbs_info
->down_skip
= 0;
456 cs_dbs_info
->requested_freq
= policy
->cur
;
458 struct od_ops
*od_ops
= cdata
->gov_ops
;
459 struct od_cpu_dbs_info_s
*od_dbs_info
= cdata
->get_cpu_dbs_info_s(cpu
);
461 od_dbs_info
->rate_mult
= 1;
462 od_dbs_info
->sample_type
= OD_NORMAL_SAMPLE
;
463 od_ops
->powersave_bias_init_cpu(cpu
);
466 gov_queue_work(dbs_data
, policy
, delay_for_sampling_rate(sampling_rate
),
471 static int cpufreq_governor_stop(struct cpufreq_policy
*policy
,
472 struct dbs_data
*dbs_data
)
474 struct cpu_dbs_info
*cdbs
= dbs_data
->cdata
->get_cpu_cdbs(policy
->cpu
);
475 struct cpu_common_dbs_info
*shared
= cdbs
->shared
;
477 /* State should be equivalent to START */
478 if (!shared
|| !shared
->policy
)
481 gov_cancel_work(dbs_data
, policy
);
483 shared
->policy
= NULL
;
484 mutex_destroy(&shared
->timer_mutex
);
488 static int cpufreq_governor_limits(struct cpufreq_policy
*policy
,
489 struct dbs_data
*dbs_data
)
491 struct common_dbs_data
*cdata
= dbs_data
->cdata
;
492 unsigned int cpu
= policy
->cpu
;
493 struct cpu_dbs_info
*cdbs
= cdata
->get_cpu_cdbs(cpu
);
495 /* State should be equivalent to START */
496 if (!cdbs
->shared
|| !cdbs
->shared
->policy
)
499 mutex_lock(&cdbs
->shared
->timer_mutex
);
500 if (policy
->max
< cdbs
->shared
->policy
->cur
)
501 __cpufreq_driver_target(cdbs
->shared
->policy
, policy
->max
,
503 else if (policy
->min
> cdbs
->shared
->policy
->cur
)
504 __cpufreq_driver_target(cdbs
->shared
->policy
, policy
->min
,
506 dbs_check_cpu(dbs_data
, cpu
);
507 mutex_unlock(&cdbs
->shared
->timer_mutex
);
512 int cpufreq_governor_dbs(struct cpufreq_policy
*policy
,
513 struct common_dbs_data
*cdata
, unsigned int event
)
515 struct dbs_data
*dbs_data
;
518 /* Lock governor to block concurrent initialization of governor */
519 mutex_lock(&cdata
->mutex
);
521 if (have_governor_per_policy())
522 dbs_data
= policy
->governor_data
;
524 dbs_data
= cdata
->gdbs_data
;
526 if (!dbs_data
&& (event
!= CPUFREQ_GOV_POLICY_INIT
)) {
532 case CPUFREQ_GOV_POLICY_INIT
:
533 ret
= cpufreq_governor_init(policy
, dbs_data
, cdata
);
535 case CPUFREQ_GOV_POLICY_EXIT
:
536 ret
= cpufreq_governor_exit(policy
, dbs_data
);
538 case CPUFREQ_GOV_START
:
539 ret
= cpufreq_governor_start(policy
, dbs_data
);
541 case CPUFREQ_GOV_STOP
:
542 ret
= cpufreq_governor_stop(policy
, dbs_data
);
544 case CPUFREQ_GOV_LIMITS
:
545 ret
= cpufreq_governor_limits(policy
, dbs_data
);
552 mutex_unlock(&cdata
->mutex
);
556 EXPORT_SYMBOL_GPL(cpufreq_governor_dbs
);
This page took 0.060271 seconds and 6 git commands to generate.