2 * intel_pstate.c: Native P state management for Intel processors
4 * (C) Copyright 2012 Intel Corporation
5 * Author: Dirk Brandewie <dirk.j.brandewie@intel.com>
7 * This program is free software; you can redistribute it and/or
8 * modify it under the terms of the GNU General Public License
9 * as published by the Free Software Foundation; version 2
13 #include <linux/kernel.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/module.h>
16 #include <linux/ktime.h>
17 #include <linux/hrtimer.h>
18 #include <linux/tick.h>
19 #include <linux/slab.h>
20 #include <linux/sched.h>
21 #include <linux/list.h>
22 #include <linux/cpu.h>
23 #include <linux/cpufreq.h>
24 #include <linux/sysfs.h>
25 #include <linux/types.h>
27 #include <linux/debugfs.h>
28 #include <trace/events/power.h>
30 #include <asm/div64.h>
32 #include <asm/cpu_device_id.h>
34 #define SAMPLE_COUNT 3
37 #define int_tofp(X) ((int64_t)(X) << FRAC_BITS)
38 #define fp_toint(X) ((X) >> FRAC_BITS)
40 static inline int32_t mul_fp(int32_t x
, int32_t y
)
42 return ((int64_t)x
* (int64_t)y
) >> FRAC_BITS
;
45 static inline int32_t div_fp(int32_t x
, int32_t y
)
47 return div_s64((int64_t)x
<< FRAC_BITS
, (int64_t)y
);
79 struct timer_list timer
;
81 struct pstate_adjust_policy
*pstate_policy
;
82 struct pstate_data pstate
;
90 struct sample samples
[SAMPLE_COUNT
];
93 static struct cpudata
**all_cpu_data
;
94 struct pstate_adjust_policy
{
103 static struct pstate_adjust_policy default_policy
= {
104 .sample_rate_ms
= 10,
122 static struct perf_limits limits
= {
125 .max_perf
= int_tofp(1),
128 .max_policy_pct
= 100,
129 .max_sysfs_pct
= 100,
132 static inline void pid_reset(struct _pid
*pid
, int setpoint
, int busy
,
133 int deadband
, int integral
) {
134 pid
->setpoint
= setpoint
;
135 pid
->deadband
= deadband
;
136 pid
->integral
= int_tofp(integral
);
137 pid
->last_err
= setpoint
- busy
;
140 static inline void pid_p_gain_set(struct _pid
*pid
, int percent
)
142 pid
->p_gain
= div_fp(int_tofp(percent
), int_tofp(100));
145 static inline void pid_i_gain_set(struct _pid
*pid
, int percent
)
147 pid
->i_gain
= div_fp(int_tofp(percent
), int_tofp(100));
150 static inline void pid_d_gain_set(struct _pid
*pid
, int percent
)
153 pid
->d_gain
= div_fp(int_tofp(percent
), int_tofp(100));
156 static signed int pid_calc(struct _pid
*pid
, int busy
)
158 signed int err
, result
;
159 int32_t pterm
, dterm
, fp_error
;
160 int32_t integral_limit
;
162 err
= pid
->setpoint
- busy
;
163 fp_error
= int_tofp(err
);
165 if (abs(err
) <= pid
->deadband
)
168 pterm
= mul_fp(pid
->p_gain
, fp_error
);
170 pid
->integral
+= fp_error
;
172 /* limit the integral term */
173 integral_limit
= int_tofp(30);
174 if (pid
->integral
> integral_limit
)
175 pid
->integral
= integral_limit
;
176 if (pid
->integral
< -integral_limit
)
177 pid
->integral
= -integral_limit
;
179 dterm
= mul_fp(pid
->d_gain
, (err
- pid
->last_err
));
182 result
= pterm
+ mul_fp(pid
->integral
, pid
->i_gain
) + dterm
;
184 return (signed int)fp_toint(result
);
187 static inline void intel_pstate_busy_pid_reset(struct cpudata
*cpu
)
189 pid_p_gain_set(&cpu
->pid
, cpu
->pstate_policy
->p_gain_pct
);
190 pid_d_gain_set(&cpu
->pid
, cpu
->pstate_policy
->d_gain_pct
);
191 pid_i_gain_set(&cpu
->pid
, cpu
->pstate_policy
->i_gain_pct
);
194 cpu
->pstate_policy
->setpoint
,
196 cpu
->pstate_policy
->deadband
,
200 static inline void intel_pstate_reset_all_pid(void)
203 for_each_online_cpu(cpu
) {
204 if (all_cpu_data
[cpu
])
205 intel_pstate_busy_pid_reset(all_cpu_data
[cpu
]);
209 /************************** debugfs begin ************************/
210 static int pid_param_set(void *data
, u64 val
)
213 intel_pstate_reset_all_pid();
216 static int pid_param_get(void *data
, u64
*val
)
221 DEFINE_SIMPLE_ATTRIBUTE(fops_pid_param
, pid_param_get
,
222 pid_param_set
, "%llu\n");
229 static struct pid_param pid_files
[] = {
230 {"sample_rate_ms", &default_policy
.sample_rate_ms
},
231 {"d_gain_pct", &default_policy
.d_gain_pct
},
232 {"i_gain_pct", &default_policy
.i_gain_pct
},
233 {"deadband", &default_policy
.deadband
},
234 {"setpoint", &default_policy
.setpoint
},
235 {"p_gain_pct", &default_policy
.p_gain_pct
},
239 static struct dentry
*debugfs_parent
;
240 static void intel_pstate_debug_expose_params(void)
244 debugfs_parent
= debugfs_create_dir("pstate_snb", NULL
);
245 if (IS_ERR_OR_NULL(debugfs_parent
))
247 while (pid_files
[i
].name
) {
248 debugfs_create_file(pid_files
[i
].name
, 0660,
249 debugfs_parent
, pid_files
[i
].value
,
255 /************************** debugfs end ************************/
257 /************************** sysfs begin ************************/
258 #define show_one(file_name, object) \
259 static ssize_t show_##file_name \
260 (struct kobject *kobj, struct attribute *attr, char *buf) \
262 return sprintf(buf, "%u\n", limits.object); \
265 static ssize_t
store_no_turbo(struct kobject
*a
, struct attribute
*b
,
266 const char *buf
, size_t count
)
270 ret
= sscanf(buf
, "%u", &input
);
273 limits
.no_turbo
= clamp_t(int, input
, 0 , 1);
278 static ssize_t
store_max_perf_pct(struct kobject
*a
, struct attribute
*b
,
279 const char *buf
, size_t count
)
283 ret
= sscanf(buf
, "%u", &input
);
287 limits
.max_sysfs_pct
= clamp_t(int, input
, 0 , 100);
288 limits
.max_perf_pct
= min(limits
.max_policy_pct
, limits
.max_sysfs_pct
);
289 limits
.max_perf
= div_fp(int_tofp(limits
.max_perf_pct
), int_tofp(100));
293 static ssize_t
store_min_perf_pct(struct kobject
*a
, struct attribute
*b
,
294 const char *buf
, size_t count
)
298 ret
= sscanf(buf
, "%u", &input
);
301 limits
.min_perf_pct
= clamp_t(int, input
, 0 , 100);
302 limits
.min_perf
= div_fp(int_tofp(limits
.min_perf_pct
), int_tofp(100));
307 show_one(no_turbo
, no_turbo
);
308 show_one(max_perf_pct
, max_perf_pct
);
309 show_one(min_perf_pct
, min_perf_pct
);
311 define_one_global_rw(no_turbo
);
312 define_one_global_rw(max_perf_pct
);
313 define_one_global_rw(min_perf_pct
);
315 static struct attribute
*intel_pstate_attributes
[] = {
322 static struct attribute_group intel_pstate_attr_group
= {
323 .attrs
= intel_pstate_attributes
,
325 static struct kobject
*intel_pstate_kobject
;
327 static void intel_pstate_sysfs_expose_params(void)
331 intel_pstate_kobject
= kobject_create_and_add("intel_pstate",
332 &cpu_subsys
.dev_root
->kobj
);
333 BUG_ON(!intel_pstate_kobject
);
334 rc
= sysfs_create_group(intel_pstate_kobject
,
335 &intel_pstate_attr_group
);
339 /************************** sysfs end ************************/
341 static int intel_pstate_min_pstate(void)
344 rdmsrl(MSR_PLATFORM_INFO
, value
);
345 return (value
>> 40) & 0xFF;
348 static int intel_pstate_max_pstate(void)
351 rdmsrl(MSR_PLATFORM_INFO
, value
);
352 return (value
>> 8) & 0xFF;
355 static int intel_pstate_turbo_pstate(void)
359 rdmsrl(MSR_NHM_TURBO_RATIO_LIMIT
, value
);
360 nont
= intel_pstate_max_pstate();
361 ret
= ((value
) & 255);
367 static void intel_pstate_get_min_max(struct cpudata
*cpu
, int *min
, int *max
)
369 int max_perf
= cpu
->pstate
.turbo_pstate
;
372 max_perf
= cpu
->pstate
.max_pstate
;
374 max_perf
= fp_toint(mul_fp(int_tofp(max_perf
), limits
.max_perf
));
375 *max
= clamp_t(int, max_perf
,
376 cpu
->pstate
.min_pstate
, cpu
->pstate
.turbo_pstate
);
378 min_perf
= fp_toint(mul_fp(int_tofp(max_perf
), limits
.min_perf
));
379 *min
= clamp_t(int, min_perf
,
380 cpu
->pstate
.min_pstate
, max_perf
);
383 static void intel_pstate_set_pstate(struct cpudata
*cpu
, int pstate
)
385 int max_perf
, min_perf
;
387 intel_pstate_get_min_max(cpu
, &min_perf
, &max_perf
);
389 pstate
= clamp_t(int, pstate
, min_perf
, max_perf
);
391 if (pstate
== cpu
->pstate
.current_pstate
)
394 trace_cpu_frequency(pstate
* 100000, cpu
->cpu
);
396 cpu
->pstate
.current_pstate
= pstate
;
398 wrmsrl(MSR_IA32_PERF_CTL
, BIT(32) | (pstate
<< 8));
400 wrmsrl(MSR_IA32_PERF_CTL
, pstate
<< 8);
404 static inline void intel_pstate_pstate_increase(struct cpudata
*cpu
, int steps
)
407 target
= cpu
->pstate
.current_pstate
+ steps
;
409 intel_pstate_set_pstate(cpu
, target
);
412 static inline void intel_pstate_pstate_decrease(struct cpudata
*cpu
, int steps
)
415 target
= cpu
->pstate
.current_pstate
- steps
;
416 intel_pstate_set_pstate(cpu
, target
);
419 static void intel_pstate_get_cpu_pstates(struct cpudata
*cpu
)
421 sprintf(cpu
->name
, "Intel 2nd generation core");
423 cpu
->pstate
.min_pstate
= intel_pstate_min_pstate();
424 cpu
->pstate
.max_pstate
= intel_pstate_max_pstate();
425 cpu
->pstate
.turbo_pstate
= intel_pstate_turbo_pstate();
428 * goto max pstate so we don't slow up boot if we are built-in if we are
429 * a module we will take care of it during normal operation
431 intel_pstate_set_pstate(cpu
, cpu
->pstate
.max_pstate
);
434 static inline void intel_pstate_calc_busy(struct cpudata
*cpu
,
435 struct sample
*sample
)
438 core_pct
= div64_u64(sample
->aperf
* 100, sample
->mperf
);
439 sample
->freq
= cpu
->pstate
.max_pstate
* core_pct
* 1000;
441 sample
->core_pct_busy
= core_pct
;
444 static inline void intel_pstate_sample(struct cpudata
*cpu
)
448 rdmsrl(MSR_IA32_APERF
, aperf
);
449 rdmsrl(MSR_IA32_MPERF
, mperf
);
450 cpu
->sample_ptr
= (cpu
->sample_ptr
+ 1) % SAMPLE_COUNT
;
451 cpu
->samples
[cpu
->sample_ptr
].aperf
= aperf
;
452 cpu
->samples
[cpu
->sample_ptr
].mperf
= mperf
;
453 cpu
->samples
[cpu
->sample_ptr
].aperf
-= cpu
->prev_aperf
;
454 cpu
->samples
[cpu
->sample_ptr
].mperf
-= cpu
->prev_mperf
;
456 intel_pstate_calc_busy(cpu
, &cpu
->samples
[cpu
->sample_ptr
]);
458 cpu
->prev_aperf
= aperf
;
459 cpu
->prev_mperf
= mperf
;
462 static inline void intel_pstate_set_sample_time(struct cpudata
*cpu
)
464 int sample_time
, delay
;
466 sample_time
= cpu
->pstate_policy
->sample_rate_ms
;
467 delay
= msecs_to_jiffies(sample_time
);
468 mod_timer_pinned(&cpu
->timer
, jiffies
+ delay
);
471 static inline int intel_pstate_get_scaled_busy(struct cpudata
*cpu
)
474 int32_t core_busy
, max_pstate
, current_pstate
;
476 core_busy
= int_tofp(cpu
->samples
[cpu
->sample_ptr
].core_pct_busy
);
477 max_pstate
= int_tofp(cpu
->pstate
.max_pstate
);
478 current_pstate
= int_tofp(cpu
->pstate
.current_pstate
);
479 busy_scaled
= mul_fp(core_busy
, div_fp(max_pstate
, current_pstate
));
481 return fp_toint(busy_scaled
);
484 static inline void intel_pstate_adjust_busy_pstate(struct cpudata
*cpu
)
492 busy_scaled
= intel_pstate_get_scaled_busy(cpu
);
494 ctl
= pid_calc(pid
, busy_scaled
);
498 intel_pstate_pstate_increase(cpu
, steps
);
500 intel_pstate_pstate_decrease(cpu
, steps
);
503 static void intel_pstate_timer_func(unsigned long __data
)
505 struct cpudata
*cpu
= (struct cpudata
*) __data
;
507 intel_pstate_sample(cpu
);
508 intel_pstate_adjust_busy_pstate(cpu
);
510 if (cpu
->pstate
.current_pstate
== cpu
->pstate
.min_pstate
) {
511 cpu
->min_pstate_count
++;
512 if (!(cpu
->min_pstate_count
% 5)) {
513 intel_pstate_set_pstate(cpu
, cpu
->pstate
.max_pstate
);
516 cpu
->min_pstate_count
= 0;
518 intel_pstate_set_sample_time(cpu
);
521 #define ICPU(model, policy) \
522 { X86_VENDOR_INTEL, 6, model, X86_FEATURE_ANY, (unsigned long)&policy }
524 static const struct x86_cpu_id intel_pstate_cpu_ids
[] = {
525 ICPU(0x2a, default_policy
),
526 ICPU(0x2d, default_policy
),
527 ICPU(0x3a, default_policy
),
528 ICPU(0x3c, default_policy
),
529 ICPU(0x3e, default_policy
),
530 ICPU(0x3f, default_policy
),
531 ICPU(0x45, default_policy
),
532 ICPU(0x46, default_policy
),
535 MODULE_DEVICE_TABLE(x86cpu
, intel_pstate_cpu_ids
);
537 static int intel_pstate_init_cpu(unsigned int cpunum
)
540 const struct x86_cpu_id
*id
;
543 id
= x86_match_cpu(intel_pstate_cpu_ids
);
547 all_cpu_data
[cpunum
] = kzalloc(sizeof(struct cpudata
), GFP_KERNEL
);
548 if (!all_cpu_data
[cpunum
])
551 cpu
= all_cpu_data
[cpunum
];
553 intel_pstate_get_cpu_pstates(cpu
);
557 (struct pstate_adjust_policy
*)id
->driver_data
;
558 init_timer_deferrable(&cpu
->timer
);
559 cpu
->timer
.function
= intel_pstate_timer_func
;
562 cpu
->timer
.expires
= jiffies
+ HZ
/100;
563 intel_pstate_busy_pid_reset(cpu
);
564 intel_pstate_sample(cpu
);
565 intel_pstate_set_pstate(cpu
, cpu
->pstate
.max_pstate
);
567 add_timer_on(&cpu
->timer
, cpunum
);
569 pr_info("Intel pstate controlling: cpu %d\n", cpunum
);
574 static unsigned int intel_pstate_get(unsigned int cpu_num
)
576 struct sample
*sample
;
579 cpu
= all_cpu_data
[cpu_num
];
582 sample
= &cpu
->samples
[cpu
->sample_ptr
];
586 static int intel_pstate_set_policy(struct cpufreq_policy
*policy
)
590 cpu
= all_cpu_data
[policy
->cpu
];
592 if (!policy
->cpuinfo
.max_freq
)
595 if (policy
->policy
== CPUFREQ_POLICY_PERFORMANCE
) {
596 limits
.min_perf_pct
= 100;
597 limits
.min_perf
= int_tofp(1);
598 limits
.max_perf_pct
= 100;
599 limits
.max_perf
= int_tofp(1);
603 limits
.min_perf_pct
= (policy
->min
* 100) / policy
->cpuinfo
.max_freq
;
604 limits
.min_perf_pct
= clamp_t(int, limits
.min_perf_pct
, 0 , 100);
605 limits
.min_perf
= div_fp(int_tofp(limits
.min_perf_pct
), int_tofp(100));
607 limits
.max_policy_pct
= policy
->max
* 100 / policy
->cpuinfo
.max_freq
;
608 limits
.max_policy_pct
= clamp_t(int, limits
.max_policy_pct
, 0 , 100);
609 limits
.max_perf_pct
= min(limits
.max_policy_pct
, limits
.max_sysfs_pct
);
610 limits
.max_perf
= div_fp(int_tofp(limits
.max_perf_pct
), int_tofp(100));
615 static int intel_pstate_verify_policy(struct cpufreq_policy
*policy
)
617 cpufreq_verify_within_limits(policy
,
618 policy
->cpuinfo
.min_freq
,
619 policy
->cpuinfo
.max_freq
);
621 if ((policy
->policy
!= CPUFREQ_POLICY_POWERSAVE
) &&
622 (policy
->policy
!= CPUFREQ_POLICY_PERFORMANCE
))
628 static int intel_pstate_cpu_exit(struct cpufreq_policy
*policy
)
630 int cpu
= policy
->cpu
;
632 del_timer(&all_cpu_data
[cpu
]->timer
);
633 kfree(all_cpu_data
[cpu
]);
634 all_cpu_data
[cpu
] = NULL
;
638 static int intel_pstate_cpu_init(struct cpufreq_policy
*policy
)
640 int rc
, min_pstate
, max_pstate
;
643 rc
= intel_pstate_init_cpu(policy
->cpu
);
647 cpu
= all_cpu_data
[policy
->cpu
];
649 if (!limits
.no_turbo
&&
650 limits
.min_perf_pct
== 100 && limits
.max_perf_pct
== 100)
651 policy
->policy
= CPUFREQ_POLICY_PERFORMANCE
;
653 policy
->policy
= CPUFREQ_POLICY_POWERSAVE
;
655 intel_pstate_get_min_max(cpu
, &min_pstate
, &max_pstate
);
656 policy
->min
= min_pstate
* 100000;
657 policy
->max
= max_pstate
* 100000;
659 /* cpuinfo and default policy values */
660 policy
->cpuinfo
.min_freq
= cpu
->pstate
.min_pstate
* 100000;
661 policy
->cpuinfo
.max_freq
= cpu
->pstate
.turbo_pstate
* 100000;
662 policy
->cpuinfo
.transition_latency
= CPUFREQ_ETERNAL
;
663 cpumask_set_cpu(policy
->cpu
, policy
->cpus
);
668 static struct cpufreq_driver intel_pstate_driver
= {
669 .flags
= CPUFREQ_CONST_LOOPS
,
670 .verify
= intel_pstate_verify_policy
,
671 .setpolicy
= intel_pstate_set_policy
,
672 .get
= intel_pstate_get
,
673 .init
= intel_pstate_cpu_init
,
674 .exit
= intel_pstate_cpu_exit
,
675 .name
= "intel_pstate",
678 static int __initdata no_load
;
680 static int intel_pstate_msrs_not_valid(void)
682 /* Check that all the msr's we are using are valid. */
683 u64 aperf
, mperf
, tmp
;
685 rdmsrl(MSR_IA32_APERF
, aperf
);
686 rdmsrl(MSR_IA32_MPERF
, mperf
);
688 if (!intel_pstate_min_pstate() ||
689 !intel_pstate_max_pstate() ||
690 !intel_pstate_turbo_pstate())
693 rdmsrl(MSR_IA32_APERF
, tmp
);
697 rdmsrl(MSR_IA32_MPERF
, tmp
);
703 static int __init
intel_pstate_init(void)
706 const struct x86_cpu_id
*id
;
711 id
= x86_match_cpu(intel_pstate_cpu_ids
);
715 if (intel_pstate_msrs_not_valid())
718 pr_info("Intel P-state driver initializing.\n");
720 all_cpu_data
= vzalloc(sizeof(void *) * num_possible_cpus());
724 rc
= cpufreq_register_driver(&intel_pstate_driver
);
728 intel_pstate_debug_expose_params();
729 intel_pstate_sysfs_expose_params();
733 for_each_online_cpu(cpu
) {
734 if (all_cpu_data
[cpu
]) {
735 del_timer_sync(&all_cpu_data
[cpu
]->timer
);
736 kfree(all_cpu_data
[cpu
]);
744 device_initcall(intel_pstate_init
);
746 static int __init
intel_pstate_setup(char *str
)
751 if (!strcmp(str
, "disable"))
755 early_param("intel_pstate", intel_pstate_setup
);
757 MODULE_AUTHOR("Dirk Brandewie <dirk.j.brandewie@intel.com>");
758 MODULE_DESCRIPTION("'intel_pstate' - P state driver Intel Core processors");
759 MODULE_LICENSE("GPL");