2 * Intel Running Average Power Limit (RAPL) Driver
3 * Copyright (c) 2013, Intel Corporation.
5 * This program is free software; you can redistribute it and/or modify it
6 * under the terms and conditions of the GNU General Public License,
7 * version 2, as published by the Free Software Foundation.
9 * This program is distributed in the hope it will be useful, but WITHOUT
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License for
14 * You should have received a copy of the GNU General Public License along with
15 * this program; if not, write to the Free Software Foundation, Inc.
18 #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt
20 #include <linux/kernel.h>
21 #include <linux/module.h>
22 #include <linux/list.h>
23 #include <linux/types.h>
24 #include <linux/device.h>
25 #include <linux/slab.h>
26 #include <linux/log2.h>
27 #include <linux/bitmap.h>
28 #include <linux/delay.h>
29 #include <linux/sysfs.h>
30 #include <linux/cpu.h>
31 #include <linux/powercap.h>
32 #include <asm/iosf_mbi.h>
34 #include <asm/processor.h>
35 #include <asm/cpu_device_id.h>
37 /* bitmasks for RAPL MSRs, used by primitive access functions */
38 #define ENERGY_STATUS_MASK 0xffffffff
40 #define POWER_LIMIT1_MASK 0x7FFF
41 #define POWER_LIMIT1_ENABLE BIT(15)
42 #define POWER_LIMIT1_CLAMP BIT(16)
44 #define POWER_LIMIT2_MASK (0x7FFFULL<<32)
45 #define POWER_LIMIT2_ENABLE BIT_ULL(47)
46 #define POWER_LIMIT2_CLAMP BIT_ULL(48)
47 #define POWER_PACKAGE_LOCK BIT_ULL(63)
48 #define POWER_PP_LOCK BIT(31)
50 #define TIME_WINDOW1_MASK (0x7FULL<<17)
51 #define TIME_WINDOW2_MASK (0x7FULL<<49)
53 #define POWER_UNIT_OFFSET 0
54 #define POWER_UNIT_MASK 0x0F
56 #define ENERGY_UNIT_OFFSET 0x08
57 #define ENERGY_UNIT_MASK 0x1F00
59 #define TIME_UNIT_OFFSET 0x10
60 #define TIME_UNIT_MASK 0xF0000
62 #define POWER_INFO_MAX_MASK (0x7fffULL<<32)
63 #define POWER_INFO_MIN_MASK (0x7fffULL<<16)
64 #define POWER_INFO_MAX_TIME_WIN_MASK (0x3fULL<<48)
65 #define POWER_INFO_THERMAL_SPEC_MASK 0x7fff
67 #define PERF_STATUS_THROTTLE_TIME_MASK 0xffffffff
68 #define PP_POLICY_MASK 0x1F
70 /* Non HW constants */
71 #define RAPL_PRIMITIVE_DERIVED BIT(1) /* not from raw data */
72 #define RAPL_PRIMITIVE_DUMMY BIT(2)
74 #define TIME_WINDOW_MAX_MSEC 40000
75 #define TIME_WINDOW_MIN_MSEC 250
78 ARBITRARY_UNIT
, /* no translation */
84 enum rapl_domain_type
{
85 RAPL_DOMAIN_PACKAGE
, /* entire package/socket */
86 RAPL_DOMAIN_PP0
, /* core power plane */
87 RAPL_DOMAIN_PP1
, /* graphics uncore */
88 RAPL_DOMAIN_DRAM
,/* DRAM control_type */
92 enum rapl_domain_msr_id
{
93 RAPL_DOMAIN_MSR_LIMIT
,
94 RAPL_DOMAIN_MSR_STATUS
,
96 RAPL_DOMAIN_MSR_POLICY
,
101 /* per domain data, some are optional */
102 enum rapl_primitives
{
108 PL1_ENABLE
, /* power limit 1, aka long term */
109 PL1_CLAMP
, /* allow frequency to go below OS request */
110 PL2_ENABLE
, /* power limit 2, aka short term, instantaneous */
113 TIME_WINDOW1
, /* long term */
114 TIME_WINDOW2
, /* short term */
123 /* below are not raw primitive data */
128 #define NR_RAW_PRIMITIVES (NR_RAPL_PRIMITIVES - 2)
130 /* Can be expanded to include events, etc.*/
131 struct rapl_domain_data
{
132 u64 primitives
[NR_RAPL_PRIMITIVES
];
133 unsigned long timestamp
;
137 #define DOMAIN_STATE_INACTIVE BIT(0)
138 #define DOMAIN_STATE_POWER_LIMIT_SET BIT(1)
139 #define DOMAIN_STATE_BIOS_LOCKED BIT(2)
141 #define NR_POWER_LIMITS (2)
142 struct rapl_power_limit
{
143 struct powercap_zone_constraint
*constraint
;
144 int prim_id
; /* primitive ID used to enable */
145 struct rapl_domain
*domain
;
149 static const char pl1_name
[] = "long_term";
150 static const char pl2_name
[] = "short_term";
154 enum rapl_domain_type id
;
155 int msrs
[RAPL_DOMAIN_MSR_MAX
];
156 struct powercap_zone power_zone
;
157 struct rapl_domain_data rdd
;
158 struct rapl_power_limit rpl
[NR_POWER_LIMITS
];
159 u64 attr_map
; /* track capabilities */
163 #define power_zone_to_rapl_domain(_zone) \
164 container_of(_zone, struct rapl_domain, power_zone)
167 /* Each physical package contains multiple domains, these are the common
168 * data across RAPL domains within a package.
170 struct rapl_package
{
171 unsigned int id
; /* physical package/socket id */
172 unsigned int nr_domains
;
173 unsigned long domain_map
; /* bit map of active domains */
174 unsigned int power_unit
;
175 unsigned int energy_unit
;
176 unsigned int time_unit
;
177 struct rapl_domain
*domains
; /* array of domains, sized at runtime */
178 struct powercap_zone
*power_zone
; /* keep track of parent zone */
179 int nr_cpus
; /* active cpus on the package, topology info is lost during
180 * cpu hotplug. so we have to track ourselves.
182 unsigned long power_limit_irq
; /* keep track of package power limit
183 * notify interrupt enable status.
185 struct list_head plist
;
188 struct rapl_defaults
{
189 int (*check_unit
)(struct rapl_package
*rp
, int cpu
);
190 void (*set_floor_freq
)(struct rapl_domain
*rd
, bool mode
);
191 u64 (*compute_time_window
)(struct rapl_package
*rp
, u64 val
,
194 static struct rapl_defaults
*rapl_defaults
;
196 /* Sideband MBI registers */
197 #define IOSF_CPU_POWER_BUDGET_CTL (0x2)
199 #define PACKAGE_PLN_INT_SAVED BIT(0)
200 #define MAX_PRIM_NAME (32)
202 /* per domain data. used to describe individual knobs such that access function
203 * can be consolidated into one instead of many inline functions.
205 struct rapl_primitive_info
{
209 enum rapl_domain_msr_id id
;
214 #define PRIMITIVE_INFO_INIT(p, m, s, i, u, f) { \
223 static void rapl_init_domains(struct rapl_package
*rp
);
224 static int rapl_read_data_raw(struct rapl_domain
*rd
,
225 enum rapl_primitives prim
,
226 bool xlate
, u64
*data
);
227 static int rapl_write_data_raw(struct rapl_domain
*rd
,
228 enum rapl_primitives prim
,
229 unsigned long long value
);
230 static u64
rapl_unit_xlate(int package
, enum unit_type type
, u64 value
,
232 static void package_power_limit_irq_save(int package_id
);
234 static LIST_HEAD(rapl_packages
); /* guarded by CPU hotplug lock */
236 static const char * const rapl_domain_names
[] = {
243 static struct powercap_control_type
*control_type
; /* PowerCap Controller */
245 /* caller to ensure CPU hotplug lock is held */
246 static struct rapl_package
*find_package_by_id(int id
)
248 struct rapl_package
*rp
;
250 list_for_each_entry(rp
, &rapl_packages
, plist
) {
258 /* caller to ensure CPU hotplug lock is held */
259 static int find_active_cpu_on_package(int package_id
)
263 for_each_online_cpu(i
) {
264 if (topology_physical_package_id(i
) == package_id
)
267 /* all CPUs on this package are offline */
272 /* caller must hold cpu hotplug lock */
273 static void rapl_cleanup_data(void)
275 struct rapl_package
*p
, *tmp
;
277 list_for_each_entry_safe(p
, tmp
, &rapl_packages
, plist
) {
284 static int get_energy_counter(struct powercap_zone
*power_zone
, u64
*energy_raw
)
286 struct rapl_domain
*rd
;
289 /* prevent CPU hotplug, make sure the RAPL domain does not go
290 * away while reading the counter.
293 rd
= power_zone_to_rapl_domain(power_zone
);
295 if (!rapl_read_data_raw(rd
, ENERGY_COUNTER
, true, &energy_now
)) {
296 *energy_raw
= energy_now
;
306 static int get_max_energy_counter(struct powercap_zone
*pcd_dev
, u64
*energy
)
308 *energy
= rapl_unit_xlate(0, ENERGY_UNIT
, ENERGY_STATUS_MASK
, 0);
312 static int release_zone(struct powercap_zone
*power_zone
)
314 struct rapl_domain
*rd
= power_zone_to_rapl_domain(power_zone
);
315 struct rapl_package
*rp
;
317 /* package zone is the last zone of a package, we can free
318 * memory here since all children has been unregistered.
320 if (rd
->id
== RAPL_DOMAIN_PACKAGE
) {
321 rp
= find_package_by_id(rd
->package_id
);
323 dev_warn(&power_zone
->dev
, "no package id %s\n",
335 static int find_nr_power_limit(struct rapl_domain
*rd
)
339 for (i
= 0; i
< NR_POWER_LIMITS
; i
++) {
340 if (rd
->rpl
[i
].name
== NULL
)
347 static int set_domain_enable(struct powercap_zone
*power_zone
, bool mode
)
349 struct rapl_domain
*rd
= power_zone_to_rapl_domain(power_zone
);
351 if (rd
->state
& DOMAIN_STATE_BIOS_LOCKED
)
355 rapl_write_data_raw(rd
, PL1_ENABLE
, mode
);
356 rapl_defaults
->set_floor_freq(rd
, mode
);
362 static int get_domain_enable(struct powercap_zone
*power_zone
, bool *mode
)
364 struct rapl_domain
*rd
= power_zone_to_rapl_domain(power_zone
);
367 if (rd
->state
& DOMAIN_STATE_BIOS_LOCKED
) {
372 if (rapl_read_data_raw(rd
, PL1_ENABLE
, true, &val
)) {
382 /* per RAPL domain ops, in the order of rapl_domain_type */
383 static struct powercap_zone_ops zone_ops
[] = {
384 /* RAPL_DOMAIN_PACKAGE */
386 .get_energy_uj
= get_energy_counter
,
387 .get_max_energy_range_uj
= get_max_energy_counter
,
388 .release
= release_zone
,
389 .set_enable
= set_domain_enable
,
390 .get_enable
= get_domain_enable
,
392 /* RAPL_DOMAIN_PP0 */
394 .get_energy_uj
= get_energy_counter
,
395 .get_max_energy_range_uj
= get_max_energy_counter
,
396 .release
= release_zone
,
397 .set_enable
= set_domain_enable
,
398 .get_enable
= get_domain_enable
,
400 /* RAPL_DOMAIN_PP1 */
402 .get_energy_uj
= get_energy_counter
,
403 .get_max_energy_range_uj
= get_max_energy_counter
,
404 .release
= release_zone
,
405 .set_enable
= set_domain_enable
,
406 .get_enable
= get_domain_enable
,
408 /* RAPL_DOMAIN_DRAM */
410 .get_energy_uj
= get_energy_counter
,
411 .get_max_energy_range_uj
= get_max_energy_counter
,
412 .release
= release_zone
,
413 .set_enable
= set_domain_enable
,
414 .get_enable
= get_domain_enable
,
418 static int set_power_limit(struct powercap_zone
*power_zone
, int id
,
421 struct rapl_domain
*rd
;
422 struct rapl_package
*rp
;
426 rd
= power_zone_to_rapl_domain(power_zone
);
427 rp
= find_package_by_id(rd
->package_id
);
433 if (rd
->state
& DOMAIN_STATE_BIOS_LOCKED
) {
434 dev_warn(&power_zone
->dev
, "%s locked by BIOS, monitoring only\n",
440 switch (rd
->rpl
[id
].prim_id
) {
442 rapl_write_data_raw(rd
, POWER_LIMIT1
, power_limit
);
445 rapl_write_data_raw(rd
, POWER_LIMIT2
, power_limit
);
451 package_power_limit_irq_save(rd
->package_id
);
457 static int get_current_power_limit(struct powercap_zone
*power_zone
, int id
,
460 struct rapl_domain
*rd
;
466 rd
= power_zone_to_rapl_domain(power_zone
);
467 switch (rd
->rpl
[id
].prim_id
) {
478 if (rapl_read_data_raw(rd
, prim
, true, &val
))
488 static int set_time_window(struct powercap_zone
*power_zone
, int id
,
491 struct rapl_domain
*rd
;
495 rd
= power_zone_to_rapl_domain(power_zone
);
496 switch (rd
->rpl
[id
].prim_id
) {
498 rapl_write_data_raw(rd
, TIME_WINDOW1
, window
);
501 rapl_write_data_raw(rd
, TIME_WINDOW2
, window
);
510 static int get_time_window(struct powercap_zone
*power_zone
, int id
, u64
*data
)
512 struct rapl_domain
*rd
;
517 rd
= power_zone_to_rapl_domain(power_zone
);
518 switch (rd
->rpl
[id
].prim_id
) {
520 ret
= rapl_read_data_raw(rd
, TIME_WINDOW1
, true, &val
);
523 ret
= rapl_read_data_raw(rd
, TIME_WINDOW2
, true, &val
);
536 static const char *get_constraint_name(struct powercap_zone
*power_zone
, int id
)
538 struct rapl_power_limit
*rpl
;
539 struct rapl_domain
*rd
;
541 rd
= power_zone_to_rapl_domain(power_zone
);
542 rpl
= (struct rapl_power_limit
*) &rd
->rpl
[id
];
548 static int get_max_power(struct powercap_zone
*power_zone
, int id
,
551 struct rapl_domain
*rd
;
557 rd
= power_zone_to_rapl_domain(power_zone
);
558 switch (rd
->rpl
[id
].prim_id
) {
560 prim
= THERMAL_SPEC_POWER
;
569 if (rapl_read_data_raw(rd
, prim
, true, &val
))
579 static struct powercap_zone_constraint_ops constraint_ops
= {
580 .set_power_limit_uw
= set_power_limit
,
581 .get_power_limit_uw
= get_current_power_limit
,
582 .set_time_window_us
= set_time_window
,
583 .get_time_window_us
= get_time_window
,
584 .get_max_power_uw
= get_max_power
,
585 .get_name
= get_constraint_name
,
588 /* called after domain detection and package level data are set */
589 static void rapl_init_domains(struct rapl_package
*rp
)
592 struct rapl_domain
*rd
= rp
->domains
;
594 for (i
= 0; i
< RAPL_DOMAIN_MAX
; i
++) {
595 unsigned int mask
= rp
->domain_map
& (1 << i
);
597 case BIT(RAPL_DOMAIN_PACKAGE
):
598 rd
->name
= rapl_domain_names
[RAPL_DOMAIN_PACKAGE
];
599 rd
->id
= RAPL_DOMAIN_PACKAGE
;
600 rd
->msrs
[0] = MSR_PKG_POWER_LIMIT
;
601 rd
->msrs
[1] = MSR_PKG_ENERGY_STATUS
;
602 rd
->msrs
[2] = MSR_PKG_PERF_STATUS
;
604 rd
->msrs
[4] = MSR_PKG_POWER_INFO
;
605 rd
->rpl
[0].prim_id
= PL1_ENABLE
;
606 rd
->rpl
[0].name
= pl1_name
;
607 rd
->rpl
[1].prim_id
= PL2_ENABLE
;
608 rd
->rpl
[1].name
= pl2_name
;
610 case BIT(RAPL_DOMAIN_PP0
):
611 rd
->name
= rapl_domain_names
[RAPL_DOMAIN_PP0
];
612 rd
->id
= RAPL_DOMAIN_PP0
;
613 rd
->msrs
[0] = MSR_PP0_POWER_LIMIT
;
614 rd
->msrs
[1] = MSR_PP0_ENERGY_STATUS
;
616 rd
->msrs
[3] = MSR_PP0_POLICY
;
618 rd
->rpl
[0].prim_id
= PL1_ENABLE
;
619 rd
->rpl
[0].name
= pl1_name
;
621 case BIT(RAPL_DOMAIN_PP1
):
622 rd
->name
= rapl_domain_names
[RAPL_DOMAIN_PP1
];
623 rd
->id
= RAPL_DOMAIN_PP1
;
624 rd
->msrs
[0] = MSR_PP1_POWER_LIMIT
;
625 rd
->msrs
[1] = MSR_PP1_ENERGY_STATUS
;
627 rd
->msrs
[3] = MSR_PP1_POLICY
;
629 rd
->rpl
[0].prim_id
= PL1_ENABLE
;
630 rd
->rpl
[0].name
= pl1_name
;
632 case BIT(RAPL_DOMAIN_DRAM
):
633 rd
->name
= rapl_domain_names
[RAPL_DOMAIN_DRAM
];
634 rd
->id
= RAPL_DOMAIN_DRAM
;
635 rd
->msrs
[0] = MSR_DRAM_POWER_LIMIT
;
636 rd
->msrs
[1] = MSR_DRAM_ENERGY_STATUS
;
637 rd
->msrs
[2] = MSR_DRAM_PERF_STATUS
;
639 rd
->msrs
[4] = MSR_DRAM_POWER_INFO
;
640 rd
->rpl
[0].prim_id
= PL1_ENABLE
;
641 rd
->rpl
[0].name
= pl1_name
;
645 rd
->package_id
= rp
->id
;
651 static u64
rapl_unit_xlate(int package
, enum unit_type type
, u64 value
,
655 struct rapl_package
*rp
;
657 rp
= find_package_by_id(package
);
663 units
= rp
->power_unit
;
666 units
= rp
->energy_unit
;
669 return rapl_defaults
->compute_time_window(rp
, value
, to_raw
);
676 return div64_u64(value
, units
);
683 /* in the order of enum rapl_primitives */
684 static struct rapl_primitive_info rpi
[] = {
685 /* name, mask, shift, msr index, unit divisor */
686 PRIMITIVE_INFO_INIT(ENERGY_COUNTER
, ENERGY_STATUS_MASK
, 0,
687 RAPL_DOMAIN_MSR_STATUS
, ENERGY_UNIT
, 0),
688 PRIMITIVE_INFO_INIT(POWER_LIMIT1
, POWER_LIMIT1_MASK
, 0,
689 RAPL_DOMAIN_MSR_LIMIT
, POWER_UNIT
, 0),
690 PRIMITIVE_INFO_INIT(POWER_LIMIT2
, POWER_LIMIT2_MASK
, 32,
691 RAPL_DOMAIN_MSR_LIMIT
, POWER_UNIT
, 0),
692 PRIMITIVE_INFO_INIT(FW_LOCK
, POWER_PP_LOCK
, 31,
693 RAPL_DOMAIN_MSR_LIMIT
, ARBITRARY_UNIT
, 0),
694 PRIMITIVE_INFO_INIT(PL1_ENABLE
, POWER_LIMIT1_ENABLE
, 15,
695 RAPL_DOMAIN_MSR_LIMIT
, ARBITRARY_UNIT
, 0),
696 PRIMITIVE_INFO_INIT(PL1_CLAMP
, POWER_LIMIT1_CLAMP
, 16,
697 RAPL_DOMAIN_MSR_LIMIT
, ARBITRARY_UNIT
, 0),
698 PRIMITIVE_INFO_INIT(PL2_ENABLE
, POWER_LIMIT2_ENABLE
, 47,
699 RAPL_DOMAIN_MSR_LIMIT
, ARBITRARY_UNIT
, 0),
700 PRIMITIVE_INFO_INIT(PL2_CLAMP
, POWER_LIMIT2_CLAMP
, 48,
701 RAPL_DOMAIN_MSR_LIMIT
, ARBITRARY_UNIT
, 0),
702 PRIMITIVE_INFO_INIT(TIME_WINDOW1
, TIME_WINDOW1_MASK
, 17,
703 RAPL_DOMAIN_MSR_LIMIT
, TIME_UNIT
, 0),
704 PRIMITIVE_INFO_INIT(TIME_WINDOW2
, TIME_WINDOW2_MASK
, 49,
705 RAPL_DOMAIN_MSR_LIMIT
, TIME_UNIT
, 0),
706 PRIMITIVE_INFO_INIT(THERMAL_SPEC_POWER
, POWER_INFO_THERMAL_SPEC_MASK
,
707 0, RAPL_DOMAIN_MSR_INFO
, POWER_UNIT
, 0),
708 PRIMITIVE_INFO_INIT(MAX_POWER
, POWER_INFO_MAX_MASK
, 32,
709 RAPL_DOMAIN_MSR_INFO
, POWER_UNIT
, 0),
710 PRIMITIVE_INFO_INIT(MIN_POWER
, POWER_INFO_MIN_MASK
, 16,
711 RAPL_DOMAIN_MSR_INFO
, POWER_UNIT
, 0),
712 PRIMITIVE_INFO_INIT(MAX_TIME_WINDOW
, POWER_INFO_MAX_TIME_WIN_MASK
, 48,
713 RAPL_DOMAIN_MSR_INFO
, TIME_UNIT
, 0),
714 PRIMITIVE_INFO_INIT(THROTTLED_TIME
, PERF_STATUS_THROTTLE_TIME_MASK
, 0,
715 RAPL_DOMAIN_MSR_PERF
, TIME_UNIT
, 0),
716 PRIMITIVE_INFO_INIT(PRIORITY_LEVEL
, PP_POLICY_MASK
, 0,
717 RAPL_DOMAIN_MSR_POLICY
, ARBITRARY_UNIT
, 0),
719 PRIMITIVE_INFO_INIT(AVERAGE_POWER
, 0, 0, 0, POWER_UNIT
,
720 RAPL_PRIMITIVE_DERIVED
),
724 /* Read primitive data based on its related struct rapl_primitive_info.
725 * if xlate flag is set, return translated data based on data units, i.e.
726 * time, energy, and power.
727 * RAPL MSRs are non-architectual and are laid out not consistently across
728 * domains. Here we use primitive info to allow writing consolidated access
730 * For a given primitive, it is processed by MSR mask and shift. Unit conversion
731 * is pre-assigned based on RAPL unit MSRs read at init time.
732 * 63-------------------------- 31--------------------------- 0
734 * | |<- shift ----------------|
735 * 63-------------------------- 31--------------------------- 0
737 static int rapl_read_data_raw(struct rapl_domain
*rd
,
738 enum rapl_primitives prim
,
739 bool xlate
, u64
*data
)
743 struct rapl_primitive_info
*rp
= &rpi
[prim
];
746 if (!rp
->name
|| rp
->flag
& RAPL_PRIMITIVE_DUMMY
)
749 msr
= rd
->msrs
[rp
->id
];
752 /* use physical package id to look up active cpus */
753 cpu
= find_active_cpu_on_package(rd
->package_id
);
757 /* special-case package domain, which uses a different bit*/
758 if (prim
== FW_LOCK
&& rd
->id
== RAPL_DOMAIN_PACKAGE
) {
759 rp
->mask
= POWER_PACKAGE_LOCK
;
762 /* non-hardware data are collected by the polling thread */
763 if (rp
->flag
& RAPL_PRIMITIVE_DERIVED
) {
764 *data
= rd
->rdd
.primitives
[prim
];
768 if (rdmsrl_safe_on_cpu(cpu
, msr
, &value
)) {
769 pr_debug("failed to read msr 0x%x on cpu %d\n", msr
, cpu
);
773 final
= value
& rp
->mask
;
774 final
= final
>> rp
->shift
;
776 *data
= rapl_unit_xlate(rd
->package_id
, rp
->unit
, final
, 0);
783 /* Similar use of primitive info in the read counterpart */
784 static int rapl_write_data_raw(struct rapl_domain
*rd
,
785 enum rapl_primitives prim
,
786 unsigned long long value
)
790 struct rapl_primitive_info
*rp
= &rpi
[prim
];
793 cpu
= find_active_cpu_on_package(rd
->package_id
);
796 msr
= rd
->msrs
[rp
->id
];
797 if (rdmsrl_safe_on_cpu(cpu
, msr
, &msr_val
)) {
798 dev_dbg(&rd
->power_zone
.dev
,
799 "failed to read msr 0x%x on cpu %d\n", msr
, cpu
);
802 value
= rapl_unit_xlate(rd
->package_id
, rp
->unit
, value
, 1);
803 msr_val
&= ~rp
->mask
;
804 msr_val
|= value
<< rp
->shift
;
805 if (wrmsrl_safe_on_cpu(cpu
, msr
, msr_val
)) {
806 dev_dbg(&rd
->power_zone
.dev
,
807 "failed to write msr 0x%x on cpu %d\n", msr
, cpu
);
815 * Raw RAPL data stored in MSRs are in certain scales. We need to
816 * convert them into standard units based on the units reported in
817 * the RAPL unit MSRs. This is specific to CPUs as the method to
818 * calculate units differ on different CPUs.
819 * We convert the units to below format based on CPUs.
821 * energy unit: microJoules : Represented in microJoules by default
822 * power unit : microWatts : Represented in milliWatts by default
823 * time unit : microseconds: Represented in seconds by default
825 static int rapl_check_unit_core(struct rapl_package
*rp
, int cpu
)
830 if (rdmsrl_safe_on_cpu(cpu
, MSR_RAPL_POWER_UNIT
, &msr_val
)) {
831 pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n",
832 MSR_RAPL_POWER_UNIT
, cpu
);
836 value
= (msr_val
& ENERGY_UNIT_MASK
) >> ENERGY_UNIT_OFFSET
;
837 rp
->energy_unit
= 1000000 / (1 << value
);
839 value
= (msr_val
& POWER_UNIT_MASK
) >> POWER_UNIT_OFFSET
;
840 rp
->power_unit
= 1000000 / (1 << value
);
842 value
= (msr_val
& TIME_UNIT_MASK
) >> TIME_UNIT_OFFSET
;
843 rp
->time_unit
= 1000000 / (1 << value
);
845 pr_debug("Core CPU package %d energy=%duJ, time=%dus, power=%duW\n",
846 rp
->id
, rp
->energy_unit
, rp
->time_unit
, rp
->power_unit
);
851 static int rapl_check_unit_atom(struct rapl_package
*rp
, int cpu
)
856 if (rdmsrl_safe_on_cpu(cpu
, MSR_RAPL_POWER_UNIT
, &msr_val
)) {
857 pr_err("Failed to read power unit MSR 0x%x on CPU %d, exit.\n",
858 MSR_RAPL_POWER_UNIT
, cpu
);
861 value
= (msr_val
& ENERGY_UNIT_MASK
) >> ENERGY_UNIT_OFFSET
;
862 rp
->energy_unit
= 1 << value
;
864 value
= (msr_val
& POWER_UNIT_MASK
) >> POWER_UNIT_OFFSET
;
865 rp
->power_unit
= (1 << value
) * 1000;
867 value
= (msr_val
& TIME_UNIT_MASK
) >> TIME_UNIT_OFFSET
;
868 rp
->time_unit
= 1000000 / (1 << value
);
870 pr_debug("Atom package %d energy=%duJ, time=%dus, power=%duW\n",
871 rp
->id
, rp
->energy_unit
, rp
->time_unit
, rp
->power_unit
);
878 * When package power limit is set artificially low by RAPL, LVT
879 * thermal interrupt for package power limit should be ignored
880 * since we are not really exceeding the real limit. The intention
881 * is to avoid excessive interrupts while we are trying to save power.
882 * A useful feature might be routing the package_power_limit interrupt
883 * to userspace via eventfd. once we have a usecase, this is simple
884 * to do by adding an atomic notifier.
887 static void package_power_limit_irq_save(int package_id
)
891 struct rapl_package
*rp
;
893 rp
= find_package_by_id(package_id
);
897 if (!boot_cpu_has(X86_FEATURE_PTS
) || !boot_cpu_has(X86_FEATURE_PLN
))
900 cpu
= find_active_cpu_on_package(package_id
);
903 /* save the state of PLN irq mask bit before disabling it */
904 rdmsr_safe_on_cpu(cpu
, MSR_IA32_PACKAGE_THERM_INTERRUPT
, &l
, &h
);
905 if (!(rp
->power_limit_irq
& PACKAGE_PLN_INT_SAVED
)) {
906 rp
->power_limit_irq
= l
& PACKAGE_THERM_INT_PLN_ENABLE
;
907 rp
->power_limit_irq
|= PACKAGE_PLN_INT_SAVED
;
909 l
&= ~PACKAGE_THERM_INT_PLN_ENABLE
;
910 wrmsr_on_cpu(cpu
, MSR_IA32_PACKAGE_THERM_INTERRUPT
, l
, h
);
913 /* restore per package power limit interrupt enable state */
914 static void package_power_limit_irq_restore(int package_id
)
918 struct rapl_package
*rp
;
920 rp
= find_package_by_id(package_id
);
924 if (!boot_cpu_has(X86_FEATURE_PTS
) || !boot_cpu_has(X86_FEATURE_PLN
))
927 cpu
= find_active_cpu_on_package(package_id
);
931 /* irq enable state not saved, nothing to restore */
932 if (!(rp
->power_limit_irq
& PACKAGE_PLN_INT_SAVED
))
934 rdmsr_safe_on_cpu(cpu
, MSR_IA32_PACKAGE_THERM_INTERRUPT
, &l
, &h
);
936 if (rp
->power_limit_irq
& PACKAGE_THERM_INT_PLN_ENABLE
)
937 l
|= PACKAGE_THERM_INT_PLN_ENABLE
;
939 l
&= ~PACKAGE_THERM_INT_PLN_ENABLE
;
941 wrmsr_on_cpu(cpu
, MSR_IA32_PACKAGE_THERM_INTERRUPT
, l
, h
);
944 static void set_floor_freq_default(struct rapl_domain
*rd
, bool mode
)
946 int nr_powerlimit
= find_nr_power_limit(rd
);
948 /* always enable clamp such that p-state can go below OS requested
949 * range. power capping priority over guranteed frequency.
951 rapl_write_data_raw(rd
, PL1_CLAMP
, mode
);
953 /* some domains have pl2 */
954 if (nr_powerlimit
> 1) {
955 rapl_write_data_raw(rd
, PL2_ENABLE
, mode
);
956 rapl_write_data_raw(rd
, PL2_CLAMP
, mode
);
960 static void set_floor_freq_atom(struct rapl_domain
*rd
, bool enable
)
962 static u32 power_ctrl_orig_val
;
965 if (!power_ctrl_orig_val
)
966 iosf_mbi_read(BT_MBI_UNIT_PMC
, BT_MBI_PMC_READ
,
967 IOSF_CPU_POWER_BUDGET_CTL
, &power_ctrl_orig_val
);
968 mdata
= power_ctrl_orig_val
;
970 mdata
&= ~(0x7f << 8);
973 iosf_mbi_write(BT_MBI_UNIT_PMC
, BT_MBI_PMC_WRITE
,
974 IOSF_CPU_POWER_BUDGET_CTL
, mdata
);
977 static u64
rapl_compute_time_window_core(struct rapl_package
*rp
, u64 value
,
980 u64 f
, y
; /* fraction and exp. used for time unit */
983 * Special processing based on 2^Y*(1+F/4), refer
984 * to Intel Software Developer's manual Vol.3B: CH 14.9.3.
987 f
= (value
& 0x60) >> 5;
989 value
= (1 << y
) * (4 + f
) * rp
->time_unit
/ 4;
991 do_div(value
, rp
->time_unit
);
993 f
= div64_u64(4 * (value
- (1 << y
)), 1 << y
);
994 value
= (y
& 0x1f) | ((f
& 0x3) << 5);
999 static u64
rapl_compute_time_window_atom(struct rapl_package
*rp
, u64 value
,
1003 * Atom time unit encoding is straight forward val * time_unit,
1004 * where time_unit is default to 1 sec. Never 0.
1007 return (value
) ? value
*= rp
->time_unit
: rp
->time_unit
;
1009 value
= div64_u64(value
, rp
->time_unit
);
1014 static const struct rapl_defaults rapl_defaults_core
= {
1015 .check_unit
= rapl_check_unit_core
,
1016 .set_floor_freq
= set_floor_freq_default
,
1017 .compute_time_window
= rapl_compute_time_window_core
,
1020 static const struct rapl_defaults rapl_defaults_atom
= {
1021 .check_unit
= rapl_check_unit_atom
,
1022 .set_floor_freq
= set_floor_freq_atom
,
1023 .compute_time_window
= rapl_compute_time_window_atom
,
1026 #define RAPL_CPU(_model, _ops) { \
1027 .vendor = X86_VENDOR_INTEL, \
1030 .driver_data = (kernel_ulong_t)&_ops, \
1033 static const struct x86_cpu_id rapl_ids
[] = {
1034 RAPL_CPU(0x2a, rapl_defaults_core
),/* Sandy Bridge */
1035 RAPL_CPU(0x2d, rapl_defaults_core
),/* Sandy Bridge EP */
1036 RAPL_CPU(0x37, rapl_defaults_atom
),/* Valleyview */
1037 RAPL_CPU(0x3a, rapl_defaults_core
),/* Ivy Bridge */
1038 RAPL_CPU(0x3c, rapl_defaults_core
),/* Haswell */
1039 RAPL_CPU(0x3d, rapl_defaults_core
),/* Broadwell */
1040 RAPL_CPU(0x3f, rapl_defaults_core
),/* Haswell */
1041 RAPL_CPU(0x45, rapl_defaults_core
),/* Haswell ULT */
1042 RAPL_CPU(0x4C, rapl_defaults_atom
),/* Braswell */
1043 RAPL_CPU(0x4A, rapl_defaults_atom
),/* Tangier */
1044 RAPL_CPU(0x56, rapl_defaults_core
),/* Future Xeon */
1045 RAPL_CPU(0x5A, rapl_defaults_atom
),/* Annidale */
1048 MODULE_DEVICE_TABLE(x86cpu
, rapl_ids
);
1050 /* read once for all raw primitive data for all packages, domains */
1051 static void rapl_update_domain_data(void)
1055 struct rapl_package
*rp
;
1057 list_for_each_entry(rp
, &rapl_packages
, plist
) {
1058 for (dmn
= 0; dmn
< rp
->nr_domains
; dmn
++) {
1059 pr_debug("update package %d domain %s data\n", rp
->id
,
1060 rp
->domains
[dmn
].name
);
1061 /* exclude non-raw primitives */
1062 for (prim
= 0; prim
< NR_RAW_PRIMITIVES
; prim
++)
1063 if (!rapl_read_data_raw(&rp
->domains
[dmn
], prim
,
1066 rp
->domains
[dmn
].rdd
.primitives
[prim
] =
1073 static int rapl_unregister_powercap(void)
1075 struct rapl_package
*rp
;
1076 struct rapl_domain
*rd
, *rd_package
= NULL
;
1078 /* unregister all active rapl packages from the powercap layer,
1081 list_for_each_entry(rp
, &rapl_packages
, plist
) {
1082 package_power_limit_irq_restore(rp
->id
);
1084 for (rd
= rp
->domains
; rd
< rp
->domains
+ rp
->nr_domains
;
1086 pr_debug("remove package, undo power limit on %d: %s\n",
1088 rapl_write_data_raw(rd
, PL1_ENABLE
, 0);
1089 rapl_write_data_raw(rd
, PL2_ENABLE
, 0);
1090 rapl_write_data_raw(rd
, PL1_CLAMP
, 0);
1091 rapl_write_data_raw(rd
, PL2_CLAMP
, 0);
1092 if (rd
->id
== RAPL_DOMAIN_PACKAGE
) {
1096 powercap_unregister_zone(control_type
, &rd
->power_zone
);
1098 /* do the package zone last */
1100 powercap_unregister_zone(control_type
,
1101 &rd_package
->power_zone
);
1103 powercap_unregister_control_type(control_type
);
1108 static int rapl_package_register_powercap(struct rapl_package
*rp
)
1110 struct rapl_domain
*rd
;
1112 char dev_name
[17]; /* max domain name = 7 + 1 + 8 for int + 1 for null*/
1113 struct powercap_zone
*power_zone
= NULL
;
1116 /* first we register package domain as the parent zone*/
1117 for (rd
= rp
->domains
; rd
< rp
->domains
+ rp
->nr_domains
; rd
++) {
1118 if (rd
->id
== RAPL_DOMAIN_PACKAGE
) {
1119 nr_pl
= find_nr_power_limit(rd
);
1120 pr_debug("register socket %d package domain %s\n",
1122 memset(dev_name
, 0, sizeof(dev_name
));
1123 snprintf(dev_name
, sizeof(dev_name
), "%s-%d",
1125 power_zone
= powercap_register_zone(&rd
->power_zone
,
1131 if (IS_ERR(power_zone
)) {
1132 pr_debug("failed to register package, %d\n",
1134 ret
= PTR_ERR(power_zone
);
1137 /* track parent zone in per package/socket data */
1138 rp
->power_zone
= power_zone
;
1139 /* done, only one package domain per socket */
1144 pr_err("no package domain found, unknown topology!\n");
1148 /* now register domains as children of the socket/package*/
1149 for (rd
= rp
->domains
; rd
< rp
->domains
+ rp
->nr_domains
; rd
++) {
1150 if (rd
->id
== RAPL_DOMAIN_PACKAGE
)
1152 /* number of power limits per domain varies */
1153 nr_pl
= find_nr_power_limit(rd
);
1154 power_zone
= powercap_register_zone(&rd
->power_zone
,
1155 control_type
, rd
->name
,
1157 &zone_ops
[rd
->id
], nr_pl
,
1160 if (IS_ERR(power_zone
)) {
1161 pr_debug("failed to register power_zone, %d:%s:%s\n",
1162 rp
->id
, rd
->name
, dev_name
);
1163 ret
= PTR_ERR(power_zone
);
1171 /* clean up previously initialized domains within the package if we
1172 * failed after the first domain setup.
1174 while (--rd
>= rp
->domains
) {
1175 pr_debug("unregister package %d domain %s\n", rp
->id
, rd
->name
);
1176 powercap_unregister_zone(control_type
, &rd
->power_zone
);
1182 static int rapl_register_powercap(void)
1184 struct rapl_domain
*rd
;
1185 struct rapl_package
*rp
;
1188 control_type
= powercap_register_control_type(NULL
, "intel-rapl", NULL
);
1189 if (IS_ERR(control_type
)) {
1190 pr_debug("failed to register powercap control_type.\n");
1191 return PTR_ERR(control_type
);
1193 /* read the initial data */
1194 rapl_update_domain_data();
1195 list_for_each_entry(rp
, &rapl_packages
, plist
)
1196 if (rapl_package_register_powercap(rp
))
1197 goto err_cleanup_package
;
1200 err_cleanup_package
:
1201 /* clean up previously initialized packages */
1202 list_for_each_entry_continue_reverse(rp
, &rapl_packages
, plist
) {
1203 for (rd
= rp
->domains
; rd
< rp
->domains
+ rp
->nr_domains
;
1205 pr_debug("unregister zone/package %d, %s domain\n",
1207 powercap_unregister_zone(control_type
, &rd
->power_zone
);
1214 static int rapl_check_domain(int cpu
, int domain
)
1220 case RAPL_DOMAIN_PACKAGE
:
1221 msr
= MSR_PKG_ENERGY_STATUS
;
1223 case RAPL_DOMAIN_PP0
:
1224 msr
= MSR_PP0_ENERGY_STATUS
;
1226 case RAPL_DOMAIN_PP1
:
1227 msr
= MSR_PP1_ENERGY_STATUS
;
1229 case RAPL_DOMAIN_DRAM
:
1230 msr
= MSR_DRAM_ENERGY_STATUS
;
1233 pr_err("invalid domain id %d\n", domain
);
1236 /* make sure domain counters are available and contains non-zero
1237 * values, otherwise skip it.
1239 if (rdmsrl_safe_on_cpu(cpu
, msr
, &val
) || !val
)
1245 /* Detect active and valid domains for the given CPU, caller must
1246 * ensure the CPU belongs to the targeted package and CPU hotlug is disabled.
1248 static int rapl_detect_domains(struct rapl_package
*rp
, int cpu
)
1252 struct rapl_domain
*rd
;
1255 for (i
= 0; i
< RAPL_DOMAIN_MAX
; i
++) {
1256 /* use physical package id to read counters */
1257 if (!rapl_check_domain(cpu
, i
)) {
1258 rp
->domain_map
|= 1 << i
;
1259 pr_info("Found RAPL domain %s\n", rapl_domain_names
[i
]);
1262 rp
->nr_domains
= bitmap_weight(&rp
->domain_map
, RAPL_DOMAIN_MAX
);
1263 if (!rp
->nr_domains
) {
1264 pr_err("no valid rapl domains found in package %d\n", rp
->id
);
1268 pr_debug("found %d domains on package %d\n", rp
->nr_domains
, rp
->id
);
1270 rp
->domains
= kcalloc(rp
->nr_domains
+ 1, sizeof(struct rapl_domain
),
1276 rapl_init_domains(rp
);
1278 for (rd
= rp
->domains
; rd
< rp
->domains
+ rp
->nr_domains
; rd
++) {
1279 /* check if the domain is locked by BIOS */
1280 if (rapl_read_data_raw(rd
, FW_LOCK
, false, &locked
)) {
1281 pr_info("RAPL package %d domain %s locked by BIOS\n",
1283 rd
->state
|= DOMAIN_STATE_BIOS_LOCKED
;
1292 static bool is_package_new(int package
)
1294 struct rapl_package
*rp
;
1296 /* caller prevents cpu hotplug, there will be no new packages added
1297 * or deleted while traversing the package list, no need for locking.
1299 list_for_each_entry(rp
, &rapl_packages
, plist
)
1300 if (package
== rp
->id
)
1306 /* RAPL interface can be made of a two-level hierarchy: package level and domain
1307 * level. We first detect the number of packages then domains of each package.
1308 * We have to consider the possiblity of CPU online/offline due to hotplug and
1311 static int rapl_detect_topology(void)
1315 struct rapl_package
*new_package
, *rp
;
1317 for_each_online_cpu(i
) {
1318 phy_package_id
= topology_physical_package_id(i
);
1319 if (is_package_new(phy_package_id
)) {
1320 new_package
= kzalloc(sizeof(*rp
), GFP_KERNEL
);
1322 rapl_cleanup_data();
1325 /* add the new package to the list */
1326 new_package
->id
= phy_package_id
;
1327 new_package
->nr_cpus
= 1;
1329 /* check if the package contains valid domains */
1330 if (rapl_detect_domains(new_package
, i
) ||
1331 rapl_defaults
->check_unit(new_package
, i
)) {
1332 kfree(new_package
->domains
);
1334 /* free up the packages already initialized */
1335 rapl_cleanup_data();
1338 INIT_LIST_HEAD(&new_package
->plist
);
1339 list_add(&new_package
->plist
, &rapl_packages
);
1341 rp
= find_package_by_id(phy_package_id
);
1350 /* called from CPU hotplug notifier, hotplug lock held */
1351 static void rapl_remove_package(struct rapl_package
*rp
)
1353 struct rapl_domain
*rd
, *rd_package
= NULL
;
1355 for (rd
= rp
->domains
; rd
< rp
->domains
+ rp
->nr_domains
; rd
++) {
1356 if (rd
->id
== RAPL_DOMAIN_PACKAGE
) {
1360 pr_debug("remove package %d, %s domain\n", rp
->id
, rd
->name
);
1361 powercap_unregister_zone(control_type
, &rd
->power_zone
);
1363 /* do parent zone last */
1364 powercap_unregister_zone(control_type
, &rd_package
->power_zone
);
1365 list_del(&rp
->plist
);
1369 /* called from CPU hotplug notifier, hotplug lock held */
1370 static int rapl_add_package(int cpu
)
1374 struct rapl_package
*rp
;
1376 phy_package_id
= topology_physical_package_id(cpu
);
1377 rp
= kzalloc(sizeof(struct rapl_package
), GFP_KERNEL
);
1381 /* add the new package to the list */
1382 rp
->id
= phy_package_id
;
1384 /* check if the package contains valid domains */
1385 if (rapl_detect_domains(rp
, cpu
) ||
1386 rapl_defaults
->check_unit(rp
, cpu
)) {
1388 goto err_free_package
;
1390 if (!rapl_package_register_powercap(rp
)) {
1391 INIT_LIST_HEAD(&rp
->plist
);
1392 list_add(&rp
->plist
, &rapl_packages
);
1403 /* Handles CPU hotplug on multi-socket systems.
1404 * If a CPU goes online as the first CPU of the physical package
1405 * we add the RAPL package to the system. Similarly, when the last
1406 * CPU of the package is removed, we remove the RAPL package and its
1407 * associated domains. Cooling devices are handled accordingly at
1410 static int rapl_cpu_callback(struct notifier_block
*nfb
,
1411 unsigned long action
, void *hcpu
)
1413 unsigned long cpu
= (unsigned long)hcpu
;
1415 struct rapl_package
*rp
;
1417 phy_package_id
= topology_physical_package_id(cpu
);
1420 case CPU_ONLINE_FROZEN
:
1421 case CPU_DOWN_FAILED
:
1422 case CPU_DOWN_FAILED_FROZEN
:
1423 rp
= find_package_by_id(phy_package_id
);
1427 rapl_add_package(cpu
);
1429 case CPU_DOWN_PREPARE
:
1430 case CPU_DOWN_PREPARE_FROZEN
:
1431 rp
= find_package_by_id(phy_package_id
);
1434 if (--rp
->nr_cpus
== 0)
1435 rapl_remove_package(rp
);
1441 static struct notifier_block rapl_cpu_notifier
= {
1442 .notifier_call
= rapl_cpu_callback
,
1445 static int __init
rapl_init(void)
1448 const struct x86_cpu_id
*id
;
1450 id
= x86_match_cpu(rapl_ids
);
1452 pr_err("driver does not support CPU family %d model %d\n",
1453 boot_cpu_data
.x86
, boot_cpu_data
.x86_model
);
1458 rapl_defaults
= (struct rapl_defaults
*)id
->driver_data
;
1460 cpu_notifier_register_begin();
1462 /* prevent CPU hotplug during detection */
1464 ret
= rapl_detect_topology();
1468 if (rapl_register_powercap()) {
1469 rapl_cleanup_data();
1473 __register_hotcpu_notifier(&rapl_cpu_notifier
);
1476 cpu_notifier_register_done();
1481 static void __exit
rapl_exit(void)
1483 cpu_notifier_register_begin();
1485 __unregister_hotcpu_notifier(&rapl_cpu_notifier
);
1486 rapl_unregister_powercap();
1487 rapl_cleanup_data();
1489 cpu_notifier_register_done();
1492 module_init(rapl_init
);
1493 module_exit(rapl_exit
);
1495 MODULE_DESCRIPTION("Driver for Intel RAPL (Running Average Power Limit)");
1496 MODULE_AUTHOR("Jacob Pan <jacob.jun.pan@intel.com>");
1497 MODULE_LICENSE("GPL v2");