Commit | Line | Data |
---|---|---|
b7169166 RR |
1 | /* |
2 | * Performance events - AMD IBS | |
3 | * | |
4 | * Copyright (C) 2011 Advanced Micro Devices, Inc., Robert Richter | |
5 | * | |
6 | * For licencing details see kernel-base/COPYING | |
7 | */ | |
8 | ||
9 | #include <linux/perf_event.h> | |
10 | #include <linux/module.h> | |
11 | #include <linux/pci.h> | |
12 | ||
13 | #include <asm/apic.h> | |
14 | ||
15 | static u32 ibs_caps; | |
16 | ||
17 | #if defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) | |
18 | ||
19 | static struct pmu perf_ibs; | |
20 | ||
21 | static int perf_ibs_init(struct perf_event *event) | |
22 | { | |
23 | if (perf_ibs.type != event->attr.type) | |
24 | return -ENOENT; | |
25 | return 0; | |
26 | } | |
27 | ||
28 | static int perf_ibs_add(struct perf_event *event, int flags) | |
29 | { | |
30 | return 0; | |
31 | } | |
32 | ||
33 | static void perf_ibs_del(struct perf_event *event, int flags) | |
34 | { | |
35 | } | |
36 | ||
37 | static struct pmu perf_ibs = { | |
38 | .event_init= perf_ibs_init, | |
39 | .add= perf_ibs_add, | |
40 | .del= perf_ibs_del, | |
41 | }; | |
42 | ||
43 | static __init int perf_event_ibs_init(void) | |
44 | { | |
45 | if (!ibs_caps) | |
46 | return -ENODEV; /* ibs not supported by the cpu */ | |
47 | ||
48 | perf_pmu_register(&perf_ibs, "ibs", -1); | |
49 | printk(KERN_INFO "perf: AMD IBS detected (0x%08x)\n", ibs_caps); | |
50 | ||
51 | return 0; | |
52 | } | |
53 | ||
54 | #else /* defined(CONFIG_PERF_EVENTS) && defined(CONFIG_CPU_SUP_AMD) */ | |
55 | ||
56 | static __init int perf_event_ibs_init(void) { return 0; } | |
57 | ||
58 | #endif | |
59 | ||
60 | /* IBS - apic initialization, for perf and oprofile */ | |
61 | ||
62 | static __init u32 __get_ibs_caps(void) | |
63 | { | |
64 | u32 caps; | |
65 | unsigned int max_level; | |
66 | ||
67 | if (!boot_cpu_has(X86_FEATURE_IBS)) | |
68 | return 0; | |
69 | ||
70 | /* check IBS cpuid feature flags */ | |
71 | max_level = cpuid_eax(0x80000000); | |
72 | if (max_level < IBS_CPUID_FEATURES) | |
73 | return IBS_CAPS_DEFAULT; | |
74 | ||
75 | caps = cpuid_eax(IBS_CPUID_FEATURES); | |
76 | if (!(caps & IBS_CAPS_AVAIL)) | |
77 | /* cpuid flags not valid */ | |
78 | return IBS_CAPS_DEFAULT; | |
79 | ||
80 | return caps; | |
81 | } | |
82 | ||
83 | u32 get_ibs_caps(void) | |
84 | { | |
85 | return ibs_caps; | |
86 | } | |
87 | ||
88 | EXPORT_SYMBOL(get_ibs_caps); | |
89 | ||
90 | static inline int get_eilvt(int offset) | |
91 | { | |
92 | return !setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 1); | |
93 | } | |
94 | ||
95 | static inline int put_eilvt(int offset) | |
96 | { | |
97 | return !setup_APIC_eilvt(offset, 0, 0, 1); | |
98 | } | |
99 | ||
100 | /* | |
101 | * Check and reserve APIC extended interrupt LVT offset for IBS if available. | |
102 | */ | |
103 | static inline int ibs_eilvt_valid(void) | |
104 | { | |
105 | int offset; | |
106 | u64 val; | |
107 | int valid = 0; | |
108 | ||
109 | preempt_disable(); | |
110 | ||
111 | rdmsrl(MSR_AMD64_IBSCTL, val); | |
112 | offset = val & IBSCTL_LVT_OFFSET_MASK; | |
113 | ||
114 | if (!(val & IBSCTL_LVT_OFFSET_VALID)) { | |
115 | pr_err(FW_BUG "cpu %d, invalid IBS interrupt offset %d (MSR%08X=0x%016llx)\n", | |
116 | smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); | |
117 | goto out; | |
118 | } | |
119 | ||
120 | if (!get_eilvt(offset)) { | |
121 | pr_err(FW_BUG "cpu %d, IBS interrupt offset %d not available (MSR%08X=0x%016llx)\n", | |
122 | smp_processor_id(), offset, MSR_AMD64_IBSCTL, val); | |
123 | goto out; | |
124 | } | |
125 | ||
126 | valid = 1; | |
127 | out: | |
128 | preempt_enable(); | |
129 | ||
130 | return valid; | |
131 | } | |
132 | ||
133 | static int setup_ibs_ctl(int ibs_eilvt_off) | |
134 | { | |
135 | struct pci_dev *cpu_cfg; | |
136 | int nodes; | |
137 | u32 value = 0; | |
138 | ||
139 | nodes = 0; | |
140 | cpu_cfg = NULL; | |
141 | do { | |
142 | cpu_cfg = pci_get_device(PCI_VENDOR_ID_AMD, | |
143 | PCI_DEVICE_ID_AMD_10H_NB_MISC, | |
144 | cpu_cfg); | |
145 | if (!cpu_cfg) | |
146 | break; | |
147 | ++nodes; | |
148 | pci_write_config_dword(cpu_cfg, IBSCTL, ibs_eilvt_off | |
149 | | IBSCTL_LVT_OFFSET_VALID); | |
150 | pci_read_config_dword(cpu_cfg, IBSCTL, &value); | |
151 | if (value != (ibs_eilvt_off | IBSCTL_LVT_OFFSET_VALID)) { | |
152 | pci_dev_put(cpu_cfg); | |
153 | printk(KERN_DEBUG "Failed to setup IBS LVT offset, " | |
154 | "IBSCTL = 0x%08x\n", value); | |
155 | return -EINVAL; | |
156 | } | |
157 | } while (1); | |
158 | ||
159 | if (!nodes) { | |
160 | printk(KERN_DEBUG "No CPU node configured for IBS\n"); | |
161 | return -ENODEV; | |
162 | } | |
163 | ||
164 | return 0; | |
165 | } | |
166 | ||
167 | /* | |
168 | * This runs only on the current cpu. We try to find an LVT offset and | |
169 | * setup the local APIC. For this we must disable preemption. On | |
170 | * success we initialize all nodes with this offset. This updates then | |
171 | * the offset in the IBS_CTL per-node msr. The per-core APIC setup of | |
172 | * the IBS interrupt vector is handled by perf_ibs_cpu_notifier that | |
173 | * is using the new offset. | |
174 | */ | |
175 | static int force_ibs_eilvt_setup(void) | |
176 | { | |
177 | int offset; | |
178 | int ret; | |
179 | ||
180 | preempt_disable(); | |
181 | /* find the next free available EILVT entry, skip offset 0 */ | |
182 | for (offset = 1; offset < APIC_EILVT_NR_MAX; offset++) { | |
183 | if (get_eilvt(offset)) | |
184 | break; | |
185 | } | |
186 | preempt_enable(); | |
187 | ||
188 | if (offset == APIC_EILVT_NR_MAX) { | |
189 | printk(KERN_DEBUG "No EILVT entry available\n"); | |
190 | return -EBUSY; | |
191 | } | |
192 | ||
193 | ret = setup_ibs_ctl(offset); | |
194 | if (ret) | |
195 | goto out; | |
196 | ||
197 | if (!ibs_eilvt_valid()) { | |
198 | ret = -EFAULT; | |
199 | goto out; | |
200 | } | |
201 | ||
202 | pr_err(FW_BUG "using offset %d for IBS interrupts\n", offset); | |
203 | pr_err(FW_BUG "workaround enabled for IBS LVT offset\n"); | |
204 | ||
205 | return 0; | |
206 | out: | |
207 | preempt_disable(); | |
208 | put_eilvt(offset); | |
209 | preempt_enable(); | |
210 | return ret; | |
211 | } | |
212 | ||
213 | static inline int get_ibs_lvt_offset(void) | |
214 | { | |
215 | u64 val; | |
216 | ||
217 | rdmsrl(MSR_AMD64_IBSCTL, val); | |
218 | if (!(val & IBSCTL_LVT_OFFSET_VALID)) | |
219 | return -EINVAL; | |
220 | ||
221 | return val & IBSCTL_LVT_OFFSET_MASK; | |
222 | } | |
223 | ||
224 | static void setup_APIC_ibs(void *dummy) | |
225 | { | |
226 | int offset; | |
227 | ||
228 | offset = get_ibs_lvt_offset(); | |
229 | if (offset < 0) | |
230 | goto failed; | |
231 | ||
232 | if (!setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_NMI, 0)) | |
233 | return; | |
234 | failed: | |
235 | pr_warn("perf: IBS APIC setup failed on cpu #%d\n", | |
236 | smp_processor_id()); | |
237 | } | |
238 | ||
239 | static void clear_APIC_ibs(void *dummy) | |
240 | { | |
241 | int offset; | |
242 | ||
243 | offset = get_ibs_lvt_offset(); | |
244 | if (offset >= 0) | |
245 | setup_APIC_eilvt(offset, 0, APIC_EILVT_MSG_FIX, 1); | |
246 | } | |
247 | ||
248 | static int __cpuinit | |
249 | perf_ibs_cpu_notifier(struct notifier_block *self, unsigned long action, void *hcpu) | |
250 | { | |
251 | switch (action & ~CPU_TASKS_FROZEN) { | |
252 | case CPU_STARTING: | |
253 | setup_APIC_ibs(NULL); | |
254 | break; | |
255 | case CPU_DYING: | |
256 | clear_APIC_ibs(NULL); | |
257 | break; | |
258 | default: | |
259 | break; | |
260 | } | |
261 | ||
262 | return NOTIFY_OK; | |
263 | } | |
264 | ||
265 | static __init int amd_ibs_init(void) | |
266 | { | |
267 | u32 caps; | |
268 | int ret; | |
269 | ||
270 | caps = __get_ibs_caps(); | |
271 | if (!caps) | |
272 | return -ENODEV; /* ibs not supported by the cpu */ | |
273 | ||
274 | if (!ibs_eilvt_valid()) { | |
275 | ret = force_ibs_eilvt_setup(); | |
276 | if (ret) { | |
277 | pr_err("Failed to setup IBS, %d\n", ret); | |
278 | return ret; | |
279 | } | |
280 | } | |
281 | ||
282 | get_online_cpus(); | |
283 | ibs_caps = caps; | |
284 | /* make ibs_caps visible to other cpus: */ | |
285 | smp_mb(); | |
286 | perf_cpu_notifier(perf_ibs_cpu_notifier); | |
287 | smp_call_function(setup_APIC_ibs, NULL, 1); | |
288 | put_online_cpus(); | |
289 | ||
290 | return perf_event_ibs_init(); | |
291 | } | |
292 | ||
293 | /* Since we need the pci subsystem to init ibs we can't do this earlier: */ | |
294 | device_initcall(amd_ibs_init); |