Commit | Line | Data |
---|---|---|
14f70012 DCZ |
1 | /* |
2 | * Linux performance counter support for MIPS. | |
3 | * | |
4 | * Copyright (C) 2010 MIPS Technologies, Inc. | |
5 | * Author: Deng-Cheng Zhu | |
6 | * | |
7 | * This code is based on the implementation for ARM, which is in turn | |
8 | * based on the sparc64 perf event code and the x86 code. Performance | |
9 | * counter access is based on the MIPS Oprofile code. | |
10 | * | |
11 | * This program is free software; you can redistribute it and/or modify | |
12 | * it under the terms of the GNU General Public License version 2 as | |
13 | * published by the Free Software Foundation. | |
14 | */ | |
15 | ||
16 | #include <linux/cpumask.h> | |
17 | #include <linux/interrupt.h> | |
18 | #include <linux/smp.h> | |
19 | #include <linux/kernel.h> | |
20 | #include <linux/perf_event.h> | |
21 | #include <linux/uaccess.h> | |
22 | ||
23 | #include <asm/irq.h> | |
24 | #include <asm/irq_regs.h> | |
25 | #include <asm/stacktrace.h> | |
26 | #include <asm/time.h> /* For perf_irq */ | |
27 | ||
28 | /* These are for 32bit counters. For 64bit ones, define them accordingly. */ | |
29 | #define MAX_PERIOD ((1ULL << 32) - 1) | |
30 | #define VALID_COUNT 0x7fffffff | |
31 | #define TOTAL_BITS 32 | |
32 | #define HIGHEST_BIT 31 | |
33 | ||
34 | #define MIPS_MAX_HWEVENTS 4 | |
35 | ||
36 | struct cpu_hw_events { | |
37 | /* Array of events on this cpu. */ | |
38 | struct perf_event *events[MIPS_MAX_HWEVENTS]; | |
39 | ||
40 | /* | |
41 | * Set the bit (indexed by the counter number) when the counter | |
42 | * is used for an event. | |
43 | */ | |
44 | unsigned long used_mask[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; | |
45 | ||
46 | /* | |
47 | * The borrowed MSB for the performance counter. A MIPS performance | |
48 | * counter uses its bit 31 (for 32bit counters) or bit 63 (for 64bit | |
49 | * counters) as a factor of determining whether a counter overflow | |
50 | * should be signaled. So here we use a separate MSB for each | |
51 | * counter to make things easy. | |
52 | */ | |
53 | unsigned long msbs[BITS_TO_LONGS(MIPS_MAX_HWEVENTS)]; | |
54 | ||
55 | /* | |
56 | * Software copy of the control register for each performance counter. | |
57 | * MIPS CPUs vary in performance counters. They use this differently, | |
58 | * and even may not use it. | |
59 | */ | |
60 | unsigned int saved_ctrl[MIPS_MAX_HWEVENTS]; | |
61 | }; | |
62 | DEFINE_PER_CPU(struct cpu_hw_events, cpu_hw_events) = { | |
63 | .saved_ctrl = {0}, | |
64 | }; | |
65 | ||
66 | /* The description of MIPS performance events. */ | |
67 | struct mips_perf_event { | |
68 | unsigned int event_id; | |
69 | /* | |
70 | * MIPS performance counters are indexed starting from 0. | |
71 | * CNTR_EVEN indicates the indexes of the counters to be used are | |
72 | * even numbers. | |
73 | */ | |
74 | unsigned int cntr_mask; | |
75 | #define CNTR_EVEN 0x55555555 | |
76 | #define CNTR_ODD 0xaaaaaaaa | |
77 | #ifdef CONFIG_MIPS_MT_SMP | |
78 | enum { | |
79 | T = 0, | |
80 | V = 1, | |
81 | P = 2, | |
82 | } range; | |
83 | #else | |
84 | #define T | |
85 | #define V | |
86 | #define P | |
87 | #endif | |
88 | }; | |
89 | ||
90 | #define UNSUPPORTED_PERF_EVENT_ID 0xffffffff | |
91 | #define C(x) PERF_COUNT_HW_CACHE_##x | |
92 | ||
93 | struct mips_pmu { | |
94 | const char *name; | |
95 | int irq; | |
96 | irqreturn_t (*handle_irq)(int irq, void *dev); | |
97 | int (*handle_shared_irq)(void); | |
98 | void (*start)(void); | |
99 | void (*stop)(void); | |
100 | int (*alloc_counter)(struct cpu_hw_events *cpuc, | |
101 | struct hw_perf_event *hwc); | |
102 | u64 (*read_counter)(unsigned int idx); | |
103 | void (*write_counter)(unsigned int idx, u64 val); | |
104 | void (*enable_event)(struct hw_perf_event *evt, int idx); | |
105 | void (*disable_event)(int idx); | |
106 | const struct mips_perf_event (*general_event_map)[PERF_COUNT_HW_MAX]; | |
107 | const struct mips_perf_event (*cache_event_map) | |
108 | [PERF_COUNT_HW_CACHE_MAX] | |
109 | [PERF_COUNT_HW_CACHE_OP_MAX] | |
110 | [PERF_COUNT_HW_CACHE_RESULT_MAX]; | |
111 | unsigned int num_counters; | |
112 | }; | |
113 | ||
114 | static const struct mips_pmu *mipspmu; | |
115 | ||
116 | static int | |
117 | mipspmu_event_set_period(struct perf_event *event, | |
118 | struct hw_perf_event *hwc, | |
119 | int idx) | |
120 | { | |
121 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | |
122 | s64 left = local64_read(&hwc->period_left); | |
123 | s64 period = hwc->sample_period; | |
124 | int ret = 0; | |
125 | u64 uleft; | |
126 | unsigned long flags; | |
127 | ||
128 | if (unlikely(left <= -period)) { | |
129 | left = period; | |
130 | local64_set(&hwc->period_left, left); | |
131 | hwc->last_period = period; | |
132 | ret = 1; | |
133 | } | |
134 | ||
135 | if (unlikely(left <= 0)) { | |
136 | left += period; | |
137 | local64_set(&hwc->period_left, left); | |
138 | hwc->last_period = period; | |
139 | ret = 1; | |
140 | } | |
141 | ||
142 | if (left > (s64)MAX_PERIOD) | |
143 | left = MAX_PERIOD; | |
144 | ||
145 | local64_set(&hwc->prev_count, (u64)-left); | |
146 | ||
147 | local_irq_save(flags); | |
148 | uleft = (u64)(-left) & MAX_PERIOD; | |
149 | uleft > VALID_COUNT ? | |
150 | set_bit(idx, cpuc->msbs) : clear_bit(idx, cpuc->msbs); | |
151 | mipspmu->write_counter(idx, (u64)(-left) & VALID_COUNT); | |
152 | local_irq_restore(flags); | |
153 | ||
154 | perf_event_update_userpage(event); | |
155 | ||
156 | return ret; | |
157 | } | |
158 | ||
159 | static int mipspmu_enable(struct perf_event *event) | |
160 | { | |
161 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | |
162 | struct hw_perf_event *hwc = &event->hw; | |
163 | int idx; | |
164 | int err = 0; | |
165 | ||
166 | /* To look for a free counter for this event. */ | |
167 | idx = mipspmu->alloc_counter(cpuc, hwc); | |
168 | if (idx < 0) { | |
169 | err = idx; | |
170 | goto out; | |
171 | } | |
172 | ||
173 | /* | |
174 | * If there is an event in the counter we are going to use then | |
175 | * make sure it is disabled. | |
176 | */ | |
177 | event->hw.idx = idx; | |
178 | mipspmu->disable_event(idx); | |
179 | cpuc->events[idx] = event; | |
180 | ||
181 | /* Set the period for the event. */ | |
182 | mipspmu_event_set_period(event, hwc, idx); | |
183 | ||
184 | /* Enable the event. */ | |
185 | mipspmu->enable_event(hwc, idx); | |
186 | ||
187 | /* Propagate our changes to the userspace mapping. */ | |
188 | perf_event_update_userpage(event); | |
189 | ||
190 | out: | |
191 | return err; | |
192 | } | |
193 | ||
194 | static void mipspmu_event_update(struct perf_event *event, | |
195 | struct hw_perf_event *hwc, | |
196 | int idx) | |
197 | { | |
198 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | |
199 | unsigned long flags; | |
200 | int shift = 64 - TOTAL_BITS; | |
201 | s64 prev_raw_count, new_raw_count; | |
202 | s64 delta; | |
203 | ||
204 | again: | |
205 | prev_raw_count = local64_read(&hwc->prev_count); | |
206 | local_irq_save(flags); | |
207 | /* Make the counter value be a "real" one. */ | |
208 | new_raw_count = mipspmu->read_counter(idx); | |
209 | if (new_raw_count & (test_bit(idx, cpuc->msbs) << HIGHEST_BIT)) { | |
210 | new_raw_count &= VALID_COUNT; | |
211 | clear_bit(idx, cpuc->msbs); | |
212 | } else | |
213 | new_raw_count |= (test_bit(idx, cpuc->msbs) << HIGHEST_BIT); | |
214 | local_irq_restore(flags); | |
215 | ||
216 | if (local64_cmpxchg(&hwc->prev_count, prev_raw_count, | |
217 | new_raw_count) != prev_raw_count) | |
218 | goto again; | |
219 | ||
220 | delta = (new_raw_count << shift) - (prev_raw_count << shift); | |
221 | delta >>= shift; | |
222 | ||
223 | local64_add(delta, &event->count); | |
224 | local64_sub(delta, &hwc->period_left); | |
225 | ||
226 | return; | |
227 | } | |
228 | ||
229 | static void mipspmu_disable(struct perf_event *event) | |
230 | { | |
231 | struct cpu_hw_events *cpuc = &__get_cpu_var(cpu_hw_events); | |
232 | struct hw_perf_event *hwc = &event->hw; | |
233 | int idx = hwc->idx; | |
234 | ||
235 | ||
236 | WARN_ON(idx < 0 || idx >= mipspmu->num_counters); | |
237 | ||
238 | /* We are working on a local event. */ | |
239 | mipspmu->disable_event(idx); | |
240 | ||
241 | barrier(); | |
242 | ||
243 | mipspmu_event_update(event, hwc, idx); | |
244 | cpuc->events[idx] = NULL; | |
245 | clear_bit(idx, cpuc->used_mask); | |
246 | ||
247 | perf_event_update_userpage(event); | |
248 | } | |
249 | ||
250 | static void mipspmu_unthrottle(struct perf_event *event) | |
251 | { | |
252 | struct hw_perf_event *hwc = &event->hw; | |
253 | ||
254 | mipspmu->enable_event(hwc, hwc->idx); | |
255 | } | |
256 | ||
257 | static void mipspmu_read(struct perf_event *event) | |
258 | { | |
259 | struct hw_perf_event *hwc = &event->hw; | |
260 | ||
261 | /* Don't read disabled counters! */ | |
262 | if (hwc->idx < 0) | |
263 | return; | |
264 | ||
265 | mipspmu_event_update(event, hwc, hwc->idx); | |
266 | } | |
267 | ||
268 | static struct pmu pmu = { | |
269 | .enable = mipspmu_enable, | |
270 | .disable = mipspmu_disable, | |
271 | .unthrottle = mipspmu_unthrottle, | |
272 | .read = mipspmu_read, | |
273 | }; | |
274 | ||
275 | static atomic_t active_events = ATOMIC_INIT(0); | |
276 | static DEFINE_MUTEX(pmu_reserve_mutex); | |
277 | static int (*save_perf_irq)(void); | |
278 | ||
279 | static int mipspmu_get_irq(void) | |
280 | { | |
281 | int err; | |
282 | ||
283 | if (mipspmu->irq >= 0) { | |
284 | /* Request my own irq handler. */ | |
285 | err = request_irq(mipspmu->irq, mipspmu->handle_irq, | |
286 | IRQF_DISABLED | IRQF_NOBALANCING, | |
287 | "mips_perf_pmu", NULL); | |
288 | if (err) { | |
289 | pr_warning("Unable to request IRQ%d for MIPS " | |
290 | "performance counters!\n", mipspmu->irq); | |
291 | } | |
292 | } else if (cp0_perfcount_irq < 0) { | |
293 | /* | |
294 | * We are sharing the irq number with the timer interrupt. | |
295 | */ | |
296 | save_perf_irq = perf_irq; | |
297 | perf_irq = mipspmu->handle_shared_irq; | |
298 | err = 0; | |
299 | } else { | |
300 | pr_warning("The platform hasn't properly defined its " | |
301 | "interrupt controller.\n"); | |
302 | err = -ENOENT; | |
303 | } | |
304 | ||
305 | return err; | |
306 | } | |
307 | ||
308 | static void mipspmu_free_irq(void) | |
309 | { | |
310 | if (mipspmu->irq >= 0) | |
311 | free_irq(mipspmu->irq, NULL); | |
312 | else if (cp0_perfcount_irq < 0) | |
313 | perf_irq = save_perf_irq; | |
314 | } | |
315 | ||
316 | static inline unsigned int | |
317 | mipspmu_perf_event_encode(const struct mips_perf_event *pev) | |
318 | { | |
319 | /* | |
320 | * Top 8 bits for range, next 16 bits for cntr_mask, lowest 8 bits for | |
321 | * event_id. | |
322 | */ | |
323 | #ifdef CONFIG_MIPS_MT_SMP | |
324 | return ((unsigned int)pev->range << 24) | | |
325 | (pev->cntr_mask & 0xffff00) | | |
326 | (pev->event_id & 0xff); | |
327 | #else | |
328 | return (pev->cntr_mask & 0xffff00) | | |
329 | (pev->event_id & 0xff); | |
330 | #endif | |
331 | } | |
332 | ||
333 | static const struct mips_perf_event * | |
334 | mipspmu_map_general_event(int idx) | |
335 | { | |
336 | const struct mips_perf_event *pev; | |
337 | ||
338 | pev = ((*mipspmu->general_event_map)[idx].event_id == | |
339 | UNSUPPORTED_PERF_EVENT_ID ? ERR_PTR(-EOPNOTSUPP) : | |
340 | &(*mipspmu->general_event_map)[idx]); | |
341 | ||
342 | return pev; | |
343 | } | |
344 | ||
345 | static const struct mips_perf_event * | |
346 | mipspmu_map_cache_event(u64 config) | |
347 | { | |
348 | unsigned int cache_type, cache_op, cache_result; | |
349 | const struct mips_perf_event *pev; | |
350 | ||
351 | cache_type = (config >> 0) & 0xff; | |
352 | if (cache_type >= PERF_COUNT_HW_CACHE_MAX) | |
353 | return ERR_PTR(-EINVAL); | |
354 | ||
355 | cache_op = (config >> 8) & 0xff; | |
356 | if (cache_op >= PERF_COUNT_HW_CACHE_OP_MAX) | |
357 | return ERR_PTR(-EINVAL); | |
358 | ||
359 | cache_result = (config >> 16) & 0xff; | |
360 | if (cache_result >= PERF_COUNT_HW_CACHE_RESULT_MAX) | |
361 | return ERR_PTR(-EINVAL); | |
362 | ||
363 | pev = &((*mipspmu->cache_event_map) | |
364 | [cache_type] | |
365 | [cache_op] | |
366 | [cache_result]); | |
367 | ||
368 | if (pev->event_id == UNSUPPORTED_PERF_EVENT_ID) | |
369 | return ERR_PTR(-EOPNOTSUPP); | |
370 | ||
371 | return pev; | |
372 | ||
373 | } | |
374 | ||
375 | static int validate_event(struct cpu_hw_events *cpuc, | |
376 | struct perf_event *event) | |
377 | { | |
378 | struct hw_perf_event fake_hwc = event->hw; | |
379 | ||
380 | if (event->pmu && event->pmu != &pmu) | |
381 | return 0; | |
382 | ||
383 | return mipspmu->alloc_counter(cpuc, &fake_hwc) >= 0; | |
384 | } | |
385 | ||
386 | static int validate_group(struct perf_event *event) | |
387 | { | |
388 | struct perf_event *sibling, *leader = event->group_leader; | |
389 | struct cpu_hw_events fake_cpuc; | |
390 | ||
391 | memset(&fake_cpuc, 0, sizeof(fake_cpuc)); | |
392 | ||
393 | if (!validate_event(&fake_cpuc, leader)) | |
394 | return -ENOSPC; | |
395 | ||
396 | list_for_each_entry(sibling, &leader->sibling_list, group_entry) { | |
397 | if (!validate_event(&fake_cpuc, sibling)) | |
398 | return -ENOSPC; | |
399 | } | |
400 | ||
401 | if (!validate_event(&fake_cpuc, event)) | |
402 | return -ENOSPC; | |
403 | ||
404 | return 0; | |
405 | } | |
406 | ||
407 | /* | |
408 | * mipsxx/rm9000/loongson2 have different performance counters, they have | |
409 | * specific low-level init routines. | |
410 | */ | |
411 | static int __hw_perf_event_init(struct perf_event *event); | |
412 | ||
413 | static void hw_perf_event_destroy(struct perf_event *event) | |
414 | { | |
415 | if (atomic_dec_and_mutex_lock(&active_events, | |
416 | &pmu_reserve_mutex)) { | |
417 | /* | |
418 | * We must not call the destroy function with interrupts | |
419 | * disabled. | |
420 | */ | |
421 | on_each_cpu(reset_counters, | |
422 | (void *)(long)mipspmu->num_counters, 1); | |
423 | mipspmu_free_irq(); | |
424 | mutex_unlock(&pmu_reserve_mutex); | |
425 | } | |
426 | } | |
427 | ||
428 | const struct pmu *hw_perf_event_init(struct perf_event *event) | |
429 | { | |
430 | int err = 0; | |
431 | ||
432 | if (!mipspmu || event->cpu >= nr_cpumask_bits || | |
433 | (event->cpu >= 0 && !cpu_online(event->cpu))) | |
434 | return ERR_PTR(-ENODEV); | |
435 | ||
436 | if (!atomic_inc_not_zero(&active_events)) { | |
437 | if (atomic_read(&active_events) > MIPS_MAX_HWEVENTS) { | |
438 | atomic_dec(&active_events); | |
439 | return ERR_PTR(-ENOSPC); | |
440 | } | |
441 | ||
442 | mutex_lock(&pmu_reserve_mutex); | |
443 | if (atomic_read(&active_events) == 0) | |
444 | err = mipspmu_get_irq(); | |
445 | ||
446 | if (!err) | |
447 | atomic_inc(&active_events); | |
448 | mutex_unlock(&pmu_reserve_mutex); | |
449 | } | |
450 | ||
451 | if (err) | |
452 | return ERR_PTR(err); | |
453 | ||
454 | err = __hw_perf_event_init(event); | |
455 | if (err) | |
456 | hw_perf_event_destroy(event); | |
457 | ||
458 | return err ? ERR_PTR(err) : &pmu; | |
459 | } | |
460 | ||
461 | void hw_perf_enable(void) | |
462 | { | |
463 | if (mipspmu) | |
464 | mipspmu->start(); | |
465 | } | |
466 | ||
467 | void hw_perf_disable(void) | |
468 | { | |
469 | if (mipspmu) | |
470 | mipspmu->stop(); | |
471 | } | |
472 | ||
473 | /* This is needed by specific irq handlers in perf_event_*.c */ | |
474 | static void | |
475 | handle_associated_event(struct cpu_hw_events *cpuc, | |
476 | int idx, struct perf_sample_data *data, struct pt_regs *regs) | |
477 | { | |
478 | struct perf_event *event = cpuc->events[idx]; | |
479 | struct hw_perf_event *hwc = &event->hw; | |
480 | ||
481 | mipspmu_event_update(event, hwc, idx); | |
482 | data->period = event->hw.last_period; | |
483 | if (!mipspmu_event_set_period(event, hwc, idx)) | |
484 | return; | |
485 | ||
486 | if (perf_event_overflow(event, 0, data, regs)) | |
487 | mipspmu->disable_event(idx); | |
488 | } |