perf_counter: unify irq output code
[deliverable/linux.git] / kernel / perf_counter.c
1 /*
2 * Performance counter core code
3 *
4 * Copyright(C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright(C) 2008 Red Hat, Inc., Ingo Molnar
6 *
7 * For licencing details see kernel-base/COPYING
8 */
9
10 #include <linux/fs.h>
11 #include <linux/cpu.h>
12 #include <linux/smp.h>
13 #include <linux/file.h>
14 #include <linux/poll.h>
15 #include <linux/sysfs.h>
16 #include <linux/ptrace.h>
17 #include <linux/percpu.h>
18 #include <linux/uaccess.h>
19 #include <linux/syscalls.h>
20 #include <linux/anon_inodes.h>
21 #include <linux/kernel_stat.h>
22 #include <linux/perf_counter.h>
23 #include <linux/mm.h>
24 #include <linux/vmstat.h>
25 #include <linux/rculist.h>
26
27 #include <asm/irq_regs.h>
28
29 /*
30 * Each CPU has a list of per CPU counters:
31 */
32 DEFINE_PER_CPU(struct perf_cpu_context, perf_cpu_context);
33
34 int perf_max_counters __read_mostly = 1;
35 static int perf_reserved_percpu __read_mostly;
36 static int perf_overcommit __read_mostly = 1;
37
38 /*
39 * Mutex for (sysadmin-configurable) counter reservations:
40 */
41 static DEFINE_MUTEX(perf_resource_mutex);
42
43 /*
44 * Architecture provided APIs - weak aliases:
45 */
46 extern __weak const struct hw_perf_counter_ops *
47 hw_perf_counter_init(struct perf_counter *counter)
48 {
49 return NULL;
50 }
51
52 u64 __weak hw_perf_save_disable(void) { return 0; }
53 void __weak hw_perf_restore(u64 ctrl) { barrier(); }
54 void __weak hw_perf_counter_setup(int cpu) { barrier(); }
55 int __weak hw_perf_group_sched_in(struct perf_counter *group_leader,
56 struct perf_cpu_context *cpuctx,
57 struct perf_counter_context *ctx, int cpu)
58 {
59 return 0;
60 }
61
62 void __weak perf_counter_print_debug(void) { }
63
64 static void
65 list_add_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
66 {
67 struct perf_counter *group_leader = counter->group_leader;
68
69 /*
70 * Depending on whether it is a standalone or sibling counter,
71 * add it straight to the context's counter list, or to the group
72 * leader's sibling list:
73 */
74 if (counter->group_leader == counter)
75 list_add_tail(&counter->list_entry, &ctx->counter_list);
76 else
77 list_add_tail(&counter->list_entry, &group_leader->sibling_list);
78
79 list_add_rcu(&counter->event_entry, &ctx->event_list);
80 }
81
82 static void
83 list_del_counter(struct perf_counter *counter, struct perf_counter_context *ctx)
84 {
85 struct perf_counter *sibling, *tmp;
86
87 list_del_init(&counter->list_entry);
88 list_del_rcu(&counter->event_entry);
89
90 /*
91 * If this was a group counter with sibling counters then
92 * upgrade the siblings to singleton counters by adding them
93 * to the context list directly:
94 */
95 list_for_each_entry_safe(sibling, tmp,
96 &counter->sibling_list, list_entry) {
97
98 list_move_tail(&sibling->list_entry, &ctx->counter_list);
99 sibling->group_leader = sibling;
100 }
101 }
102
103 static void
104 counter_sched_out(struct perf_counter *counter,
105 struct perf_cpu_context *cpuctx,
106 struct perf_counter_context *ctx)
107 {
108 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
109 return;
110
111 counter->state = PERF_COUNTER_STATE_INACTIVE;
112 counter->hw_ops->disable(counter);
113 counter->oncpu = -1;
114
115 if (!is_software_counter(counter))
116 cpuctx->active_oncpu--;
117 ctx->nr_active--;
118 if (counter->hw_event.exclusive || !cpuctx->active_oncpu)
119 cpuctx->exclusive = 0;
120 }
121
122 static void
123 group_sched_out(struct perf_counter *group_counter,
124 struct perf_cpu_context *cpuctx,
125 struct perf_counter_context *ctx)
126 {
127 struct perf_counter *counter;
128
129 if (group_counter->state != PERF_COUNTER_STATE_ACTIVE)
130 return;
131
132 counter_sched_out(group_counter, cpuctx, ctx);
133
134 /*
135 * Schedule out siblings (if any):
136 */
137 list_for_each_entry(counter, &group_counter->sibling_list, list_entry)
138 counter_sched_out(counter, cpuctx, ctx);
139
140 if (group_counter->hw_event.exclusive)
141 cpuctx->exclusive = 0;
142 }
143
144 /*
145 * Cross CPU call to remove a performance counter
146 *
147 * We disable the counter on the hardware level first. After that we
148 * remove it from the context list.
149 */
150 static void __perf_counter_remove_from_context(void *info)
151 {
152 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
153 struct perf_counter *counter = info;
154 struct perf_counter_context *ctx = counter->ctx;
155 unsigned long flags;
156 u64 perf_flags;
157
158 /*
159 * If this is a task context, we need to check whether it is
160 * the current task context of this cpu. If not it has been
161 * scheduled out before the smp call arrived.
162 */
163 if (ctx->task && cpuctx->task_ctx != ctx)
164 return;
165
166 curr_rq_lock_irq_save(&flags);
167 spin_lock(&ctx->lock);
168
169 counter_sched_out(counter, cpuctx, ctx);
170
171 counter->task = NULL;
172 ctx->nr_counters--;
173
174 /*
175 * Protect the list operation against NMI by disabling the
176 * counters on a global level. NOP for non NMI based counters.
177 */
178 perf_flags = hw_perf_save_disable();
179 list_del_counter(counter, ctx);
180 hw_perf_restore(perf_flags);
181
182 if (!ctx->task) {
183 /*
184 * Allow more per task counters with respect to the
185 * reservation:
186 */
187 cpuctx->max_pertask =
188 min(perf_max_counters - ctx->nr_counters,
189 perf_max_counters - perf_reserved_percpu);
190 }
191
192 spin_unlock(&ctx->lock);
193 curr_rq_unlock_irq_restore(&flags);
194 }
195
196
197 /*
198 * Remove the counter from a task's (or a CPU's) list of counters.
199 *
200 * Must be called with counter->mutex and ctx->mutex held.
201 *
202 * CPU counters are removed with a smp call. For task counters we only
203 * call when the task is on a CPU.
204 */
205 static void perf_counter_remove_from_context(struct perf_counter *counter)
206 {
207 struct perf_counter_context *ctx = counter->ctx;
208 struct task_struct *task = ctx->task;
209
210 if (!task) {
211 /*
212 * Per cpu counters are removed via an smp call and
213 * the removal is always sucessful.
214 */
215 smp_call_function_single(counter->cpu,
216 __perf_counter_remove_from_context,
217 counter, 1);
218 return;
219 }
220
221 retry:
222 task_oncpu_function_call(task, __perf_counter_remove_from_context,
223 counter);
224
225 spin_lock_irq(&ctx->lock);
226 /*
227 * If the context is active we need to retry the smp call.
228 */
229 if (ctx->nr_active && !list_empty(&counter->list_entry)) {
230 spin_unlock_irq(&ctx->lock);
231 goto retry;
232 }
233
234 /*
235 * The lock prevents that this context is scheduled in so we
236 * can remove the counter safely, if the call above did not
237 * succeed.
238 */
239 if (!list_empty(&counter->list_entry)) {
240 ctx->nr_counters--;
241 list_del_counter(counter, ctx);
242 counter->task = NULL;
243 }
244 spin_unlock_irq(&ctx->lock);
245 }
246
247 /*
248 * Cross CPU call to disable a performance counter
249 */
250 static void __perf_counter_disable(void *info)
251 {
252 struct perf_counter *counter = info;
253 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
254 struct perf_counter_context *ctx = counter->ctx;
255 unsigned long flags;
256
257 /*
258 * If this is a per-task counter, need to check whether this
259 * counter's task is the current task on this cpu.
260 */
261 if (ctx->task && cpuctx->task_ctx != ctx)
262 return;
263
264 curr_rq_lock_irq_save(&flags);
265 spin_lock(&ctx->lock);
266
267 /*
268 * If the counter is on, turn it off.
269 * If it is in error state, leave it in error state.
270 */
271 if (counter->state >= PERF_COUNTER_STATE_INACTIVE) {
272 if (counter == counter->group_leader)
273 group_sched_out(counter, cpuctx, ctx);
274 else
275 counter_sched_out(counter, cpuctx, ctx);
276 counter->state = PERF_COUNTER_STATE_OFF;
277 }
278
279 spin_unlock(&ctx->lock);
280 curr_rq_unlock_irq_restore(&flags);
281 }
282
283 /*
284 * Disable a counter.
285 */
286 static void perf_counter_disable(struct perf_counter *counter)
287 {
288 struct perf_counter_context *ctx = counter->ctx;
289 struct task_struct *task = ctx->task;
290
291 if (!task) {
292 /*
293 * Disable the counter on the cpu that it's on
294 */
295 smp_call_function_single(counter->cpu, __perf_counter_disable,
296 counter, 1);
297 return;
298 }
299
300 retry:
301 task_oncpu_function_call(task, __perf_counter_disable, counter);
302
303 spin_lock_irq(&ctx->lock);
304 /*
305 * If the counter is still active, we need to retry the cross-call.
306 */
307 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
308 spin_unlock_irq(&ctx->lock);
309 goto retry;
310 }
311
312 /*
313 * Since we have the lock this context can't be scheduled
314 * in, so we can change the state safely.
315 */
316 if (counter->state == PERF_COUNTER_STATE_INACTIVE)
317 counter->state = PERF_COUNTER_STATE_OFF;
318
319 spin_unlock_irq(&ctx->lock);
320 }
321
322 /*
323 * Disable a counter and all its children.
324 */
325 static void perf_counter_disable_family(struct perf_counter *counter)
326 {
327 struct perf_counter *child;
328
329 perf_counter_disable(counter);
330
331 /*
332 * Lock the mutex to protect the list of children
333 */
334 mutex_lock(&counter->mutex);
335 list_for_each_entry(child, &counter->child_list, child_list)
336 perf_counter_disable(child);
337 mutex_unlock(&counter->mutex);
338 }
339
340 static int
341 counter_sched_in(struct perf_counter *counter,
342 struct perf_cpu_context *cpuctx,
343 struct perf_counter_context *ctx,
344 int cpu)
345 {
346 if (counter->state <= PERF_COUNTER_STATE_OFF)
347 return 0;
348
349 counter->state = PERF_COUNTER_STATE_ACTIVE;
350 counter->oncpu = cpu; /* TODO: put 'cpu' into cpuctx->cpu */
351 /*
352 * The new state must be visible before we turn it on in the hardware:
353 */
354 smp_wmb();
355
356 if (counter->hw_ops->enable(counter)) {
357 counter->state = PERF_COUNTER_STATE_INACTIVE;
358 counter->oncpu = -1;
359 return -EAGAIN;
360 }
361
362 if (!is_software_counter(counter))
363 cpuctx->active_oncpu++;
364 ctx->nr_active++;
365
366 if (counter->hw_event.exclusive)
367 cpuctx->exclusive = 1;
368
369 return 0;
370 }
371
372 /*
373 * Return 1 for a group consisting entirely of software counters,
374 * 0 if the group contains any hardware counters.
375 */
376 static int is_software_only_group(struct perf_counter *leader)
377 {
378 struct perf_counter *counter;
379
380 if (!is_software_counter(leader))
381 return 0;
382 list_for_each_entry(counter, &leader->sibling_list, list_entry)
383 if (!is_software_counter(counter))
384 return 0;
385 return 1;
386 }
387
388 /*
389 * Work out whether we can put this counter group on the CPU now.
390 */
391 static int group_can_go_on(struct perf_counter *counter,
392 struct perf_cpu_context *cpuctx,
393 int can_add_hw)
394 {
395 /*
396 * Groups consisting entirely of software counters can always go on.
397 */
398 if (is_software_only_group(counter))
399 return 1;
400 /*
401 * If an exclusive group is already on, no other hardware
402 * counters can go on.
403 */
404 if (cpuctx->exclusive)
405 return 0;
406 /*
407 * If this group is exclusive and there are already
408 * counters on the CPU, it can't go on.
409 */
410 if (counter->hw_event.exclusive && cpuctx->active_oncpu)
411 return 0;
412 /*
413 * Otherwise, try to add it if all previous groups were able
414 * to go on.
415 */
416 return can_add_hw;
417 }
418
419 /*
420 * Cross CPU call to install and enable a performance counter
421 */
422 static void __perf_install_in_context(void *info)
423 {
424 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
425 struct perf_counter *counter = info;
426 struct perf_counter_context *ctx = counter->ctx;
427 struct perf_counter *leader = counter->group_leader;
428 int cpu = smp_processor_id();
429 unsigned long flags;
430 u64 perf_flags;
431 int err;
432
433 /*
434 * If this is a task context, we need to check whether it is
435 * the current task context of this cpu. If not it has been
436 * scheduled out before the smp call arrived.
437 */
438 if (ctx->task && cpuctx->task_ctx != ctx)
439 return;
440
441 curr_rq_lock_irq_save(&flags);
442 spin_lock(&ctx->lock);
443
444 /*
445 * Protect the list operation against NMI by disabling the
446 * counters on a global level. NOP for non NMI based counters.
447 */
448 perf_flags = hw_perf_save_disable();
449
450 list_add_counter(counter, ctx);
451 ctx->nr_counters++;
452 counter->prev_state = PERF_COUNTER_STATE_OFF;
453
454 /*
455 * Don't put the counter on if it is disabled or if
456 * it is in a group and the group isn't on.
457 */
458 if (counter->state != PERF_COUNTER_STATE_INACTIVE ||
459 (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE))
460 goto unlock;
461
462 /*
463 * An exclusive counter can't go on if there are already active
464 * hardware counters, and no hardware counter can go on if there
465 * is already an exclusive counter on.
466 */
467 if (!group_can_go_on(counter, cpuctx, 1))
468 err = -EEXIST;
469 else
470 err = counter_sched_in(counter, cpuctx, ctx, cpu);
471
472 if (err) {
473 /*
474 * This counter couldn't go on. If it is in a group
475 * then we have to pull the whole group off.
476 * If the counter group is pinned then put it in error state.
477 */
478 if (leader != counter)
479 group_sched_out(leader, cpuctx, ctx);
480 if (leader->hw_event.pinned)
481 leader->state = PERF_COUNTER_STATE_ERROR;
482 }
483
484 if (!err && !ctx->task && cpuctx->max_pertask)
485 cpuctx->max_pertask--;
486
487 unlock:
488 hw_perf_restore(perf_flags);
489
490 spin_unlock(&ctx->lock);
491 curr_rq_unlock_irq_restore(&flags);
492 }
493
494 /*
495 * Attach a performance counter to a context
496 *
497 * First we add the counter to the list with the hardware enable bit
498 * in counter->hw_config cleared.
499 *
500 * If the counter is attached to a task which is on a CPU we use a smp
501 * call to enable it in the task context. The task might have been
502 * scheduled away, but we check this in the smp call again.
503 *
504 * Must be called with ctx->mutex held.
505 */
506 static void
507 perf_install_in_context(struct perf_counter_context *ctx,
508 struct perf_counter *counter,
509 int cpu)
510 {
511 struct task_struct *task = ctx->task;
512
513 if (!task) {
514 /*
515 * Per cpu counters are installed via an smp call and
516 * the install is always sucessful.
517 */
518 smp_call_function_single(cpu, __perf_install_in_context,
519 counter, 1);
520 return;
521 }
522
523 counter->task = task;
524 retry:
525 task_oncpu_function_call(task, __perf_install_in_context,
526 counter);
527
528 spin_lock_irq(&ctx->lock);
529 /*
530 * we need to retry the smp call.
531 */
532 if (ctx->is_active && list_empty(&counter->list_entry)) {
533 spin_unlock_irq(&ctx->lock);
534 goto retry;
535 }
536
537 /*
538 * The lock prevents that this context is scheduled in so we
539 * can add the counter safely, if it the call above did not
540 * succeed.
541 */
542 if (list_empty(&counter->list_entry)) {
543 list_add_counter(counter, ctx);
544 ctx->nr_counters++;
545 }
546 spin_unlock_irq(&ctx->lock);
547 }
548
549 /*
550 * Cross CPU call to enable a performance counter
551 */
552 static void __perf_counter_enable(void *info)
553 {
554 struct perf_counter *counter = info;
555 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
556 struct perf_counter_context *ctx = counter->ctx;
557 struct perf_counter *leader = counter->group_leader;
558 unsigned long flags;
559 int err;
560
561 /*
562 * If this is a per-task counter, need to check whether this
563 * counter's task is the current task on this cpu.
564 */
565 if (ctx->task && cpuctx->task_ctx != ctx)
566 return;
567
568 curr_rq_lock_irq_save(&flags);
569 spin_lock(&ctx->lock);
570
571 counter->prev_state = counter->state;
572 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
573 goto unlock;
574 counter->state = PERF_COUNTER_STATE_INACTIVE;
575
576 /*
577 * If the counter is in a group and isn't the group leader,
578 * then don't put it on unless the group is on.
579 */
580 if (leader != counter && leader->state != PERF_COUNTER_STATE_ACTIVE)
581 goto unlock;
582
583 if (!group_can_go_on(counter, cpuctx, 1))
584 err = -EEXIST;
585 else
586 err = counter_sched_in(counter, cpuctx, ctx,
587 smp_processor_id());
588
589 if (err) {
590 /*
591 * If this counter can't go on and it's part of a
592 * group, then the whole group has to come off.
593 */
594 if (leader != counter)
595 group_sched_out(leader, cpuctx, ctx);
596 if (leader->hw_event.pinned)
597 leader->state = PERF_COUNTER_STATE_ERROR;
598 }
599
600 unlock:
601 spin_unlock(&ctx->lock);
602 curr_rq_unlock_irq_restore(&flags);
603 }
604
605 /*
606 * Enable a counter.
607 */
608 static void perf_counter_enable(struct perf_counter *counter)
609 {
610 struct perf_counter_context *ctx = counter->ctx;
611 struct task_struct *task = ctx->task;
612
613 if (!task) {
614 /*
615 * Enable the counter on the cpu that it's on
616 */
617 smp_call_function_single(counter->cpu, __perf_counter_enable,
618 counter, 1);
619 return;
620 }
621
622 spin_lock_irq(&ctx->lock);
623 if (counter->state >= PERF_COUNTER_STATE_INACTIVE)
624 goto out;
625
626 /*
627 * If the counter is in error state, clear that first.
628 * That way, if we see the counter in error state below, we
629 * know that it has gone back into error state, as distinct
630 * from the task having been scheduled away before the
631 * cross-call arrived.
632 */
633 if (counter->state == PERF_COUNTER_STATE_ERROR)
634 counter->state = PERF_COUNTER_STATE_OFF;
635
636 retry:
637 spin_unlock_irq(&ctx->lock);
638 task_oncpu_function_call(task, __perf_counter_enable, counter);
639
640 spin_lock_irq(&ctx->lock);
641
642 /*
643 * If the context is active and the counter is still off,
644 * we need to retry the cross-call.
645 */
646 if (ctx->is_active && counter->state == PERF_COUNTER_STATE_OFF)
647 goto retry;
648
649 /*
650 * Since we have the lock this context can't be scheduled
651 * in, so we can change the state safely.
652 */
653 if (counter->state == PERF_COUNTER_STATE_OFF)
654 counter->state = PERF_COUNTER_STATE_INACTIVE;
655 out:
656 spin_unlock_irq(&ctx->lock);
657 }
658
659 /*
660 * Enable a counter and all its children.
661 */
662 static void perf_counter_enable_family(struct perf_counter *counter)
663 {
664 struct perf_counter *child;
665
666 perf_counter_enable(counter);
667
668 /*
669 * Lock the mutex to protect the list of children
670 */
671 mutex_lock(&counter->mutex);
672 list_for_each_entry(child, &counter->child_list, child_list)
673 perf_counter_enable(child);
674 mutex_unlock(&counter->mutex);
675 }
676
677 void __perf_counter_sched_out(struct perf_counter_context *ctx,
678 struct perf_cpu_context *cpuctx)
679 {
680 struct perf_counter *counter;
681 u64 flags;
682
683 spin_lock(&ctx->lock);
684 ctx->is_active = 0;
685 if (likely(!ctx->nr_counters))
686 goto out;
687
688 flags = hw_perf_save_disable();
689 if (ctx->nr_active) {
690 list_for_each_entry(counter, &ctx->counter_list, list_entry)
691 group_sched_out(counter, cpuctx, ctx);
692 }
693 hw_perf_restore(flags);
694 out:
695 spin_unlock(&ctx->lock);
696 }
697
698 /*
699 * Called from scheduler to remove the counters of the current task,
700 * with interrupts disabled.
701 *
702 * We stop each counter and update the counter value in counter->count.
703 *
704 * This does not protect us against NMI, but disable()
705 * sets the disabled bit in the control field of counter _before_
706 * accessing the counter control register. If a NMI hits, then it will
707 * not restart the counter.
708 */
709 void perf_counter_task_sched_out(struct task_struct *task, int cpu)
710 {
711 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
712 struct perf_counter_context *ctx = &task->perf_counter_ctx;
713 struct pt_regs *regs;
714
715 if (likely(!cpuctx->task_ctx))
716 return;
717
718 regs = task_pt_regs(task);
719 perf_swcounter_event(PERF_COUNT_CONTEXT_SWITCHES, 1, 1, regs);
720 __perf_counter_sched_out(ctx, cpuctx);
721
722 cpuctx->task_ctx = NULL;
723 }
724
725 static void perf_counter_cpu_sched_out(struct perf_cpu_context *cpuctx)
726 {
727 __perf_counter_sched_out(&cpuctx->ctx, cpuctx);
728 }
729
730 static int
731 group_sched_in(struct perf_counter *group_counter,
732 struct perf_cpu_context *cpuctx,
733 struct perf_counter_context *ctx,
734 int cpu)
735 {
736 struct perf_counter *counter, *partial_group;
737 int ret;
738
739 if (group_counter->state == PERF_COUNTER_STATE_OFF)
740 return 0;
741
742 ret = hw_perf_group_sched_in(group_counter, cpuctx, ctx, cpu);
743 if (ret)
744 return ret < 0 ? ret : 0;
745
746 group_counter->prev_state = group_counter->state;
747 if (counter_sched_in(group_counter, cpuctx, ctx, cpu))
748 return -EAGAIN;
749
750 /*
751 * Schedule in siblings as one group (if any):
752 */
753 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
754 counter->prev_state = counter->state;
755 if (counter_sched_in(counter, cpuctx, ctx, cpu)) {
756 partial_group = counter;
757 goto group_error;
758 }
759 }
760
761 return 0;
762
763 group_error:
764 /*
765 * Groups can be scheduled in as one unit only, so undo any
766 * partial group before returning:
767 */
768 list_for_each_entry(counter, &group_counter->sibling_list, list_entry) {
769 if (counter == partial_group)
770 break;
771 counter_sched_out(counter, cpuctx, ctx);
772 }
773 counter_sched_out(group_counter, cpuctx, ctx);
774
775 return -EAGAIN;
776 }
777
778 static void
779 __perf_counter_sched_in(struct perf_counter_context *ctx,
780 struct perf_cpu_context *cpuctx, int cpu)
781 {
782 struct perf_counter *counter;
783 u64 flags;
784 int can_add_hw = 1;
785
786 spin_lock(&ctx->lock);
787 ctx->is_active = 1;
788 if (likely(!ctx->nr_counters))
789 goto out;
790
791 flags = hw_perf_save_disable();
792
793 /*
794 * First go through the list and put on any pinned groups
795 * in order to give them the best chance of going on.
796 */
797 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
798 if (counter->state <= PERF_COUNTER_STATE_OFF ||
799 !counter->hw_event.pinned)
800 continue;
801 if (counter->cpu != -1 && counter->cpu != cpu)
802 continue;
803
804 if (group_can_go_on(counter, cpuctx, 1))
805 group_sched_in(counter, cpuctx, ctx, cpu);
806
807 /*
808 * If this pinned group hasn't been scheduled,
809 * put it in error state.
810 */
811 if (counter->state == PERF_COUNTER_STATE_INACTIVE)
812 counter->state = PERF_COUNTER_STATE_ERROR;
813 }
814
815 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
816 /*
817 * Ignore counters in OFF or ERROR state, and
818 * ignore pinned counters since we did them already.
819 */
820 if (counter->state <= PERF_COUNTER_STATE_OFF ||
821 counter->hw_event.pinned)
822 continue;
823
824 /*
825 * Listen to the 'cpu' scheduling filter constraint
826 * of counters:
827 */
828 if (counter->cpu != -1 && counter->cpu != cpu)
829 continue;
830
831 if (group_can_go_on(counter, cpuctx, can_add_hw)) {
832 if (group_sched_in(counter, cpuctx, ctx, cpu))
833 can_add_hw = 0;
834 }
835 }
836 hw_perf_restore(flags);
837 out:
838 spin_unlock(&ctx->lock);
839 }
840
841 /*
842 * Called from scheduler to add the counters of the current task
843 * with interrupts disabled.
844 *
845 * We restore the counter value and then enable it.
846 *
847 * This does not protect us against NMI, but enable()
848 * sets the enabled bit in the control field of counter _before_
849 * accessing the counter control register. If a NMI hits, then it will
850 * keep the counter running.
851 */
852 void perf_counter_task_sched_in(struct task_struct *task, int cpu)
853 {
854 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
855 struct perf_counter_context *ctx = &task->perf_counter_ctx;
856
857 __perf_counter_sched_in(ctx, cpuctx, cpu);
858 cpuctx->task_ctx = ctx;
859 }
860
861 static void perf_counter_cpu_sched_in(struct perf_cpu_context *cpuctx, int cpu)
862 {
863 struct perf_counter_context *ctx = &cpuctx->ctx;
864
865 __perf_counter_sched_in(ctx, cpuctx, cpu);
866 }
867
868 int perf_counter_task_disable(void)
869 {
870 struct task_struct *curr = current;
871 struct perf_counter_context *ctx = &curr->perf_counter_ctx;
872 struct perf_counter *counter;
873 unsigned long flags;
874 u64 perf_flags;
875 int cpu;
876
877 if (likely(!ctx->nr_counters))
878 return 0;
879
880 curr_rq_lock_irq_save(&flags);
881 cpu = smp_processor_id();
882
883 /* force the update of the task clock: */
884 __task_delta_exec(curr, 1);
885
886 perf_counter_task_sched_out(curr, cpu);
887
888 spin_lock(&ctx->lock);
889
890 /*
891 * Disable all the counters:
892 */
893 perf_flags = hw_perf_save_disable();
894
895 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
896 if (counter->state != PERF_COUNTER_STATE_ERROR)
897 counter->state = PERF_COUNTER_STATE_OFF;
898 }
899
900 hw_perf_restore(perf_flags);
901
902 spin_unlock(&ctx->lock);
903
904 curr_rq_unlock_irq_restore(&flags);
905
906 return 0;
907 }
908
909 int perf_counter_task_enable(void)
910 {
911 struct task_struct *curr = current;
912 struct perf_counter_context *ctx = &curr->perf_counter_ctx;
913 struct perf_counter *counter;
914 unsigned long flags;
915 u64 perf_flags;
916 int cpu;
917
918 if (likely(!ctx->nr_counters))
919 return 0;
920
921 curr_rq_lock_irq_save(&flags);
922 cpu = smp_processor_id();
923
924 /* force the update of the task clock: */
925 __task_delta_exec(curr, 1);
926
927 perf_counter_task_sched_out(curr, cpu);
928
929 spin_lock(&ctx->lock);
930
931 /*
932 * Disable all the counters:
933 */
934 perf_flags = hw_perf_save_disable();
935
936 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
937 if (counter->state > PERF_COUNTER_STATE_OFF)
938 continue;
939 counter->state = PERF_COUNTER_STATE_INACTIVE;
940 counter->hw_event.disabled = 0;
941 }
942 hw_perf_restore(perf_flags);
943
944 spin_unlock(&ctx->lock);
945
946 perf_counter_task_sched_in(curr, cpu);
947
948 curr_rq_unlock_irq_restore(&flags);
949
950 return 0;
951 }
952
953 /*
954 * Round-robin a context's counters:
955 */
956 static void rotate_ctx(struct perf_counter_context *ctx)
957 {
958 struct perf_counter *counter;
959 u64 perf_flags;
960
961 if (!ctx->nr_counters)
962 return;
963
964 spin_lock(&ctx->lock);
965 /*
966 * Rotate the first entry last (works just fine for group counters too):
967 */
968 perf_flags = hw_perf_save_disable();
969 list_for_each_entry(counter, &ctx->counter_list, list_entry) {
970 list_move_tail(&counter->list_entry, &ctx->counter_list);
971 break;
972 }
973 hw_perf_restore(perf_flags);
974
975 spin_unlock(&ctx->lock);
976 }
977
978 void perf_counter_task_tick(struct task_struct *curr, int cpu)
979 {
980 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
981 struct perf_counter_context *ctx = &curr->perf_counter_ctx;
982 const int rotate_percpu = 0;
983
984 if (rotate_percpu)
985 perf_counter_cpu_sched_out(cpuctx);
986 perf_counter_task_sched_out(curr, cpu);
987
988 if (rotate_percpu)
989 rotate_ctx(&cpuctx->ctx);
990 rotate_ctx(ctx);
991
992 if (rotate_percpu)
993 perf_counter_cpu_sched_in(cpuctx, cpu);
994 perf_counter_task_sched_in(curr, cpu);
995 }
996
997 /*
998 * Cross CPU call to read the hardware counter
999 */
1000 static void __read(void *info)
1001 {
1002 struct perf_counter *counter = info;
1003 unsigned long flags;
1004
1005 curr_rq_lock_irq_save(&flags);
1006 counter->hw_ops->read(counter);
1007 curr_rq_unlock_irq_restore(&flags);
1008 }
1009
1010 static u64 perf_counter_read(struct perf_counter *counter)
1011 {
1012 /*
1013 * If counter is enabled and currently active on a CPU, update the
1014 * value in the counter structure:
1015 */
1016 if (counter->state == PERF_COUNTER_STATE_ACTIVE) {
1017 smp_call_function_single(counter->oncpu,
1018 __read, counter, 1);
1019 }
1020
1021 return atomic64_read(&counter->count);
1022 }
1023
1024 /*
1025 * Cross CPU call to switch performance data pointers
1026 */
1027 static void __perf_switch_irq_data(void *info)
1028 {
1029 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
1030 struct perf_counter *counter = info;
1031 struct perf_counter_context *ctx = counter->ctx;
1032 struct perf_data *oldirqdata = counter->irqdata;
1033
1034 /*
1035 * If this is a task context, we need to check whether it is
1036 * the current task context of this cpu. If not it has been
1037 * scheduled out before the smp call arrived.
1038 */
1039 if (ctx->task) {
1040 if (cpuctx->task_ctx != ctx)
1041 return;
1042 spin_lock(&ctx->lock);
1043 }
1044
1045 /* Change the pointer NMI safe */
1046 atomic_long_set((atomic_long_t *)&counter->irqdata,
1047 (unsigned long) counter->usrdata);
1048 counter->usrdata = oldirqdata;
1049
1050 if (ctx->task)
1051 spin_unlock(&ctx->lock);
1052 }
1053
1054 static struct perf_data *perf_switch_irq_data(struct perf_counter *counter)
1055 {
1056 struct perf_counter_context *ctx = counter->ctx;
1057 struct perf_data *oldirqdata = counter->irqdata;
1058 struct task_struct *task = ctx->task;
1059
1060 if (!task) {
1061 smp_call_function_single(counter->cpu,
1062 __perf_switch_irq_data,
1063 counter, 1);
1064 return counter->usrdata;
1065 }
1066
1067 retry:
1068 spin_lock_irq(&ctx->lock);
1069 if (counter->state != PERF_COUNTER_STATE_ACTIVE) {
1070 counter->irqdata = counter->usrdata;
1071 counter->usrdata = oldirqdata;
1072 spin_unlock_irq(&ctx->lock);
1073 return oldirqdata;
1074 }
1075 spin_unlock_irq(&ctx->lock);
1076 task_oncpu_function_call(task, __perf_switch_irq_data, counter);
1077 /* Might have failed, because task was scheduled out */
1078 if (counter->irqdata == oldirqdata)
1079 goto retry;
1080
1081 return counter->usrdata;
1082 }
1083
1084 static void put_context(struct perf_counter_context *ctx)
1085 {
1086 if (ctx->task)
1087 put_task_struct(ctx->task);
1088 }
1089
1090 static struct perf_counter_context *find_get_context(pid_t pid, int cpu)
1091 {
1092 struct perf_cpu_context *cpuctx;
1093 struct perf_counter_context *ctx;
1094 struct task_struct *task;
1095
1096 /*
1097 * If cpu is not a wildcard then this is a percpu counter:
1098 */
1099 if (cpu != -1) {
1100 /* Must be root to operate on a CPU counter: */
1101 if (!capable(CAP_SYS_ADMIN))
1102 return ERR_PTR(-EACCES);
1103
1104 if (cpu < 0 || cpu > num_possible_cpus())
1105 return ERR_PTR(-EINVAL);
1106
1107 /*
1108 * We could be clever and allow to attach a counter to an
1109 * offline CPU and activate it when the CPU comes up, but
1110 * that's for later.
1111 */
1112 if (!cpu_isset(cpu, cpu_online_map))
1113 return ERR_PTR(-ENODEV);
1114
1115 cpuctx = &per_cpu(perf_cpu_context, cpu);
1116 ctx = &cpuctx->ctx;
1117
1118 return ctx;
1119 }
1120
1121 rcu_read_lock();
1122 if (!pid)
1123 task = current;
1124 else
1125 task = find_task_by_vpid(pid);
1126 if (task)
1127 get_task_struct(task);
1128 rcu_read_unlock();
1129
1130 if (!task)
1131 return ERR_PTR(-ESRCH);
1132
1133 ctx = &task->perf_counter_ctx;
1134 ctx->task = task;
1135
1136 /* Reuse ptrace permission checks for now. */
1137 if (!ptrace_may_access(task, PTRACE_MODE_READ)) {
1138 put_context(ctx);
1139 return ERR_PTR(-EACCES);
1140 }
1141
1142 return ctx;
1143 }
1144
1145 static void free_counter_rcu(struct rcu_head *head)
1146 {
1147 struct perf_counter *counter;
1148
1149 counter = container_of(head, struct perf_counter, rcu_head);
1150 kfree(counter);
1151 }
1152
1153 static void free_counter(struct perf_counter *counter)
1154 {
1155 if (counter->destroy)
1156 counter->destroy(counter);
1157
1158 call_rcu(&counter->rcu_head, free_counter_rcu);
1159 }
1160
1161 /*
1162 * Called when the last reference to the file is gone.
1163 */
1164 static int perf_release(struct inode *inode, struct file *file)
1165 {
1166 struct perf_counter *counter = file->private_data;
1167 struct perf_counter_context *ctx = counter->ctx;
1168
1169 file->private_data = NULL;
1170
1171 mutex_lock(&ctx->mutex);
1172 mutex_lock(&counter->mutex);
1173
1174 perf_counter_remove_from_context(counter);
1175
1176 mutex_unlock(&counter->mutex);
1177 mutex_unlock(&ctx->mutex);
1178
1179 free_counter(counter);
1180 put_context(ctx);
1181
1182 return 0;
1183 }
1184
1185 /*
1186 * Read the performance counter - simple non blocking version for now
1187 */
1188 static ssize_t
1189 perf_read_hw(struct perf_counter *counter, char __user *buf, size_t count)
1190 {
1191 u64 cntval;
1192
1193 if (count != sizeof(cntval))
1194 return -EINVAL;
1195
1196 /*
1197 * Return end-of-file for a read on a counter that is in
1198 * error state (i.e. because it was pinned but it couldn't be
1199 * scheduled on to the CPU at some point).
1200 */
1201 if (counter->state == PERF_COUNTER_STATE_ERROR)
1202 return 0;
1203
1204 mutex_lock(&counter->mutex);
1205 cntval = perf_counter_read(counter);
1206 mutex_unlock(&counter->mutex);
1207
1208 return put_user(cntval, (u64 __user *) buf) ? -EFAULT : sizeof(cntval);
1209 }
1210
1211 static ssize_t
1212 perf_copy_usrdata(struct perf_data *usrdata, char __user *buf, size_t count)
1213 {
1214 if (!usrdata->len)
1215 return 0;
1216
1217 count = min(count, (size_t)usrdata->len);
1218 if (copy_to_user(buf, usrdata->data + usrdata->rd_idx, count))
1219 return -EFAULT;
1220
1221 /* Adjust the counters */
1222 usrdata->len -= count;
1223 if (!usrdata->len)
1224 usrdata->rd_idx = 0;
1225 else
1226 usrdata->rd_idx += count;
1227
1228 return count;
1229 }
1230
1231 static ssize_t
1232 perf_read_irq_data(struct perf_counter *counter,
1233 char __user *buf,
1234 size_t count,
1235 int nonblocking)
1236 {
1237 struct perf_data *irqdata, *usrdata;
1238 DECLARE_WAITQUEUE(wait, current);
1239 ssize_t res, res2;
1240
1241 irqdata = counter->irqdata;
1242 usrdata = counter->usrdata;
1243
1244 if (usrdata->len + irqdata->len >= count)
1245 goto read_pending;
1246
1247 if (nonblocking)
1248 return -EAGAIN;
1249
1250 spin_lock_irq(&counter->waitq.lock);
1251 __add_wait_queue(&counter->waitq, &wait);
1252 for (;;) {
1253 set_current_state(TASK_INTERRUPTIBLE);
1254 if (usrdata->len + irqdata->len >= count)
1255 break;
1256
1257 if (signal_pending(current))
1258 break;
1259
1260 if (counter->state == PERF_COUNTER_STATE_ERROR)
1261 break;
1262
1263 spin_unlock_irq(&counter->waitq.lock);
1264 schedule();
1265 spin_lock_irq(&counter->waitq.lock);
1266 }
1267 __remove_wait_queue(&counter->waitq, &wait);
1268 __set_current_state(TASK_RUNNING);
1269 spin_unlock_irq(&counter->waitq.lock);
1270
1271 if (usrdata->len + irqdata->len < count &&
1272 counter->state != PERF_COUNTER_STATE_ERROR)
1273 return -ERESTARTSYS;
1274 read_pending:
1275 mutex_lock(&counter->mutex);
1276
1277 /* Drain pending data first: */
1278 res = perf_copy_usrdata(usrdata, buf, count);
1279 if (res < 0 || res == count)
1280 goto out;
1281
1282 /* Switch irq buffer: */
1283 usrdata = perf_switch_irq_data(counter);
1284 res2 = perf_copy_usrdata(usrdata, buf + res, count - res);
1285 if (res2 < 0) {
1286 if (!res)
1287 res = -EFAULT;
1288 } else {
1289 res += res2;
1290 }
1291 out:
1292 mutex_unlock(&counter->mutex);
1293
1294 return res;
1295 }
1296
1297 static ssize_t
1298 perf_read(struct file *file, char __user *buf, size_t count, loff_t *ppos)
1299 {
1300 struct perf_counter *counter = file->private_data;
1301
1302 switch (counter->hw_event.record_type) {
1303 case PERF_RECORD_SIMPLE:
1304 return perf_read_hw(counter, buf, count);
1305
1306 case PERF_RECORD_IRQ:
1307 case PERF_RECORD_GROUP:
1308 return perf_read_irq_data(counter, buf, count,
1309 file->f_flags & O_NONBLOCK);
1310 }
1311 return -EINVAL;
1312 }
1313
1314 static unsigned int perf_poll(struct file *file, poll_table *wait)
1315 {
1316 struct perf_counter *counter = file->private_data;
1317 unsigned int events = 0;
1318 unsigned long flags;
1319
1320 poll_wait(file, &counter->waitq, wait);
1321
1322 spin_lock_irqsave(&counter->waitq.lock, flags);
1323 if (counter->usrdata->len || counter->irqdata->len)
1324 events |= POLLIN;
1325 spin_unlock_irqrestore(&counter->waitq.lock, flags);
1326
1327 return events;
1328 }
1329
1330 static long perf_ioctl(struct file *file, unsigned int cmd, unsigned long arg)
1331 {
1332 struct perf_counter *counter = file->private_data;
1333 int err = 0;
1334
1335 switch (cmd) {
1336 case PERF_COUNTER_IOC_ENABLE:
1337 perf_counter_enable_family(counter);
1338 break;
1339 case PERF_COUNTER_IOC_DISABLE:
1340 perf_counter_disable_family(counter);
1341 break;
1342 default:
1343 err = -ENOTTY;
1344 }
1345 return err;
1346 }
1347
1348 static const struct file_operations perf_fops = {
1349 .release = perf_release,
1350 .read = perf_read,
1351 .poll = perf_poll,
1352 .unlocked_ioctl = perf_ioctl,
1353 .compat_ioctl = perf_ioctl,
1354 };
1355
1356 /*
1357 * Output
1358 */
1359
1360 static void perf_counter_store_irq(struct perf_counter *counter, u64 data)
1361 {
1362 struct perf_data *irqdata = counter->irqdata;
1363
1364 if (irqdata->len > PERF_DATA_BUFLEN - sizeof(u64)) {
1365 irqdata->overrun++;
1366 } else {
1367 u64 *p = (u64 *) &irqdata->data[irqdata->len];
1368
1369 *p = data;
1370 irqdata->len += sizeof(u64);
1371 }
1372 }
1373
1374 static void perf_counter_handle_group(struct perf_counter *counter)
1375 {
1376 struct perf_counter *leader, *sub;
1377
1378 leader = counter->group_leader;
1379 list_for_each_entry(sub, &leader->sibling_list, list_entry) {
1380 if (sub != counter)
1381 sub->hw_ops->read(sub);
1382 perf_counter_store_irq(counter, sub->hw_event.event_config);
1383 perf_counter_store_irq(counter, atomic64_read(&sub->count));
1384 }
1385 }
1386
1387 void perf_counter_output(struct perf_counter *counter,
1388 int nmi, struct pt_regs *regs)
1389 {
1390 switch (counter->hw_event.record_type) {
1391 case PERF_RECORD_SIMPLE:
1392 return;
1393
1394 case PERF_RECORD_IRQ:
1395 perf_counter_store_irq(counter, instruction_pointer(regs));
1396 break;
1397
1398 case PERF_RECORD_GROUP:
1399 perf_counter_handle_group(counter);
1400 break;
1401 }
1402
1403 if (nmi) {
1404 counter->wakeup_pending = 1;
1405 set_perf_counter_pending();
1406 } else
1407 wake_up(&counter->waitq);
1408 }
1409
1410 /*
1411 * Generic software counter infrastructure
1412 */
1413
1414 static void perf_swcounter_update(struct perf_counter *counter)
1415 {
1416 struct hw_perf_counter *hwc = &counter->hw;
1417 u64 prev, now;
1418 s64 delta;
1419
1420 again:
1421 prev = atomic64_read(&hwc->prev_count);
1422 now = atomic64_read(&hwc->count);
1423 if (atomic64_cmpxchg(&hwc->prev_count, prev, now) != prev)
1424 goto again;
1425
1426 delta = now - prev;
1427
1428 atomic64_add(delta, &counter->count);
1429 atomic64_sub(delta, &hwc->period_left);
1430 }
1431
1432 static void perf_swcounter_set_period(struct perf_counter *counter)
1433 {
1434 struct hw_perf_counter *hwc = &counter->hw;
1435 s64 left = atomic64_read(&hwc->period_left);
1436 s64 period = hwc->irq_period;
1437
1438 if (unlikely(left <= -period)) {
1439 left = period;
1440 atomic64_set(&hwc->period_left, left);
1441 }
1442
1443 if (unlikely(left <= 0)) {
1444 left += period;
1445 atomic64_add(period, &hwc->period_left);
1446 }
1447
1448 atomic64_set(&hwc->prev_count, -left);
1449 atomic64_set(&hwc->count, -left);
1450 }
1451
1452 static enum hrtimer_restart perf_swcounter_hrtimer(struct hrtimer *hrtimer)
1453 {
1454 struct perf_counter *counter;
1455 struct pt_regs *regs;
1456
1457 counter = container_of(hrtimer, struct perf_counter, hw.hrtimer);
1458 counter->hw_ops->read(counter);
1459
1460 regs = get_irq_regs();
1461 /*
1462 * In case we exclude kernel IPs or are somehow not in interrupt
1463 * context, provide the next best thing, the user IP.
1464 */
1465 if ((counter->hw_event.exclude_kernel || !regs) &&
1466 !counter->hw_event.exclude_user)
1467 regs = task_pt_regs(current);
1468
1469 if (regs)
1470 perf_counter_output(counter, 0, regs);
1471
1472 hrtimer_forward_now(hrtimer, ns_to_ktime(counter->hw.irq_period));
1473
1474 return HRTIMER_RESTART;
1475 }
1476
1477 static void perf_swcounter_overflow(struct perf_counter *counter,
1478 int nmi, struct pt_regs *regs)
1479 {
1480 perf_swcounter_update(counter);
1481 perf_swcounter_set_period(counter);
1482 perf_counter_output(counter, nmi, regs);
1483 }
1484
1485 static int perf_swcounter_match(struct perf_counter *counter,
1486 enum perf_event_types type,
1487 u32 event, struct pt_regs *regs)
1488 {
1489 if (counter->state != PERF_COUNTER_STATE_ACTIVE)
1490 return 0;
1491
1492 if (counter->hw_event.raw_type)
1493 return 0;
1494
1495 if (counter->hw_event.type != type)
1496 return 0;
1497
1498 if (counter->hw_event.event_id != event)
1499 return 0;
1500
1501 if (counter->hw_event.exclude_user && user_mode(regs))
1502 return 0;
1503
1504 if (counter->hw_event.exclude_kernel && !user_mode(regs))
1505 return 0;
1506
1507 return 1;
1508 }
1509
1510 static void perf_swcounter_add(struct perf_counter *counter, u64 nr,
1511 int nmi, struct pt_regs *regs)
1512 {
1513 int neg = atomic64_add_negative(nr, &counter->hw.count);
1514 if (counter->hw.irq_period && !neg)
1515 perf_swcounter_overflow(counter, nmi, regs);
1516 }
1517
1518 static void perf_swcounter_ctx_event(struct perf_counter_context *ctx,
1519 enum perf_event_types type, u32 event,
1520 u64 nr, int nmi, struct pt_regs *regs)
1521 {
1522 struct perf_counter *counter;
1523
1524 if (system_state != SYSTEM_RUNNING || list_empty(&ctx->event_list))
1525 return;
1526
1527 rcu_read_lock();
1528 list_for_each_entry_rcu(counter, &ctx->event_list, event_entry) {
1529 if (perf_swcounter_match(counter, type, event, regs))
1530 perf_swcounter_add(counter, nr, nmi, regs);
1531 }
1532 rcu_read_unlock();
1533 }
1534
1535 static void __perf_swcounter_event(enum perf_event_types type, u32 event,
1536 u64 nr, int nmi, struct pt_regs *regs)
1537 {
1538 struct perf_cpu_context *cpuctx = &get_cpu_var(perf_cpu_context);
1539
1540 perf_swcounter_ctx_event(&cpuctx->ctx, type, event, nr, nmi, regs);
1541 if (cpuctx->task_ctx) {
1542 perf_swcounter_ctx_event(cpuctx->task_ctx, type, event,
1543 nr, nmi, regs);
1544 }
1545
1546 put_cpu_var(perf_cpu_context);
1547 }
1548
1549 void perf_swcounter_event(u32 event, u64 nr, int nmi, struct pt_regs *regs)
1550 {
1551 __perf_swcounter_event(PERF_TYPE_SOFTWARE, event, nr, nmi, regs);
1552 }
1553
1554 static void perf_swcounter_read(struct perf_counter *counter)
1555 {
1556 perf_swcounter_update(counter);
1557 }
1558
1559 static int perf_swcounter_enable(struct perf_counter *counter)
1560 {
1561 perf_swcounter_set_period(counter);
1562 return 0;
1563 }
1564
1565 static void perf_swcounter_disable(struct perf_counter *counter)
1566 {
1567 perf_swcounter_update(counter);
1568 }
1569
1570 static const struct hw_perf_counter_ops perf_ops_generic = {
1571 .enable = perf_swcounter_enable,
1572 .disable = perf_swcounter_disable,
1573 .read = perf_swcounter_read,
1574 };
1575
1576 /*
1577 * Software counter: cpu wall time clock
1578 */
1579
1580 static void cpu_clock_perf_counter_update(struct perf_counter *counter)
1581 {
1582 int cpu = raw_smp_processor_id();
1583 s64 prev;
1584 u64 now;
1585
1586 now = cpu_clock(cpu);
1587 prev = atomic64_read(&counter->hw.prev_count);
1588 atomic64_set(&counter->hw.prev_count, now);
1589 atomic64_add(now - prev, &counter->count);
1590 }
1591
1592 static int cpu_clock_perf_counter_enable(struct perf_counter *counter)
1593 {
1594 struct hw_perf_counter *hwc = &counter->hw;
1595 int cpu = raw_smp_processor_id();
1596
1597 atomic64_set(&hwc->prev_count, cpu_clock(cpu));
1598 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1599 hwc->hrtimer.function = perf_swcounter_hrtimer;
1600 if (hwc->irq_period) {
1601 __hrtimer_start_range_ns(&hwc->hrtimer,
1602 ns_to_ktime(hwc->irq_period), 0,
1603 HRTIMER_MODE_REL, 0);
1604 }
1605
1606 return 0;
1607 }
1608
1609 static void cpu_clock_perf_counter_disable(struct perf_counter *counter)
1610 {
1611 hrtimer_cancel(&counter->hw.hrtimer);
1612 cpu_clock_perf_counter_update(counter);
1613 }
1614
1615 static void cpu_clock_perf_counter_read(struct perf_counter *counter)
1616 {
1617 cpu_clock_perf_counter_update(counter);
1618 }
1619
1620 static const struct hw_perf_counter_ops perf_ops_cpu_clock = {
1621 .enable = cpu_clock_perf_counter_enable,
1622 .disable = cpu_clock_perf_counter_disable,
1623 .read = cpu_clock_perf_counter_read,
1624 };
1625
1626 /*
1627 * Software counter: task time clock
1628 */
1629
1630 /*
1631 * Called from within the scheduler:
1632 */
1633 static u64 task_clock_perf_counter_val(struct perf_counter *counter, int update)
1634 {
1635 struct task_struct *curr = counter->task;
1636 u64 delta;
1637
1638 delta = __task_delta_exec(curr, update);
1639
1640 return curr->se.sum_exec_runtime + delta;
1641 }
1642
1643 static void task_clock_perf_counter_update(struct perf_counter *counter, u64 now)
1644 {
1645 u64 prev;
1646 s64 delta;
1647
1648 prev = atomic64_read(&counter->hw.prev_count);
1649
1650 atomic64_set(&counter->hw.prev_count, now);
1651
1652 delta = now - prev;
1653
1654 atomic64_add(delta, &counter->count);
1655 }
1656
1657 static int task_clock_perf_counter_enable(struct perf_counter *counter)
1658 {
1659 struct hw_perf_counter *hwc = &counter->hw;
1660
1661 atomic64_set(&hwc->prev_count, task_clock_perf_counter_val(counter, 0));
1662 hrtimer_init(&hwc->hrtimer, CLOCK_MONOTONIC, HRTIMER_MODE_REL);
1663 hwc->hrtimer.function = perf_swcounter_hrtimer;
1664 if (hwc->irq_period) {
1665 __hrtimer_start_range_ns(&hwc->hrtimer,
1666 ns_to_ktime(hwc->irq_period), 0,
1667 HRTIMER_MODE_REL, 0);
1668 }
1669
1670 return 0;
1671 }
1672
1673 static void task_clock_perf_counter_disable(struct perf_counter *counter)
1674 {
1675 hrtimer_cancel(&counter->hw.hrtimer);
1676 task_clock_perf_counter_update(counter,
1677 task_clock_perf_counter_val(counter, 0));
1678 }
1679
1680 static void task_clock_perf_counter_read(struct perf_counter *counter)
1681 {
1682 task_clock_perf_counter_update(counter,
1683 task_clock_perf_counter_val(counter, 1));
1684 }
1685
1686 static const struct hw_perf_counter_ops perf_ops_task_clock = {
1687 .enable = task_clock_perf_counter_enable,
1688 .disable = task_clock_perf_counter_disable,
1689 .read = task_clock_perf_counter_read,
1690 };
1691
1692 /*
1693 * Software counter: cpu migrations
1694 */
1695
1696 static inline u64 get_cpu_migrations(struct perf_counter *counter)
1697 {
1698 struct task_struct *curr = counter->ctx->task;
1699
1700 if (curr)
1701 return curr->se.nr_migrations;
1702 return cpu_nr_migrations(smp_processor_id());
1703 }
1704
1705 static void cpu_migrations_perf_counter_update(struct perf_counter *counter)
1706 {
1707 u64 prev, now;
1708 s64 delta;
1709
1710 prev = atomic64_read(&counter->hw.prev_count);
1711 now = get_cpu_migrations(counter);
1712
1713 atomic64_set(&counter->hw.prev_count, now);
1714
1715 delta = now - prev;
1716
1717 atomic64_add(delta, &counter->count);
1718 }
1719
1720 static void cpu_migrations_perf_counter_read(struct perf_counter *counter)
1721 {
1722 cpu_migrations_perf_counter_update(counter);
1723 }
1724
1725 static int cpu_migrations_perf_counter_enable(struct perf_counter *counter)
1726 {
1727 if (counter->prev_state <= PERF_COUNTER_STATE_OFF)
1728 atomic64_set(&counter->hw.prev_count,
1729 get_cpu_migrations(counter));
1730 return 0;
1731 }
1732
1733 static void cpu_migrations_perf_counter_disable(struct perf_counter *counter)
1734 {
1735 cpu_migrations_perf_counter_update(counter);
1736 }
1737
1738 static const struct hw_perf_counter_ops perf_ops_cpu_migrations = {
1739 .enable = cpu_migrations_perf_counter_enable,
1740 .disable = cpu_migrations_perf_counter_disable,
1741 .read = cpu_migrations_perf_counter_read,
1742 };
1743
1744 #ifdef CONFIG_EVENT_PROFILE
1745 void perf_tpcounter_event(int event_id)
1746 {
1747 struct pt_regs *regs = get_irq_regs();
1748
1749 if (!regs)
1750 regs = task_pt_regs(current);
1751
1752 __perf_swcounter_event(PERF_TYPE_TRACEPOINT, event_id, 1, 1, regs);
1753 }
1754
1755 extern int ftrace_profile_enable(int);
1756 extern void ftrace_profile_disable(int);
1757
1758 static void tp_perf_counter_destroy(struct perf_counter *counter)
1759 {
1760 ftrace_profile_disable(counter->hw_event.event_id);
1761 }
1762
1763 static const struct hw_perf_counter_ops *
1764 tp_perf_counter_init(struct perf_counter *counter)
1765 {
1766 int event_id = counter->hw_event.event_id;
1767 int ret;
1768
1769 ret = ftrace_profile_enable(event_id);
1770 if (ret)
1771 return NULL;
1772
1773 counter->destroy = tp_perf_counter_destroy;
1774 counter->hw.irq_period = counter->hw_event.irq_period;
1775
1776 return &perf_ops_generic;
1777 }
1778 #else
1779 static const struct hw_perf_counter_ops *
1780 tp_perf_counter_init(struct perf_counter *counter)
1781 {
1782 return NULL;
1783 }
1784 #endif
1785
1786 static const struct hw_perf_counter_ops *
1787 sw_perf_counter_init(struct perf_counter *counter)
1788 {
1789 struct perf_counter_hw_event *hw_event = &counter->hw_event;
1790 const struct hw_perf_counter_ops *hw_ops = NULL;
1791 struct hw_perf_counter *hwc = &counter->hw;
1792
1793 /*
1794 * Software counters (currently) can't in general distinguish
1795 * between user, kernel and hypervisor events.
1796 * However, context switches and cpu migrations are considered
1797 * to be kernel events, and page faults are never hypervisor
1798 * events.
1799 */
1800 switch (counter->hw_event.event_id) {
1801 case PERF_COUNT_CPU_CLOCK:
1802 hw_ops = &perf_ops_cpu_clock;
1803
1804 if (hw_event->irq_period && hw_event->irq_period < 10000)
1805 hw_event->irq_period = 10000;
1806 break;
1807 case PERF_COUNT_TASK_CLOCK:
1808 /*
1809 * If the user instantiates this as a per-cpu counter,
1810 * use the cpu_clock counter instead.
1811 */
1812 if (counter->ctx->task)
1813 hw_ops = &perf_ops_task_clock;
1814 else
1815 hw_ops = &perf_ops_cpu_clock;
1816
1817 if (hw_event->irq_period && hw_event->irq_period < 10000)
1818 hw_event->irq_period = 10000;
1819 break;
1820 case PERF_COUNT_PAGE_FAULTS:
1821 case PERF_COUNT_PAGE_FAULTS_MIN:
1822 case PERF_COUNT_PAGE_FAULTS_MAJ:
1823 case PERF_COUNT_CONTEXT_SWITCHES:
1824 hw_ops = &perf_ops_generic;
1825 break;
1826 case PERF_COUNT_CPU_MIGRATIONS:
1827 if (!counter->hw_event.exclude_kernel)
1828 hw_ops = &perf_ops_cpu_migrations;
1829 break;
1830 }
1831
1832 if (hw_ops)
1833 hwc->irq_period = hw_event->irq_period;
1834
1835 return hw_ops;
1836 }
1837
1838 /*
1839 * Allocate and initialize a counter structure
1840 */
1841 static struct perf_counter *
1842 perf_counter_alloc(struct perf_counter_hw_event *hw_event,
1843 int cpu,
1844 struct perf_counter_context *ctx,
1845 struct perf_counter *group_leader,
1846 gfp_t gfpflags)
1847 {
1848 const struct hw_perf_counter_ops *hw_ops;
1849 struct perf_counter *counter;
1850
1851 counter = kzalloc(sizeof(*counter), gfpflags);
1852 if (!counter)
1853 return NULL;
1854
1855 /*
1856 * Single counters are their own group leaders, with an
1857 * empty sibling list:
1858 */
1859 if (!group_leader)
1860 group_leader = counter;
1861
1862 mutex_init(&counter->mutex);
1863 INIT_LIST_HEAD(&counter->list_entry);
1864 INIT_LIST_HEAD(&counter->event_entry);
1865 INIT_LIST_HEAD(&counter->sibling_list);
1866 init_waitqueue_head(&counter->waitq);
1867
1868 INIT_LIST_HEAD(&counter->child_list);
1869
1870 counter->irqdata = &counter->data[0];
1871 counter->usrdata = &counter->data[1];
1872 counter->cpu = cpu;
1873 counter->hw_event = *hw_event;
1874 counter->wakeup_pending = 0;
1875 counter->group_leader = group_leader;
1876 counter->hw_ops = NULL;
1877 counter->ctx = ctx;
1878
1879 counter->state = PERF_COUNTER_STATE_INACTIVE;
1880 if (hw_event->disabled)
1881 counter->state = PERF_COUNTER_STATE_OFF;
1882
1883 hw_ops = NULL;
1884
1885 if (hw_event->raw_type)
1886 hw_ops = hw_perf_counter_init(counter);
1887 else switch (hw_event->type) {
1888 case PERF_TYPE_HARDWARE:
1889 hw_ops = hw_perf_counter_init(counter);
1890 break;
1891
1892 case PERF_TYPE_SOFTWARE:
1893 hw_ops = sw_perf_counter_init(counter);
1894 break;
1895
1896 case PERF_TYPE_TRACEPOINT:
1897 hw_ops = tp_perf_counter_init(counter);
1898 break;
1899 }
1900
1901 if (!hw_ops) {
1902 kfree(counter);
1903 return NULL;
1904 }
1905 counter->hw_ops = hw_ops;
1906
1907 return counter;
1908 }
1909
1910 /**
1911 * sys_perf_counter_open - open a performance counter, associate it to a task/cpu
1912 *
1913 * @hw_event_uptr: event type attributes for monitoring/sampling
1914 * @pid: target pid
1915 * @cpu: target cpu
1916 * @group_fd: group leader counter fd
1917 */
1918 SYSCALL_DEFINE5(perf_counter_open,
1919 const struct perf_counter_hw_event __user *, hw_event_uptr,
1920 pid_t, pid, int, cpu, int, group_fd, unsigned long, flags)
1921 {
1922 struct perf_counter *counter, *group_leader;
1923 struct perf_counter_hw_event hw_event;
1924 struct perf_counter_context *ctx;
1925 struct file *counter_file = NULL;
1926 struct file *group_file = NULL;
1927 int fput_needed = 0;
1928 int fput_needed2 = 0;
1929 int ret;
1930
1931 /* for future expandability... */
1932 if (flags)
1933 return -EINVAL;
1934
1935 if (copy_from_user(&hw_event, hw_event_uptr, sizeof(hw_event)) != 0)
1936 return -EFAULT;
1937
1938 /*
1939 * Get the target context (task or percpu):
1940 */
1941 ctx = find_get_context(pid, cpu);
1942 if (IS_ERR(ctx))
1943 return PTR_ERR(ctx);
1944
1945 /*
1946 * Look up the group leader (we will attach this counter to it):
1947 */
1948 group_leader = NULL;
1949 if (group_fd != -1) {
1950 ret = -EINVAL;
1951 group_file = fget_light(group_fd, &fput_needed);
1952 if (!group_file)
1953 goto err_put_context;
1954 if (group_file->f_op != &perf_fops)
1955 goto err_put_context;
1956
1957 group_leader = group_file->private_data;
1958 /*
1959 * Do not allow a recursive hierarchy (this new sibling
1960 * becoming part of another group-sibling):
1961 */
1962 if (group_leader->group_leader != group_leader)
1963 goto err_put_context;
1964 /*
1965 * Do not allow to attach to a group in a different
1966 * task or CPU context:
1967 */
1968 if (group_leader->ctx != ctx)
1969 goto err_put_context;
1970 /*
1971 * Only a group leader can be exclusive or pinned
1972 */
1973 if (hw_event.exclusive || hw_event.pinned)
1974 goto err_put_context;
1975 }
1976
1977 ret = -EINVAL;
1978 counter = perf_counter_alloc(&hw_event, cpu, ctx, group_leader,
1979 GFP_KERNEL);
1980 if (!counter)
1981 goto err_put_context;
1982
1983 ret = anon_inode_getfd("[perf_counter]", &perf_fops, counter, 0);
1984 if (ret < 0)
1985 goto err_free_put_context;
1986
1987 counter_file = fget_light(ret, &fput_needed2);
1988 if (!counter_file)
1989 goto err_free_put_context;
1990
1991 counter->filp = counter_file;
1992 mutex_lock(&ctx->mutex);
1993 perf_install_in_context(ctx, counter, cpu);
1994 mutex_unlock(&ctx->mutex);
1995
1996 fput_light(counter_file, fput_needed2);
1997
1998 out_fput:
1999 fput_light(group_file, fput_needed);
2000
2001 return ret;
2002
2003 err_free_put_context:
2004 kfree(counter);
2005
2006 err_put_context:
2007 put_context(ctx);
2008
2009 goto out_fput;
2010 }
2011
2012 /*
2013 * Initialize the perf_counter context in a task_struct:
2014 */
2015 static void
2016 __perf_counter_init_context(struct perf_counter_context *ctx,
2017 struct task_struct *task)
2018 {
2019 memset(ctx, 0, sizeof(*ctx));
2020 spin_lock_init(&ctx->lock);
2021 mutex_init(&ctx->mutex);
2022 INIT_LIST_HEAD(&ctx->counter_list);
2023 INIT_LIST_HEAD(&ctx->event_list);
2024 ctx->task = task;
2025 }
2026
2027 /*
2028 * inherit a counter from parent task to child task:
2029 */
2030 static struct perf_counter *
2031 inherit_counter(struct perf_counter *parent_counter,
2032 struct task_struct *parent,
2033 struct perf_counter_context *parent_ctx,
2034 struct task_struct *child,
2035 struct perf_counter *group_leader,
2036 struct perf_counter_context *child_ctx)
2037 {
2038 struct perf_counter *child_counter;
2039
2040 /*
2041 * Instead of creating recursive hierarchies of counters,
2042 * we link inherited counters back to the original parent,
2043 * which has a filp for sure, which we use as the reference
2044 * count:
2045 */
2046 if (parent_counter->parent)
2047 parent_counter = parent_counter->parent;
2048
2049 child_counter = perf_counter_alloc(&parent_counter->hw_event,
2050 parent_counter->cpu, child_ctx,
2051 group_leader, GFP_KERNEL);
2052 if (!child_counter)
2053 return NULL;
2054
2055 /*
2056 * Link it up in the child's context:
2057 */
2058 child_counter->task = child;
2059 list_add_counter(child_counter, child_ctx);
2060 child_ctx->nr_counters++;
2061
2062 child_counter->parent = parent_counter;
2063 /*
2064 * inherit into child's child as well:
2065 */
2066 child_counter->hw_event.inherit = 1;
2067
2068 /*
2069 * Get a reference to the parent filp - we will fput it
2070 * when the child counter exits. This is safe to do because
2071 * we are in the parent and we know that the filp still
2072 * exists and has a nonzero count:
2073 */
2074 atomic_long_inc(&parent_counter->filp->f_count);
2075
2076 /*
2077 * Link this into the parent counter's child list
2078 */
2079 mutex_lock(&parent_counter->mutex);
2080 list_add_tail(&child_counter->child_list, &parent_counter->child_list);
2081
2082 /*
2083 * Make the child state follow the state of the parent counter,
2084 * not its hw_event.disabled bit. We hold the parent's mutex,
2085 * so we won't race with perf_counter_{en,dis}able_family.
2086 */
2087 if (parent_counter->state >= PERF_COUNTER_STATE_INACTIVE)
2088 child_counter->state = PERF_COUNTER_STATE_INACTIVE;
2089 else
2090 child_counter->state = PERF_COUNTER_STATE_OFF;
2091
2092 mutex_unlock(&parent_counter->mutex);
2093
2094 return child_counter;
2095 }
2096
2097 static int inherit_group(struct perf_counter *parent_counter,
2098 struct task_struct *parent,
2099 struct perf_counter_context *parent_ctx,
2100 struct task_struct *child,
2101 struct perf_counter_context *child_ctx)
2102 {
2103 struct perf_counter *leader;
2104 struct perf_counter *sub;
2105
2106 leader = inherit_counter(parent_counter, parent, parent_ctx,
2107 child, NULL, child_ctx);
2108 if (!leader)
2109 return -ENOMEM;
2110 list_for_each_entry(sub, &parent_counter->sibling_list, list_entry) {
2111 if (!inherit_counter(sub, parent, parent_ctx,
2112 child, leader, child_ctx))
2113 return -ENOMEM;
2114 }
2115 return 0;
2116 }
2117
2118 static void sync_child_counter(struct perf_counter *child_counter,
2119 struct perf_counter *parent_counter)
2120 {
2121 u64 parent_val, child_val;
2122
2123 parent_val = atomic64_read(&parent_counter->count);
2124 child_val = atomic64_read(&child_counter->count);
2125
2126 /*
2127 * Add back the child's count to the parent's count:
2128 */
2129 atomic64_add(child_val, &parent_counter->count);
2130
2131 /*
2132 * Remove this counter from the parent's list
2133 */
2134 mutex_lock(&parent_counter->mutex);
2135 list_del_init(&child_counter->child_list);
2136 mutex_unlock(&parent_counter->mutex);
2137
2138 /*
2139 * Release the parent counter, if this was the last
2140 * reference to it.
2141 */
2142 fput(parent_counter->filp);
2143 }
2144
2145 static void
2146 __perf_counter_exit_task(struct task_struct *child,
2147 struct perf_counter *child_counter,
2148 struct perf_counter_context *child_ctx)
2149 {
2150 struct perf_counter *parent_counter;
2151 struct perf_counter *sub, *tmp;
2152
2153 /*
2154 * If we do not self-reap then we have to wait for the
2155 * child task to unschedule (it will happen for sure),
2156 * so that its counter is at its final count. (This
2157 * condition triggers rarely - child tasks usually get
2158 * off their CPU before the parent has a chance to
2159 * get this far into the reaping action)
2160 */
2161 if (child != current) {
2162 wait_task_inactive(child, 0);
2163 list_del_init(&child_counter->list_entry);
2164 } else {
2165 struct perf_cpu_context *cpuctx;
2166 unsigned long flags;
2167 u64 perf_flags;
2168
2169 /*
2170 * Disable and unlink this counter.
2171 *
2172 * Be careful about zapping the list - IRQ/NMI context
2173 * could still be processing it:
2174 */
2175 curr_rq_lock_irq_save(&flags);
2176 perf_flags = hw_perf_save_disable();
2177
2178 cpuctx = &__get_cpu_var(perf_cpu_context);
2179
2180 group_sched_out(child_counter, cpuctx, child_ctx);
2181
2182 list_del_init(&child_counter->list_entry);
2183
2184 child_ctx->nr_counters--;
2185
2186 hw_perf_restore(perf_flags);
2187 curr_rq_unlock_irq_restore(&flags);
2188 }
2189
2190 parent_counter = child_counter->parent;
2191 /*
2192 * It can happen that parent exits first, and has counters
2193 * that are still around due to the child reference. These
2194 * counters need to be zapped - but otherwise linger.
2195 */
2196 if (parent_counter) {
2197 sync_child_counter(child_counter, parent_counter);
2198 list_for_each_entry_safe(sub, tmp, &child_counter->sibling_list,
2199 list_entry) {
2200 if (sub->parent) {
2201 sync_child_counter(sub, sub->parent);
2202 free_counter(sub);
2203 }
2204 }
2205 free_counter(child_counter);
2206 }
2207 }
2208
2209 /*
2210 * When a child task exits, feed back counter values to parent counters.
2211 *
2212 * Note: we may be running in child context, but the PID is not hashed
2213 * anymore so new counters will not be added.
2214 */
2215 void perf_counter_exit_task(struct task_struct *child)
2216 {
2217 struct perf_counter *child_counter, *tmp;
2218 struct perf_counter_context *child_ctx;
2219
2220 child_ctx = &child->perf_counter_ctx;
2221
2222 if (likely(!child_ctx->nr_counters))
2223 return;
2224
2225 list_for_each_entry_safe(child_counter, tmp, &child_ctx->counter_list,
2226 list_entry)
2227 __perf_counter_exit_task(child, child_counter, child_ctx);
2228 }
2229
2230 /*
2231 * Initialize the perf_counter context in task_struct
2232 */
2233 void perf_counter_init_task(struct task_struct *child)
2234 {
2235 struct perf_counter_context *child_ctx, *parent_ctx;
2236 struct perf_counter *counter;
2237 struct task_struct *parent = current;
2238
2239 child_ctx = &child->perf_counter_ctx;
2240 parent_ctx = &parent->perf_counter_ctx;
2241
2242 __perf_counter_init_context(child_ctx, child);
2243
2244 /*
2245 * This is executed from the parent task context, so inherit
2246 * counters that have been marked for cloning:
2247 */
2248
2249 if (likely(!parent_ctx->nr_counters))
2250 return;
2251
2252 /*
2253 * Lock the parent list. No need to lock the child - not PID
2254 * hashed yet and not running, so nobody can access it.
2255 */
2256 mutex_lock(&parent_ctx->mutex);
2257
2258 /*
2259 * We dont have to disable NMIs - we are only looking at
2260 * the list, not manipulating it:
2261 */
2262 list_for_each_entry(counter, &parent_ctx->counter_list, list_entry) {
2263 if (!counter->hw_event.inherit)
2264 continue;
2265
2266 if (inherit_group(counter, parent,
2267 parent_ctx, child, child_ctx))
2268 break;
2269 }
2270
2271 mutex_unlock(&parent_ctx->mutex);
2272 }
2273
2274 static void __cpuinit perf_counter_init_cpu(int cpu)
2275 {
2276 struct perf_cpu_context *cpuctx;
2277
2278 cpuctx = &per_cpu(perf_cpu_context, cpu);
2279 __perf_counter_init_context(&cpuctx->ctx, NULL);
2280
2281 mutex_lock(&perf_resource_mutex);
2282 cpuctx->max_pertask = perf_max_counters - perf_reserved_percpu;
2283 mutex_unlock(&perf_resource_mutex);
2284
2285 hw_perf_counter_setup(cpu);
2286 }
2287
2288 #ifdef CONFIG_HOTPLUG_CPU
2289 static void __perf_counter_exit_cpu(void *info)
2290 {
2291 struct perf_cpu_context *cpuctx = &__get_cpu_var(perf_cpu_context);
2292 struct perf_counter_context *ctx = &cpuctx->ctx;
2293 struct perf_counter *counter, *tmp;
2294
2295 list_for_each_entry_safe(counter, tmp, &ctx->counter_list, list_entry)
2296 __perf_counter_remove_from_context(counter);
2297 }
2298 static void perf_counter_exit_cpu(int cpu)
2299 {
2300 struct perf_cpu_context *cpuctx = &per_cpu(perf_cpu_context, cpu);
2301 struct perf_counter_context *ctx = &cpuctx->ctx;
2302
2303 mutex_lock(&ctx->mutex);
2304 smp_call_function_single(cpu, __perf_counter_exit_cpu, NULL, 1);
2305 mutex_unlock(&ctx->mutex);
2306 }
2307 #else
2308 static inline void perf_counter_exit_cpu(int cpu) { }
2309 #endif
2310
2311 static int __cpuinit
2312 perf_cpu_notify(struct notifier_block *self, unsigned long action, void *hcpu)
2313 {
2314 unsigned int cpu = (long)hcpu;
2315
2316 switch (action) {
2317
2318 case CPU_UP_PREPARE:
2319 case CPU_UP_PREPARE_FROZEN:
2320 perf_counter_init_cpu(cpu);
2321 break;
2322
2323 case CPU_DOWN_PREPARE:
2324 case CPU_DOWN_PREPARE_FROZEN:
2325 perf_counter_exit_cpu(cpu);
2326 break;
2327
2328 default:
2329 break;
2330 }
2331
2332 return NOTIFY_OK;
2333 }
2334
2335 static struct notifier_block __cpuinitdata perf_cpu_nb = {
2336 .notifier_call = perf_cpu_notify,
2337 };
2338
2339 static int __init perf_counter_init(void)
2340 {
2341 perf_cpu_notify(&perf_cpu_nb, (unsigned long)CPU_UP_PREPARE,
2342 (void *)(long)smp_processor_id());
2343 register_cpu_notifier(&perf_cpu_nb);
2344
2345 return 0;
2346 }
2347 early_initcall(perf_counter_init);
2348
2349 static ssize_t perf_show_reserve_percpu(struct sysdev_class *class, char *buf)
2350 {
2351 return sprintf(buf, "%d\n", perf_reserved_percpu);
2352 }
2353
2354 static ssize_t
2355 perf_set_reserve_percpu(struct sysdev_class *class,
2356 const char *buf,
2357 size_t count)
2358 {
2359 struct perf_cpu_context *cpuctx;
2360 unsigned long val;
2361 int err, cpu, mpt;
2362
2363 err = strict_strtoul(buf, 10, &val);
2364 if (err)
2365 return err;
2366 if (val > perf_max_counters)
2367 return -EINVAL;
2368
2369 mutex_lock(&perf_resource_mutex);
2370 perf_reserved_percpu = val;
2371 for_each_online_cpu(cpu) {
2372 cpuctx = &per_cpu(perf_cpu_context, cpu);
2373 spin_lock_irq(&cpuctx->ctx.lock);
2374 mpt = min(perf_max_counters - cpuctx->ctx.nr_counters,
2375 perf_max_counters - perf_reserved_percpu);
2376 cpuctx->max_pertask = mpt;
2377 spin_unlock_irq(&cpuctx->ctx.lock);
2378 }
2379 mutex_unlock(&perf_resource_mutex);
2380
2381 return count;
2382 }
2383
2384 static ssize_t perf_show_overcommit(struct sysdev_class *class, char *buf)
2385 {
2386 return sprintf(buf, "%d\n", perf_overcommit);
2387 }
2388
2389 static ssize_t
2390 perf_set_overcommit(struct sysdev_class *class, const char *buf, size_t count)
2391 {
2392 unsigned long val;
2393 int err;
2394
2395 err = strict_strtoul(buf, 10, &val);
2396 if (err)
2397 return err;
2398 if (val > 1)
2399 return -EINVAL;
2400
2401 mutex_lock(&perf_resource_mutex);
2402 perf_overcommit = val;
2403 mutex_unlock(&perf_resource_mutex);
2404
2405 return count;
2406 }
2407
2408 static SYSDEV_CLASS_ATTR(
2409 reserve_percpu,
2410 0644,
2411 perf_show_reserve_percpu,
2412 perf_set_reserve_percpu
2413 );
2414
2415 static SYSDEV_CLASS_ATTR(
2416 overcommit,
2417 0644,
2418 perf_show_overcommit,
2419 perf_set_overcommit
2420 );
2421
2422 static struct attribute *perfclass_attrs[] = {
2423 &attr_reserve_percpu.attr,
2424 &attr_overcommit.attr,
2425 NULL
2426 };
2427
2428 static struct attribute_group perfclass_attr_group = {
2429 .attrs = perfclass_attrs,
2430 .name = "perf_counters",
2431 };
2432
2433 static int __init perf_counter_sysfs_init(void)
2434 {
2435 return sysfs_create_group(&cpu_sysdev_class.kset.kobj,
2436 &perfclass_attr_group);
2437 }
2438 device_initcall(perf_counter_sysfs_init);
This page took 0.130411 seconds and 6 git commands to generate.