2 * Performance events ring-buffer code:
4 * Copyright (C) 2008 Thomas Gleixner <tglx@linutronix.de>
5 * Copyright (C) 2008-2011 Red Hat, Inc., Ingo Molnar
6 * Copyright (C) 2008-2011 Red Hat, Inc., Peter Zijlstra
7 * Copyright © 2009 Paul Mackerras, IBM Corp. <paulus@au1.ibm.com>
9 * For licensing details see kernel-base/COPYING
12 #include <linux/perf_event.h>
13 #include <linux/vmalloc.h>
14 #include <linux/slab.h>
15 #include <linux/circ_buf.h>
16 #include <linux/poll.h>
20 static void perf_output_wakeup(struct perf_output_handle
*handle
)
22 atomic_set(&handle
->rb
->poll
, POLLIN
);
24 handle
->event
->pending_wakeup
= 1;
25 irq_work_queue(&handle
->event
->pending
);
29 * We need to ensure a later event_id doesn't publish a head when a former
30 * event isn't done writing. However since we need to deal with NMIs we
31 * cannot fully serialize things.
33 * We only publish the head (and generate a wakeup) when the outer-most
36 static void perf_output_get_handle(struct perf_output_handle
*handle
)
38 struct ring_buffer
*rb
= handle
->rb
;
42 handle
->wakeup
= local_read(&rb
->wakeup
);
45 static void perf_output_put_handle(struct perf_output_handle
*handle
)
47 struct ring_buffer
*rb
= handle
->rb
;
51 head
= local_read(&rb
->head
);
54 * IRQ/NMI can happen here, which means we can miss a head update.
57 if (!local_dec_and_test(&rb
->nest
))
61 * Since the mmap() consumer (userspace) can run on a different CPU:
65 * if (LOAD ->data_tail) { LOAD ->data_head
67 * STORE $data LOAD $data
68 * smp_wmb() (B) smp_mb() (D)
69 * STORE ->data_head STORE ->data_tail
72 * Where A pairs with D, and B pairs with C.
74 * In our case (A) is a control dependency that separates the load of
75 * the ->data_tail and the stores of $data. In case ->data_tail
76 * indicates there is no room in the buffer to store $data we do not.
78 * D needs to be a full barrier since it separates the data READ
79 * from the tail WRITE.
81 * For B a WMB is sufficient since it separates two WRITEs, and for C
82 * an RMB is sufficient since it separates two READs.
84 * See perf_output_begin().
86 smp_wmb(); /* B, matches C */
87 rb
->user_page
->data_head
= head
;
90 * Now check if we missed an update -- rely on previous implied
91 * compiler barriers to force a re-read.
93 if (unlikely(head
!= local_read(&rb
->head
))) {
98 if (handle
->wakeup
!= local_read(&rb
->wakeup
))
99 perf_output_wakeup(handle
);
105 int perf_output_begin(struct perf_output_handle
*handle
,
106 struct perf_event
*event
, unsigned int size
)
108 struct ring_buffer
*rb
;
109 unsigned long tail
, offset
, head
;
110 int have_lost
, page_shift
;
112 struct perf_event_header header
;
119 * For inherited events we send all the output towards the parent.
122 event
= event
->parent
;
124 rb
= rcu_dereference(event
->rb
);
128 if (unlikely(!rb
->nr_pages
))
132 handle
->event
= event
;
134 have_lost
= local_read(&rb
->lost
);
135 if (unlikely(have_lost
)) {
136 size
+= sizeof(lost_event
);
137 if (event
->attr
.sample_id_all
)
138 size
+= event
->id_header_size
;
141 perf_output_get_handle(handle
);
144 tail
= READ_ONCE(rb
->user_page
->data_tail
);
145 offset
= head
= local_read(&rb
->head
);
146 if (!rb
->overwrite
&&
147 unlikely(CIRC_SPACE(head
, tail
, perf_data_size(rb
)) < size
))
151 * The above forms a control dependency barrier separating the
152 * @tail load above from the data stores below. Since the @tail
153 * load is required to compute the branch to fail below.
155 * A, matches D; the full memory barrier userspace SHOULD issue
156 * after reading the data and before storing the new tail
159 * See perf_output_put_handle().
163 } while (local_cmpxchg(&rb
->head
, offset
, head
) != offset
);
166 * We rely on the implied barrier() by local_cmpxchg() to ensure
167 * none of the data stores below can be lifted up by the compiler.
170 if (unlikely(head
- local_read(&rb
->wakeup
) > rb
->watermark
))
171 local_add(rb
->watermark
, &rb
->wakeup
);
173 page_shift
= PAGE_SHIFT
+ page_order(rb
);
175 handle
->page
= (offset
>> page_shift
) & (rb
->nr_pages
- 1);
176 offset
&= (1UL << page_shift
) - 1;
177 handle
->addr
= rb
->data_pages
[handle
->page
] + offset
;
178 handle
->size
= (1UL << page_shift
) - offset
;
180 if (unlikely(have_lost
)) {
181 struct perf_sample_data sample_data
;
183 lost_event
.header
.size
= sizeof(lost_event
);
184 lost_event
.header
.type
= PERF_RECORD_LOST
;
185 lost_event
.header
.misc
= 0;
186 lost_event
.id
= event
->id
;
187 lost_event
.lost
= local_xchg(&rb
->lost
, 0);
189 perf_event_header__init_id(&lost_event
.header
,
190 &sample_data
, event
);
191 perf_output_put(handle
, lost_event
);
192 perf_event__output_id_sample(event
, handle
, &sample_data
);
198 local_inc(&rb
->lost
);
199 perf_output_put_handle(handle
);
206 unsigned int perf_output_copy(struct perf_output_handle
*handle
,
207 const void *buf
, unsigned int len
)
209 return __output_copy(handle
, buf
, len
);
212 unsigned int perf_output_skip(struct perf_output_handle
*handle
,
215 return __output_skip(handle
, NULL
, len
);
218 void perf_output_end(struct perf_output_handle
*handle
)
220 perf_output_put_handle(handle
);
224 static void rb_irq_work(struct irq_work
*work
);
227 ring_buffer_init(struct ring_buffer
*rb
, long watermark
, int flags
)
229 long max_size
= perf_data_size(rb
);
232 rb
->watermark
= min(max_size
, watermark
);
235 rb
->watermark
= max_size
/ 2;
237 if (flags
& RING_BUFFER_WRITABLE
)
242 atomic_set(&rb
->refcount
, 1);
244 INIT_LIST_HEAD(&rb
->event_list
);
245 spin_lock_init(&rb
->event_lock
);
246 init_irq_work(&rb
->irq_work
, rb_irq_work
);
249 static void ring_buffer_put_async(struct ring_buffer
*rb
)
251 if (!atomic_dec_and_test(&rb
->refcount
))
254 rb
->rcu_head
.next
= (void *)rb
;
255 irq_work_queue(&rb
->irq_work
);
259 * This is called before hardware starts writing to the AUX area to
260 * obtain an output handle and make sure there's room in the buffer.
261 * When the capture completes, call perf_aux_output_end() to commit
262 * the recorded data to the buffer.
264 * The ordering is similar to that of perf_output_{begin,end}, with
265 * the exception of (B), which should be taken care of by the pmu
266 * driver, since ordering rules will differ depending on hardware.
268 void *perf_aux_output_begin(struct perf_output_handle
*handle
,
269 struct perf_event
*event
)
271 struct perf_event
*output_event
= event
;
272 unsigned long aux_head
, aux_tail
;
273 struct ring_buffer
*rb
;
275 if (output_event
->parent
)
276 output_event
= output_event
->parent
;
279 * Since this will typically be open across pmu::add/pmu::del, we
280 * grab ring_buffer's refcount instead of holding rcu read lock
281 * to make sure it doesn't disappear under us.
283 rb
= ring_buffer_get(output_event
);
287 if (!rb_has_aux(rb
) || !atomic_inc_not_zero(&rb
->aux_refcount
))
291 * Nesting is not supported for AUX area, make sure nested
292 * writers are caught early
294 if (WARN_ON_ONCE(local_xchg(&rb
->aux_nest
, 1)))
297 aux_head
= local_read(&rb
->aux_head
);
300 handle
->event
= event
;
301 handle
->head
= aux_head
;
305 * In overwrite mode, AUX data stores do not depend on aux_tail,
306 * therefore (A) control dependency barrier does not exist. The
307 * (B) <-> (C) ordering is still observed by the pmu driver.
309 if (!rb
->aux_overwrite
) {
310 aux_tail
= ACCESS_ONCE(rb
->user_page
->aux_tail
);
311 handle
->wakeup
= local_read(&rb
->aux_wakeup
) + rb
->aux_watermark
;
312 if (aux_head
- aux_tail
< perf_aux_size(rb
))
313 handle
->size
= CIRC_SPACE(aux_head
, aux_tail
, perf_aux_size(rb
));
316 * handle->size computation depends on aux_tail load; this forms a
317 * control dependency barrier separating aux_tail load from aux data
318 * store that will be enabled on successful return
320 if (!handle
->size
) { /* A, matches D */
321 event
->pending_disable
= 1;
322 perf_output_wakeup(handle
);
323 local_set(&rb
->aux_nest
, 0);
328 return handle
->rb
->aux_priv
;
334 ring_buffer_put_async(rb
);
335 handle
->event
= NULL
;
341 * Commit the data written by hardware into the ring buffer by adjusting
342 * aux_head and posting a PERF_RECORD_AUX into the perf buffer. It is the
343 * pmu driver's responsibility to observe ordering rules of the hardware,
344 * so that all the data is externally visible before this is called.
346 void perf_aux_output_end(struct perf_output_handle
*handle
, unsigned long size
,
349 struct ring_buffer
*rb
= handle
->rb
;
350 bool wakeup
= truncated
;
351 unsigned long aux_head
;
355 flags
|= PERF_AUX_FLAG_TRUNCATED
;
357 /* in overwrite mode, driver provides aux_head via handle */
358 if (rb
->aux_overwrite
) {
359 flags
|= PERF_AUX_FLAG_OVERWRITE
;
361 aux_head
= handle
->head
;
362 local_set(&rb
->aux_head
, aux_head
);
364 aux_head
= local_read(&rb
->aux_head
);
365 local_add(size
, &rb
->aux_head
);
370 * Only send RECORD_AUX if we have something useful to communicate
373 perf_event_aux_event(handle
->event
, aux_head
, size
, flags
);
376 aux_head
= rb
->user_page
->aux_head
= local_read(&rb
->aux_head
);
378 if (aux_head
- local_read(&rb
->aux_wakeup
) >= rb
->aux_watermark
) {
380 local_add(rb
->aux_watermark
, &rb
->aux_wakeup
);
385 handle
->event
->pending_disable
= 1;
386 perf_output_wakeup(handle
);
389 handle
->event
= NULL
;
391 local_set(&rb
->aux_nest
, 0);
393 ring_buffer_put_async(rb
);
397 * Skip over a given number of bytes in the AUX buffer, due to, for example,
398 * hardware's alignment constraints.
400 int perf_aux_output_skip(struct perf_output_handle
*handle
, unsigned long size
)
402 struct ring_buffer
*rb
= handle
->rb
;
403 unsigned long aux_head
;
405 if (size
> handle
->size
)
408 local_add(size
, &rb
->aux_head
);
410 aux_head
= rb
->user_page
->aux_head
= local_read(&rb
->aux_head
);
411 if (aux_head
- local_read(&rb
->aux_wakeup
) >= rb
->aux_watermark
) {
412 perf_output_wakeup(handle
);
413 local_add(rb
->aux_watermark
, &rb
->aux_wakeup
);
414 handle
->wakeup
= local_read(&rb
->aux_wakeup
) +
418 handle
->head
= aux_head
;
419 handle
->size
-= size
;
424 void *perf_get_aux(struct perf_output_handle
*handle
)
426 /* this is only valid between perf_aux_output_begin and *_end */
430 return handle
->rb
->aux_priv
;
433 #define PERF_AUX_GFP (GFP_KERNEL | __GFP_ZERO | __GFP_NOWARN | __GFP_NORETRY)
435 static struct page
*rb_alloc_aux_page(int node
, int order
)
439 if (order
> MAX_ORDER
)
443 page
= alloc_pages_node(node
, PERF_AUX_GFP
, order
);
444 } while (!page
&& order
--);
448 * Communicate the allocation size to the driver:
449 * if we managed to secure a high-order allocation,
450 * set its first page's private to this order;
451 * !PagePrivate(page) means it's just a normal page.
453 split_page(page
, order
);
454 SetPagePrivate(page
);
455 set_page_private(page
, order
);
461 static void rb_free_aux_page(struct ring_buffer
*rb
, int idx
)
463 struct page
*page
= virt_to_page(rb
->aux_pages
[idx
]);
465 ClearPagePrivate(page
);
466 page
->mapping
= NULL
;
470 static void __rb_free_aux(struct ring_buffer
*rb
)
475 rb
->free_aux(rb
->aux_priv
);
480 if (rb
->aux_nr_pages
) {
481 for (pg
= 0; pg
< rb
->aux_nr_pages
; pg
++)
482 rb_free_aux_page(rb
, pg
);
484 kfree(rb
->aux_pages
);
485 rb
->aux_nr_pages
= 0;
489 int rb_alloc_aux(struct ring_buffer
*rb
, struct perf_event
*event
,
490 pgoff_t pgoff
, int nr_pages
, long watermark
, int flags
)
492 bool overwrite
= !(flags
& RING_BUFFER_WRITABLE
);
493 int node
= (event
->cpu
== -1) ? -1 : cpu_to_node(event
->cpu
);
494 int ret
= -ENOMEM
, max_order
= 0;
499 if (event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_NO_SG
) {
501 * We need to start with the max_order that fits in nr_pages,
502 * not the other way around, hence ilog2() and not get_order.
504 max_order
= ilog2(nr_pages
);
507 * PMU requests more than one contiguous chunks of memory
508 * for SW double buffering
510 if ((event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_SW_DOUBLEBUF
) &&
519 rb
->aux_pages
= kzalloc_node(nr_pages
* sizeof(void *), GFP_KERNEL
, node
);
523 rb
->free_aux
= event
->pmu
->free_aux
;
524 for (rb
->aux_nr_pages
= 0; rb
->aux_nr_pages
< nr_pages
;) {
528 order
= min(max_order
, ilog2(nr_pages
- rb
->aux_nr_pages
));
529 page
= rb_alloc_aux_page(node
, order
);
533 for (last
= rb
->aux_nr_pages
+ (1 << page_private(page
));
534 last
> rb
->aux_nr_pages
; rb
->aux_nr_pages
++)
535 rb
->aux_pages
[rb
->aux_nr_pages
] = page_address(page
++);
539 * In overwrite mode, PMUs that don't support SG may not handle more
540 * than one contiguous allocation, since they rely on PMI to do double
541 * buffering. In this case, the entire buffer has to be one contiguous
544 if ((event
->pmu
->capabilities
& PERF_PMU_CAP_AUX_NO_SG
) &&
546 struct page
*page
= virt_to_page(rb
->aux_pages
[0]);
548 if (page_private(page
) != max_order
)
552 rb
->aux_priv
= event
->pmu
->setup_aux(event
->cpu
, rb
->aux_pages
, nr_pages
,
560 * aux_pages (and pmu driver's private data, aux_priv) will be
561 * referenced in both producer's and consumer's contexts, thus
562 * we keep a refcount here to make sure either of the two can
563 * reference them safely.
565 atomic_set(&rb
->aux_refcount
, 1);
567 rb
->aux_overwrite
= overwrite
;
568 rb
->aux_watermark
= watermark
;
570 if (!rb
->aux_watermark
&& !rb
->aux_overwrite
)
571 rb
->aux_watermark
= nr_pages
<< (PAGE_SHIFT
- 1);
575 rb
->aux_pgoff
= pgoff
;
582 void rb_free_aux(struct ring_buffer
*rb
)
584 if (atomic_dec_and_test(&rb
->aux_refcount
))
585 irq_work_queue(&rb
->irq_work
);
588 static void rb_irq_work(struct irq_work
*work
)
590 struct ring_buffer
*rb
= container_of(work
, struct ring_buffer
, irq_work
);
592 if (!atomic_read(&rb
->aux_refcount
))
595 if (rb
->rcu_head
.next
== (void *)rb
)
596 call_rcu(&rb
->rcu_head
, rb_free_rcu
);
599 #ifndef CONFIG_PERF_USE_VMALLOC
602 * Back perf_mmap() with regular GFP_KERNEL-0 pages.
606 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
608 if (pgoff
> rb
->nr_pages
)
612 return virt_to_page(rb
->user_page
);
614 return virt_to_page(rb
->data_pages
[pgoff
- 1]);
617 static void *perf_mmap_alloc_page(int cpu
)
622 node
= (cpu
== -1) ? cpu
: cpu_to_node(cpu
);
623 page
= alloc_pages_node(node
, GFP_KERNEL
| __GFP_ZERO
, 0);
627 return page_address(page
);
630 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
632 struct ring_buffer
*rb
;
636 size
= sizeof(struct ring_buffer
);
637 size
+= nr_pages
* sizeof(void *);
639 rb
= kzalloc(size
, GFP_KERNEL
);
643 rb
->user_page
= perf_mmap_alloc_page(cpu
);
647 for (i
= 0; i
< nr_pages
; i
++) {
648 rb
->data_pages
[i
] = perf_mmap_alloc_page(cpu
);
649 if (!rb
->data_pages
[i
])
650 goto fail_data_pages
;
653 rb
->nr_pages
= nr_pages
;
655 ring_buffer_init(rb
, watermark
, flags
);
660 for (i
--; i
>= 0; i
--)
661 free_page((unsigned long)rb
->data_pages
[i
]);
663 free_page((unsigned long)rb
->user_page
);
672 static void perf_mmap_free_page(unsigned long addr
)
674 struct page
*page
= virt_to_page((void *)addr
);
676 page
->mapping
= NULL
;
680 void rb_free(struct ring_buffer
*rb
)
684 perf_mmap_free_page((unsigned long)rb
->user_page
);
685 for (i
= 0; i
< rb
->nr_pages
; i
++)
686 perf_mmap_free_page((unsigned long)rb
->data_pages
[i
]);
691 static int data_page_nr(struct ring_buffer
*rb
)
693 return rb
->nr_pages
<< page_order(rb
);
697 __perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
699 /* The '>' counts in the user page. */
700 if (pgoff
> data_page_nr(rb
))
703 return vmalloc_to_page((void *)rb
->user_page
+ pgoff
* PAGE_SIZE
);
706 static void perf_mmap_unmark_page(void *addr
)
708 struct page
*page
= vmalloc_to_page(addr
);
710 page
->mapping
= NULL
;
713 static void rb_free_work(struct work_struct
*work
)
715 struct ring_buffer
*rb
;
719 rb
= container_of(work
, struct ring_buffer
, work
);
720 nr
= data_page_nr(rb
);
722 base
= rb
->user_page
;
723 /* The '<=' counts in the user page. */
724 for (i
= 0; i
<= nr
; i
++)
725 perf_mmap_unmark_page(base
+ (i
* PAGE_SIZE
));
731 void rb_free(struct ring_buffer
*rb
)
733 schedule_work(&rb
->work
);
736 struct ring_buffer
*rb_alloc(int nr_pages
, long watermark
, int cpu
, int flags
)
738 struct ring_buffer
*rb
;
742 size
= sizeof(struct ring_buffer
);
743 size
+= sizeof(void *);
745 rb
= kzalloc(size
, GFP_KERNEL
);
749 INIT_WORK(&rb
->work
, rb_free_work
);
751 all_buf
= vmalloc_user((nr_pages
+ 1) * PAGE_SIZE
);
755 rb
->user_page
= all_buf
;
756 rb
->data_pages
[0] = all_buf
+ PAGE_SIZE
;
759 rb
->page_order
= ilog2(nr_pages
);
762 ring_buffer_init(rb
, watermark
, flags
);
776 perf_mmap_to_page(struct ring_buffer
*rb
, unsigned long pgoff
)
778 if (rb
->aux_nr_pages
) {
779 /* above AUX space */
780 if (pgoff
> rb
->aux_pgoff
+ rb
->aux_nr_pages
)
784 if (pgoff
>= rb
->aux_pgoff
)
785 return virt_to_page(rb
->aux_pages
[pgoff
- rb
->aux_pgoff
]);
788 return __perf_mmap_to_page(rb
, pgoff
);