projects / deliverable / linux.git / blame
? search:
summary | shortlog | log | commit | commitdiff | tree
blob | blame (incremental) | history | HEAD
Merge tag 'sound-3.17-rc1' of git://git.kernel.org/pub/scm/linux/kernel/git/tiwai...
[deliverable/linux.git] / arch / s390 / oprofile / hwsampler.c
CommitLineData
a53c8fab 1/*
ec6a3df1
HG		 2 * Copyright IBM Corp. 2010
3 * Author: Heinz Graalfs <graalfs@de.ibm.com>
4 */
5
fcdd65b0 6#include <linux/kernel_stat.h>
ec6a3df1
HG		 7#include <linux/kernel.h>
8#include <linux/module.h>
9#include <linux/smp.h>
10#include <linux/errno.h>
11#include <linux/workqueue.h>
12#include <linux/interrupt.h>
13#include <linux/notifier.h>
14#include <linux/cpu.h>
15#include <linux/semaphore.h>
16#include <linux/oom.h>
17#include <linux/oprofile.h>
18
a0616cde 19#include <asm/facility.h>
b03d541a 20#include <asm/cpu_mf.h>
d7b250e2 21#include <asm/irq.h>
ec6a3df1
HG		 22
23#include "hwsampler.h"
dd3c4670 24#include "op_counter.h"
ec6a3df1
HG		 25
26#define MAX_NUM_SDB 511
27#define MIN_NUM_SDB 1
28
ec6a3df1
HG		 29DECLARE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
30
31struct hws_execute_parms {
32 void *buffer;
33 signed int rc;
34};
35
36DEFINE_PER_CPU(struct hws_cpu_buffer, sampler_cpu_buffer);
37EXPORT_PER_CPU_SYMBOL(sampler_cpu_buffer);
38
39static DEFINE_MUTEX(hws_sem);
40static DEFINE_MUTEX(hws_sem_oom);
41
42static unsigned char hws_flush_all;
43static unsigned int hws_oom;
e28bb79d 44static unsigned int hws_alert;
ec6a3df1
HG		 45static struct workqueue_struct *hws_wq;
46
47static unsigned int hws_state;
48enum {
49 HWS_INIT = 1,
50 HWS_DEALLOCATED,
51 HWS_STOPPED,
52 HWS_STARTED,
53 HWS_STOPPING };
54
55/* set to 1 if called by kernel during memory allocation */
56static unsigned char oom_killer_was_active;
57/* size of SDBT and SDB as of allocate API */
58static unsigned long num_sdbt = 100;
59static unsigned long num_sdb = 511;
60/* sampling interval (machine cycles) */
61static unsigned long interval;
62
63static unsigned long min_sampler_rate;
64static unsigned long max_sampler_rate;
65
ec6a3df1
HG		 66static void execute_qsi(void *parms)
67{
68 struct hws_execute_parms *ep = parms;
69
70 ep->rc = qsi(ep->buffer);
71}
72
73static void execute_ssctl(void *parms)
74{
75 struct hws_execute_parms *ep = parms;
76
cf48ad83 77 ep->rc = lsctl(ep->buffer);
ec6a3df1
HG		 78}
79
80static int smp_ctl_ssctl_stop(int cpu)
81{
82 int rc;
83 struct hws_execute_parms ep;
84 struct hws_cpu_buffer *cb;
85
86 cb = &per_cpu(sampler_cpu_buffer, cpu);
87
88 cb->ssctl.es = 0;
89 cb->ssctl.cs = 0;
90
91 ep.buffer = &cb->ssctl;
92 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
93 rc = ep.rc;
94 if (rc) {
95 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
96 dump_stack();
97 }
98
99 ep.buffer = &cb->qsi;
100 smp_call_function_single(cpu, execute_qsi, &ep, 1);
101
102 if (cb->qsi.es || cb->qsi.cs) {
103 printk(KERN_EMERG "CPUMF sampling did not stop properly.\n");
104 dump_stack();
105 }
106
107 return rc;
108}
109
110static int smp_ctl_ssctl_deactivate(int cpu)
111{
112 int rc;
113 struct hws_execute_parms ep;
114 struct hws_cpu_buffer *cb;
115
116 cb = &per_cpu(sampler_cpu_buffer, cpu);
117
118 cb->ssctl.es = 1;
119 cb->ssctl.cs = 0;
120
121 ep.buffer = &cb->ssctl;
122 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
123 rc = ep.rc;
124 if (rc)
125 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
126
127 ep.buffer = &cb->qsi;
128 smp_call_function_single(cpu, execute_qsi, &ep, 1);
129
130 if (cb->qsi.cs)
131 printk(KERN_EMERG "CPUMF sampling was not set inactive.\n");
132
133 return rc;
134}
135
136static int smp_ctl_ssctl_enable_activate(int cpu, unsigned long interval)
137{
138 int rc;
139 struct hws_execute_parms ep;
140 struct hws_cpu_buffer *cb;
141
142 cb = &per_cpu(sampler_cpu_buffer, cpu);
143
144 cb->ssctl.h = 1;
145 cb->ssctl.tear = cb->first_sdbt;
146 cb->ssctl.dear = *(unsigned long *) cb->first_sdbt;
147 cb->ssctl.interval = interval;
148 cb->ssctl.es = 1;
149 cb->ssctl.cs = 1;
150
151 ep.buffer = &cb->ssctl;
152 smp_call_function_single(cpu, execute_ssctl, &ep, 1);
153 rc = ep.rc;
154 if (rc)
155 printk(KERN_ERR "hwsampler: CPU %d CPUMF SSCTL failed.\n", cpu);
156
157 ep.buffer = &cb->qsi;
158 smp_call_function_single(cpu, execute_qsi, &ep, 1);
159 if (ep.rc)
160 printk(KERN_ERR "hwsampler: CPU %d CPUMF QSI failed.\n", cpu);
161
162 return rc;
163}
164
165static int smp_ctl_qsi(int cpu)
166{
167 struct hws_execute_parms ep;
168 struct hws_cpu_buffer *cb;
169
170 cb = &per_cpu(sampler_cpu_buffer, cpu);
171
172 ep.buffer = &cb->qsi;
173 smp_call_function_single(cpu, execute_qsi, &ep, 1);
174
175 return ep.rc;
176}
177
fde15c3a 178static void hws_ext_handler(struct ext_code ext_code,
b03d541a
JG		 179 unsigned int param32, unsigned long param64)
180{
181 struct hws_cpu_buffer *cb = &__get_cpu_var(sampler_cpu_buffer);
182
183 if (!(param32 & CPU_MF_INT_SF_MASK))
184 return;
185
e28bb79d
HB		 186 if (!hws_alert)
187 return;
188
420f42ec 189 inc_irq_stat(IRQEXT_CMS);
b03d541a
JG		 190 atomic_xchg(&cb->ext_params, atomic_read(&cb->ext_params) | param32);
191
192 if (hws_wq)
193 queue_work(hws_wq, &cb->worker);
194}
ec6a3df1
HG		 195
196static void worker(struct work_struct *work);
197
198static void add_samples_to_oprofile(unsigned cpu, unsigned long *,
199 unsigned long *dear);
200
201static void init_all_cpu_buffers(void)
202{
203 int cpu;
204 struct hws_cpu_buffer *cb;
205
206 for_each_online_cpu(cpu) {
207 cb = &per_cpu(sampler_cpu_buffer, cpu);
208 memset(cb, 0, sizeof(struct hws_cpu_buffer));
209 }
210}
211
94414ca0 212static void prepare_cpu_buffers(void)
ec6a3df1 213{
ec6a3df1 214 struct hws_cpu_buffer *cb;
94414ca0 215 int cpu;
ec6a3df1 216
ec6a3df1
HG		 217 for_each_online_cpu(cpu) {
218 cb = &per_cpu(sampler_cpu_buffer, cpu);
219 atomic_set(&cb->ext_params, 0);
220 cb->worker_entry = 0;
221 cb->sample_overflow = 0;
222 cb->req_alert = 0;
223 cb->incorrect_sdbt_entry = 0;
224 cb->invalid_entry_address = 0;
225 cb->loss_of_sample_data = 0;
226 cb->sample_auth_change_alert = 0;
227 cb->finish = 0;
228 cb->oom = 0;
229 cb->stop_mode = 0;
230 }
ec6a3df1
HG		 231}
232
233/*
234 * allocate_sdbt() - allocate sampler memory
235 * @cpu: the cpu for which sampler memory is allocated
236 *
237 * A 4K page is allocated for each requested SDBT.
238 * A maximum of 511 4K pages are allocated for the SDBs in each of the SDBTs.
239 * Set ALERT_REQ mask in each SDBs trailer.
240 * Returns zero if successful, <0 otherwise.
241 */
242static int allocate_sdbt(int cpu)
243{
244 int j, k, rc;
245 unsigned long *sdbt;
246 unsigned long sdb;
247 unsigned long *tail;
248 unsigned long *trailer;
249 struct hws_cpu_buffer *cb;
250
251 cb = &per_cpu(sampler_cpu_buffer, cpu);
252
253 if (cb->first_sdbt)
254 return -EINVAL;
255
256 sdbt = NULL;
257 tail = sdbt;
258
259 for (j = 0; j < num_sdbt; j++) {
260 sdbt = (unsigned long *)get_zeroed_page(GFP_KERNEL);
261
262 mutex_lock(&hws_sem_oom);
263 /* OOM killer might have been activated */
264 barrier();
265 if (oom_killer_was_active || !sdbt) {
266 if (sdbt)
267 free_page((unsigned long)sdbt);
268
269 goto allocate_sdbt_error;
270 }
271 if (cb->first_sdbt == 0)
272 cb->first_sdbt = (unsigned long)sdbt;
273
274 /* link current page to tail of chain */
275 if (tail)
276 *tail = (unsigned long)(void *)sdbt + 1;
277
278 mutex_unlock(&hws_sem_oom);
279
280 for (k = 0; k < num_sdb; k++) {
281 /* get and set SDB page */
282 sdb = get_zeroed_page(GFP_KERNEL);
283
284 mutex_lock(&hws_sem_oom);
285 /* OOM killer might have been activated */
286 barrier();
287 if (oom_killer_was_active || !sdb) {
288 if (sdb)
289 free_page(sdb);
290
291 goto allocate_sdbt_error;
292 }
293 *sdbt = sdb;
294 trailer = trailer_entry_ptr(*sdbt);
cf48ad83 295 *trailer = SDB_TE_ALERT_REQ_MASK;
ec6a3df1
HG		 296 sdbt++;
297 mutex_unlock(&hws_sem_oom);
298 }
299 tail = sdbt;
300 }
301 mutex_lock(&hws_sem_oom);
302 if (oom_killer_was_active)
303 goto allocate_sdbt_error;
304
305 rc = 0;
306 if (tail)
307 *tail = (unsigned long)
308 ((void *)cb->first_sdbt) + 1;
309
310allocate_sdbt_exit:
311 mutex_unlock(&hws_sem_oom);
312 return rc;
313
314allocate_sdbt_error:
315 rc = -ENOMEM;
316 goto allocate_sdbt_exit;
317}
318
319/*
320 * deallocate_sdbt() - deallocate all sampler memory
321 *
322 * For each online CPU all SDBT trees are deallocated.
323 * Returns the number of freed pages.
324 */
325static int deallocate_sdbt(void)
326{
327 int cpu;
328 int counter;
329
330 counter = 0;
331
332 for_each_online_cpu(cpu) {
333 unsigned long start;
334 unsigned long sdbt;
335 unsigned long *curr;
336 struct hws_cpu_buffer *cb;
337
338 cb = &per_cpu(sampler_cpu_buffer, cpu);
339
340 if (!cb->first_sdbt)
341 continue;
342
343 sdbt = cb->first_sdbt;
344 curr = (unsigned long *) sdbt;
345 start = sdbt;
346
347 /* we'll free the SDBT after all SDBs are processed... */
348 while (1) {
349 if (!*curr || !sdbt)
350 break;
351
352 /* watch for link entry reset if found */
353 if (is_link_entry(curr)) {
354 curr = get_next_sdbt(curr);
355 if (sdbt)
356 free_page(sdbt);
357
358 /* we are done if we reach the start */
359 if ((unsigned long) curr == start)
360 break;
361 else
362 sdbt = (unsigned long) curr;
363 } else {
364 /* process SDB pointer */
365 if (*curr) {
366 free_page(*curr);
367 curr++;
368 }
369 }
370 counter++;
371 }
372 cb->first_sdbt = 0;
373 }
374 return counter;
375}
376
377static int start_sampling(int cpu)
378{
379 int rc;
380 struct hws_cpu_buffer *cb;
381
382 cb = &per_cpu(sampler_cpu_buffer, cpu);
383 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
384 if (rc) {
385 printk(KERN_INFO "hwsampler: CPU %d ssctl failed.\n", cpu);
386 goto start_exit;
387 }
388
389 rc = -EINVAL;
390 if (!cb->qsi.es) {
391 printk(KERN_INFO "hwsampler: CPU %d ssctl not enabled.\n", cpu);
392 goto start_exit;
393 }
394
395 if (!cb->qsi.cs) {
396 printk(KERN_INFO "hwsampler: CPU %d ssctl not active.\n", cpu);
397 goto start_exit;
398 }
399
400 printk(KERN_INFO
401 "hwsampler: CPU %d, CPUMF Sampling started, interval %lu.\n",
402 cpu, interval);
403
404 rc = 0;
405
406start_exit:
407 return rc;
408}
409
410static int stop_sampling(int cpu)
411{
412 unsigned long v;
413 int rc;
414 struct hws_cpu_buffer *cb;
415
416 rc = smp_ctl_qsi(cpu);
417 WARN_ON(rc);
418
419 cb = &per_cpu(sampler_cpu_buffer, cpu);
420 if (!rc && !cb->qsi.es)
421 printk(KERN_INFO "hwsampler: CPU %d, already stopped.\n", cpu);
422
423 rc = smp_ctl_ssctl_stop(cpu);
424 if (rc) {
425 printk(KERN_INFO "hwsampler: CPU %d, ssctl stop error %d.\n",
426 cpu, rc);
427 goto stop_exit;
428 }
429
430 printk(KERN_INFO "hwsampler: CPU %d, CPUMF Sampling stopped.\n", cpu);
431
432stop_exit:
433 v = cb->req_alert;
434 if (v)
435 printk(KERN_ERR "hwsampler: CPU %d CPUMF Request alert,"
436 " count=%lu.\n", cpu, v);
437
438 v = cb->loss_of_sample_data;
439 if (v)
440 printk(KERN_ERR "hwsampler: CPU %d CPUMF Loss of sample data,"
441 " count=%lu.\n", cpu, v);
442
443 v = cb->invalid_entry_address;
444 if (v)
445 printk(KERN_ERR "hwsampler: CPU %d CPUMF Invalid entry address,"
446 " count=%lu.\n", cpu, v);
447
448 v = cb->incorrect_sdbt_entry;
449 if (v)
450 printk(KERN_ERR
451 "hwsampler: CPU %d CPUMF Incorrect SDBT address,"
452 " count=%lu.\n", cpu, v);
453
454 v = cb->sample_auth_change_alert;
455 if (v)
456 printk(KERN_ERR
457 "hwsampler: CPU %d CPUMF Sample authorization change,"
458 " count=%lu.\n", cpu, v);
459
460 return rc;
461}
462
463static int check_hardware_prerequisites(void)
464{
65a94b14 465 if (!test_facility(68))
ec6a3df1 466 return -EOPNOTSUPP;
ec6a3df1
HG		 467 return 0;
468}
469/*
470 * hws_oom_callback() - the OOM callback function
471 *
472 * In case the callback is invoked during memory allocation for the
473 * hw sampler, all obtained memory is deallocated and a flag is set
474 * so main sampler memory allocation can exit with a failure code.
475 * In case the callback is invoked during sampling the hw sampler
476 * is deactivated for all CPUs.
477 */
478static int hws_oom_callback(struct notifier_block *nfb,
479 unsigned long dummy, void *parm)
480{
481 unsigned long *freed;
482 int cpu;
483 struct hws_cpu_buffer *cb;
484
485 freed = parm;
486
487 mutex_lock(&hws_sem_oom);
488
489 if (hws_state == HWS_DEALLOCATED) {
490 /* during memory allocation */
491 if (oom_killer_was_active == 0) {
492 oom_killer_was_active = 1;
493 *freed += deallocate_sdbt();
494 }
495 } else {
496 int i;
497 cpu = get_cpu();
498 cb = &per_cpu(sampler_cpu_buffer, cpu);
499
500 if (!cb->oom) {
501 for_each_online_cpu(i) {
502 smp_ctl_ssctl_deactivate(i);
503 cb->oom = 1;
504 }
505 cb->finish = 1;
506
507 printk(KERN_INFO
508 "hwsampler: CPU %d, OOM notify during CPUMF Sampling.\n",
509 cpu);
510 }
511 }
512
513 mutex_unlock(&hws_sem_oom);
514
515 return NOTIFY_OK;
516}
517
518static struct notifier_block hws_oom_notifier = {
519 .notifier_call = hws_oom_callback
520};
521
7bb2e269 522static int hws_cpu_callback(struct notifier_block *nfb,
ec6a3df1
HG		 523 unsigned long action, void *hcpu)
524{
525 /* We do not have sampler space available for all possible CPUs.
526 All CPUs should be online when hw sampling is activated. */
b07c9015 527 return (hws_state <= HWS_DEALLOCATED) ? NOTIFY_OK : NOTIFY_BAD;
ec6a3df1
HG		 528}
529
530static struct notifier_block hws_cpu_notifier = {
531 .notifier_call = hws_cpu_callback
532};
533
534/**
535 * hwsampler_deactivate() - set hardware sampling temporarily inactive
536 * @cpu: specifies the CPU to be set inactive.
537 *
538 * Returns 0 on success, !0 on failure.
539 */
540int hwsampler_deactivate(unsigned int cpu)
541{
542 /*
543 * Deactivate hw sampling temporarily and flush the buffer
544 * by pushing all the pending samples to oprofile buffer.
545 *
546 * This function can be called under one of the following conditions:
547 * Memory unmap, task is exiting.
548 */
549 int rc;
550 struct hws_cpu_buffer *cb;
551
552 rc = 0;
553 mutex_lock(&hws_sem);
554
555 cb = &per_cpu(sampler_cpu_buffer, cpu);
556 if (hws_state == HWS_STARTED) {
557 rc = smp_ctl_qsi(cpu);
558 WARN_ON(rc);
559 if (cb->qsi.cs) {
560 rc = smp_ctl_ssctl_deactivate(cpu);
561 if (rc) {
562 printk(KERN_INFO
563 "hwsampler: CPU %d, CPUMF Deactivation failed.\n", cpu);
564 cb->finish = 1;
565 hws_state = HWS_STOPPING;
566 } else {
567 hws_flush_all = 1;
568 /* Add work to queue to read pending samples.*/
569 queue_work_on(cpu, hws_wq, &cb->worker);
570 }
571 }
572 }
573 mutex_unlock(&hws_sem);
574
575 if (hws_wq)
576 flush_workqueue(hws_wq);
577
578 return rc;
579}
580
581/**
582 * hwsampler_activate() - activate/resume hardware sampling which was deactivated
583 * @cpu: specifies the CPU to be set active.
584 *
585 * Returns 0 on success, !0 on failure.
586 */
587int hwsampler_activate(unsigned int cpu)
588{
589 /*
590 * Re-activate hw sampling. This should be called in pair with
591 * hwsampler_deactivate().
592 */
593 int rc;
594 struct hws_cpu_buffer *cb;
595
596 rc = 0;
597 mutex_lock(&hws_sem);
598
599 cb = &per_cpu(sampler_cpu_buffer, cpu);
600 if (hws_state == HWS_STARTED) {
601 rc = smp_ctl_qsi(cpu);
602 WARN_ON(rc);
603 if (!cb->qsi.cs) {
604 hws_flush_all = 0;
605 rc = smp_ctl_ssctl_enable_activate(cpu, interval);
606 if (rc) {
607 printk(KERN_ERR
608 "CPU %d, CPUMF activate sampling failed.\n",
609 cpu);
610 }
611 }
612 }
613
614 mutex_unlock(&hws_sem);
615
616 return rc;
617}
618
ec6a3df1
HG		 619static int check_qsi_on_setup(void)
620{
621 int rc;
622 unsigned int cpu;
623 struct hws_cpu_buffer *cb;
624
625 for_each_online_cpu(cpu) {
626 cb = &per_cpu(sampler_cpu_buffer, cpu);
627 rc = smp_ctl_qsi(cpu);
628 WARN_ON(rc);
629 if (rc)
630 return -EOPNOTSUPP;
631
632 if (!cb->qsi.as) {
633 printk(KERN_INFO "hwsampler: CPUMF sampling is not authorized.\n");
634 return -EINVAL;
635 }
636
637 if (cb->qsi.es) {
638 printk(KERN_WARNING "hwsampler: CPUMF is still enabled.\n");
639 rc = smp_ctl_ssctl_stop(cpu);
640 if (rc)
641 return -EINVAL;
642
643 printk(KERN_INFO
644 "CPU %d, CPUMF Sampling stopped now.\n", cpu);
645 }
646 }
647 return 0;
648}
649
650static int check_qsi_on_start(void)
651{
652 unsigned int cpu;
653 int rc;
654 struct hws_cpu_buffer *cb;
655
656 for_each_online_cpu(cpu) {
657 cb = &per_cpu(sampler_cpu_buffer, cpu);
658 rc = smp_ctl_qsi(cpu);
659 WARN_ON(rc);
660
661 if (!cb->qsi.as)
662 return -EINVAL;
663
664 if (cb->qsi.es)
665 return -EINVAL;
666
667 if (cb->qsi.cs)
668 return -EINVAL;
669 }
670 return 0;
671}
672
673static void worker_on_start(unsigned int cpu)
674{
675 struct hws_cpu_buffer *cb;
676
677 cb = &per_cpu(sampler_cpu_buffer, cpu);
678 cb->worker_entry = cb->first_sdbt;
679}
680
681static int worker_check_error(unsigned int cpu, int ext_params)
682{
683 int rc;
684 unsigned long *sdbt;
685 struct hws_cpu_buffer *cb;
686
687 rc = 0;
688 cb = &per_cpu(sampler_cpu_buffer, cpu);
689 sdbt = (unsigned long *) cb->worker_entry;
690
691 if (!sdbt || !*sdbt)
692 return -EINVAL;
693
b03d541a 694 if (ext_params & CPU_MF_INT_SF_PRA)
ec6a3df1
HG		 695 cb->req_alert++;
696
b03d541a 697 if (ext_params & CPU_MF_INT_SF_LSDA)
ec6a3df1
HG		 698 cb->loss_of_sample_data++;
699
b03d541a 700 if (ext_params & CPU_MF_INT_SF_IAE) {
ec6a3df1
HG		 701 cb->invalid_entry_address++;
702 rc = -EINVAL;
703 }
704
b03d541a 705 if (ext_params & CPU_MF_INT_SF_ISE) {
ec6a3df1
HG		 706 cb->incorrect_sdbt_entry++;
707 rc = -EINVAL;
708 }
709
b03d541a 710 if (ext_params & CPU_MF_INT_SF_SACA) {
ec6a3df1
HG		 711 cb->sample_auth_change_alert++;
712 rc = -EINVAL;
713 }
714
715 return rc;
716}
717
718static void worker_on_finish(unsigned int cpu)
719{
720 int rc, i;
721 struct hws_cpu_buffer *cb;
722
723 cb = &per_cpu(sampler_cpu_buffer, cpu);
724
725 if (cb->finish) {
726 rc = smp_ctl_qsi(cpu);
727 WARN_ON(rc);
728 if (cb->qsi.es) {
729 printk(KERN_INFO
730 "hwsampler: CPU %d, CPUMF Stop/Deactivate sampling.\n",
731 cpu);
732 rc = smp_ctl_ssctl_stop(cpu);
733 if (rc)
734 printk(KERN_INFO
735 "hwsampler: CPU %d, CPUMF Deactivation failed.\n",
736 cpu);
737
738 for_each_online_cpu(i) {
739 if (i == cpu)
740 continue;
741 if (!cb->finish) {
742 cb->finish = 1;
743 queue_work_on(i, hws_wq,
744 &cb->worker);
745 }
746 }
747 }
748 }
749}
750
751static void worker_on_interrupt(unsigned int cpu)
752{
753 unsigned long *sdbt;
754 unsigned char done;
755 struct hws_cpu_buffer *cb;
756
757 cb = &per_cpu(sampler_cpu_buffer, cpu);
758
759 sdbt = (unsigned long *) cb->worker_entry;
760
761 done = 0;
762 /* do not proceed if stop was entered,
763 * forget the buffers not yet processed */
764 while (!done && !cb->stop_mode) {
765 unsigned long *trailer;
766 struct hws_trailer_entry *te;
767 unsigned long *dear = 0;
768
769 trailer = trailer_entry_ptr(*sdbt);
770 /* leave loop if no more work to do */
cf48ad83 771 if (!(*trailer & SDB_TE_BUFFER_FULL_MASK)) {
ec6a3df1
HG		 772 done = 1;
773 if (!hws_flush_all)
774 continue;
775 }
776
777 te = (struct hws_trailer_entry *)trailer;
778 cb->sample_overflow += te->overflow;
779
780 add_samples_to_oprofile(cpu, sdbt, dear);
781
782 /* reset trailer */
783 xchg((unsigned char *) te, 0x40);
784
785 /* advance to next sdb slot in current sdbt */
786 sdbt++;
787 /* in case link bit is set use address w/o link bit */
788 if (is_link_entry(sdbt))
789 sdbt = get_next_sdbt(sdbt);
790
791 cb->worker_entry = (unsigned long)sdbt;
792 }
793}
794
795static void add_samples_to_oprofile(unsigned int cpu, unsigned long *sdbt,
796 unsigned long *dear)
797{
7e75fc3f 798 struct hws_basic_entry *sample_data_ptr;
ec6a3df1
HG		 799 unsigned long *trailer;
800
801 trailer = trailer_entry_ptr(*sdbt);
802 if (dear) {
803 if (dear > trailer)
804 return;
805 trailer = dear;
806 }
807
7e75fc3f 808 sample_data_ptr = (struct hws_basic_entry *)(*sdbt);
ec6a3df1
HG		 809
810 while ((unsigned long *)sample_data_ptr < trailer) {
811 struct pt_regs *regs = NULL;
812 struct task_struct *tsk = NULL;
813
814 /*
815 * Check sampling mode, 1 indicates basic (=customer) sampling
816 * mode.
817 */
818 if (sample_data_ptr->def != 1) {
819 /* sample slot is not yet written */
820 break;
821 } else {
822 /* make sure we don't use it twice,
823 * the next time the sampler will set it again */
824 sample_data_ptr->def = 0;
825 }
826
827 /* Get pt_regs. */
828 if (sample_data_ptr->P == 1) {
829 /* userspace sample */
830 unsigned int pid = sample_data_ptr->prim_asn;
dd3c4670
AK		 831 if (!counter_config.user)
832 goto skip_sample;
ec6a3df1
HG		 833 rcu_read_lock();
834 tsk = pid_task(find_vpid(pid), PIDTYPE_PID);
835 if (tsk)
836 regs = task_pt_regs(tsk);
837 rcu_read_unlock();
838 } else {
839 /* kernelspace sample */
dd3c4670
AK		 840 if (!counter_config.kernel)
841 goto skip_sample;
ec6a3df1
HG		 842 regs = task_pt_regs(current);
843 }
844
845 mutex_lock(&hws_sem);
846 oprofile_add_ext_hw_sample(sample_data_ptr->ia, regs, 0,
847 !sample_data_ptr->P, tsk);
848 mutex_unlock(&hws_sem);
dd3c4670 849 skip_sample:
ec6a3df1
HG		 850 sample_data_ptr++;
851 }
852}
853
854static void worker(struct work_struct *work)
855{
856 unsigned int cpu;
857 int ext_params;
858 struct hws_cpu_buffer *cb;
859
860 cb = container_of(work, struct hws_cpu_buffer, worker);
861 cpu = smp_processor_id();
862 ext_params = atomic_xchg(&cb->ext_params, 0);
863
864 if (!cb->worker_entry)
865 worker_on_start(cpu);
866
867 if (worker_check_error(cpu, ext_params))
868 return;
869
870 if (!cb->finish)
871 worker_on_interrupt(cpu);
872
873 if (cb->finish)
874 worker_on_finish(cpu);
875}
876
877/**
878 * hwsampler_allocate() - allocate memory for the hardware sampler
879 * @sdbt: number of SDBTs per online CPU (must be > 0)
880 * @sdb: number of SDBs per SDBT (minimum 1, maximum 511)
881 *
882 * Returns 0 on success, !0 on failure.
883 */
884int hwsampler_allocate(unsigned long sdbt, unsigned long sdb)
885{
886 int cpu, rc;
887 mutex_lock(&hws_sem);
888
889 rc = -EINVAL;
890 if (hws_state != HWS_DEALLOCATED)
891 goto allocate_exit;
892
893 if (sdbt < 1)
894 goto allocate_exit;
895
896 if (sdb > MAX_NUM_SDB || sdb < MIN_NUM_SDB)
897 goto allocate_exit;
898
899 num_sdbt = sdbt;
900 num_sdb = sdb;
901
902 oom_killer_was_active = 0;
903 register_oom_notifier(&hws_oom_notifier);
904
905 for_each_online_cpu(cpu) {
906 if (allocate_sdbt(cpu)) {
907 unregister_oom_notifier(&hws_oom_notifier);
908 goto allocate_error;
909 }
910 }
911 unregister_oom_notifier(&hws_oom_notifier);
912 if (oom_killer_was_active)
913 goto allocate_error;
914
915 hws_state = HWS_STOPPED;
916 rc = 0;
917
918allocate_exit:
919 mutex_unlock(&hws_sem);
920 return rc;
921
922allocate_error:
923 rc = -ENOMEM;
924 printk(KERN_ERR "hwsampler: CPUMF Memory allocation failed.\n");
925 goto allocate_exit;
926}
927
928/**
929 * hwsampler_deallocate() - deallocate hardware sampler memory
930 *
931 * Returns 0 on success, !0 on failure.
932 */
e54aafa0 933int hwsampler_deallocate(void)
ec6a3df1
HG		 934{
935 int rc;
936
937 mutex_lock(&hws_sem);
938
939 rc = -EINVAL;
940 if (hws_state != HWS_STOPPED)
941 goto deallocate_exit;
942
82003c3e 943 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
e28bb79d 944 hws_alert = 0;
ec6a3df1
HG		 945 deallocate_sdbt();
946
947 hws_state = HWS_DEALLOCATED;
948 rc = 0;
949
950deallocate_exit:
951 mutex_unlock(&hws_sem);
952
953 return rc;
954}
955
3d8dcb3c 956unsigned long hwsampler_query_min_interval(void)
ec6a3df1 957{
3d8dcb3c 958 return min_sampler_rate;
ec6a3df1
HG		 959}
960
3d8dcb3c 961unsigned long hwsampler_query_max_interval(void)
ec6a3df1 962{
3d8dcb3c 963 return max_sampler_rate;
ec6a3df1
HG		 964}
965
966unsigned long hwsampler_get_sample_overflow_count(unsigned int cpu)
967{
968 struct hws_cpu_buffer *cb;
969
970 cb = &per_cpu(sampler_cpu_buffer, cpu);
971
972 return cb->sample_overflow;
973}
974
e54aafa0 975int hwsampler_setup(void)
ec6a3df1
HG		 976{
977 int rc;
978 int cpu;
979 struct hws_cpu_buffer *cb;
980
981 mutex_lock(&hws_sem);
982
983 rc = -EINVAL;
984 if (hws_state)
985 goto setup_exit;
986
987 hws_state = HWS_INIT;
988
989 init_all_cpu_buffers();
990
991 rc = check_hardware_prerequisites();
992 if (rc)
993 goto setup_exit;
994
995 rc = check_qsi_on_setup();
996 if (rc)
997 goto setup_exit;
998
999 rc = -EINVAL;
1000 hws_wq = create_workqueue("hwsampler");
1001 if (!hws_wq)
1002 goto setup_exit;
1003
1004 register_cpu_notifier(&hws_cpu_notifier);
1005
1006 for_each_online_cpu(cpu) {
1007 cb = &per_cpu(sampler_cpu_buffer, cpu);
1008 INIT_WORK(&cb->worker, worker);
1009 rc = smp_ctl_qsi(cpu);
1010 WARN_ON(rc);
1011 if (min_sampler_rate != cb->qsi.min_sampl_rate) {
1012 if (min_sampler_rate) {
1013 printk(KERN_WARNING
1014 "hwsampler: different min sampler rate values.\n");
1015 if (min_sampler_rate < cb->qsi.min_sampl_rate)
1016 min_sampler_rate =
1017 cb->qsi.min_sampl_rate;
1018 } else
1019 min_sampler_rate = cb->qsi.min_sampl_rate;
1020 }
1021 if (max_sampler_rate != cb->qsi.max_sampl_rate) {
1022 if (max_sampler_rate) {
1023 printk(KERN_WARNING
1024 "hwsampler: different max sampler rate values.\n");
1025 if (max_sampler_rate > cb->qsi.max_sampl_rate)
1026 max_sampler_rate =
1027 cb->qsi.max_sampl_rate;
1028 } else
1029 max_sampler_rate = cb->qsi.max_sampl_rate;
1030 }
1031 }
1dad093b 1032 register_external_irq(EXT_IRQ_MEASURE_ALERT, hws_ext_handler);
ec6a3df1
HG		 1033
1034 hws_state = HWS_DEALLOCATED;
1035 rc = 0;
1036
1037setup_exit:
1038 mutex_unlock(&hws_sem);
1039 return rc;
1040}
1041
e54aafa0 1042int hwsampler_shutdown(void)
ec6a3df1
HG		 1043{
1044 int rc;
1045
1046 mutex_lock(&hws_sem);
1047
1048 rc = -EINVAL;
1049 if (hws_state == HWS_DEALLOCATED || hws_state == HWS_STOPPED) {
1050 mutex_unlock(&hws_sem);
1051
1052 if (hws_wq)
1053 flush_workqueue(hws_wq);
1054
1055 mutex_lock(&hws_sem);
1056
1057 if (hws_state == HWS_STOPPED) {
82003c3e 1058 irq_subclass_unregister(IRQ_SUBCLASS_MEASUREMENT_ALERT);
e28bb79d 1059 hws_alert = 0;
ec6a3df1
HG		 1060 deallocate_sdbt();
1061 }
1062 if (hws_wq) {
1063 destroy_workqueue(hws_wq);
1064 hws_wq = NULL;
1065 }
1066
1dad093b 1067 unregister_external_irq(EXT_IRQ_MEASURE_ALERT, hws_ext_handler);
ec6a3df1
HG		 1068 hws_state = HWS_INIT;
1069 rc = 0;
1070 }
1071 mutex_unlock(&hws_sem);
1072
1073 unregister_cpu_notifier(&hws_cpu_notifier);
1074
1075 return rc;
1076}
1077
1078/**
1079 * hwsampler_start_all() - start hardware sampling on all online CPUs
1080 * @rate: specifies the used interval when samples are taken
1081 *
1082 * Returns 0 on success, !0 on failure.
1083 */
1084int hwsampler_start_all(unsigned long rate)
1085{
1086 int rc, cpu;
1087
1088 mutex_lock(&hws_sem);
1089
1090 hws_oom = 0;
1091
1092 rc = -EINVAL;
1093 if (hws_state != HWS_STOPPED)
1094 goto start_all_exit;
1095
1096 interval = rate;
1097
1098 /* fail if rate is not valid */
1099 if (interval < min_sampler_rate || interval > max_sampler_rate)
1100 goto start_all_exit;
1101
1102 rc = check_qsi_on_start();
1103 if (rc)
1104 goto start_all_exit;
1105
94414ca0 1106 prepare_cpu_buffers();
ec6a3df1
HG		 1107
1108 for_each_online_cpu(cpu) {
1109 rc = start_sampling(cpu);
1110 if (rc)
1111 break;
1112 }
1113 if (rc) {
1114 for_each_online_cpu(cpu) {
1115 stop_sampling(cpu);
1116 }
1117 goto start_all_exit;
1118 }
1119 hws_state = HWS_STARTED;
1120 rc = 0;
1121
1122start_all_exit:
1123 mutex_unlock(&hws_sem);
1124
1125 if (rc)
1126 return rc;
1127
1128 register_oom_notifier(&hws_oom_notifier);
1129 hws_oom = 1;
1130 hws_flush_all = 0;
1131 /* now let them in, 1407 CPUMF external interrupts */
e28bb79d 1132 hws_alert = 1;
82003c3e 1133 irq_subclass_register(IRQ_SUBCLASS_MEASUREMENT_ALERT);
ec6a3df1
HG		 1134
1135 return 0;
1136}
1137
1138/**
1139 * hwsampler_stop_all() - stop hardware sampling on all online CPUs
1140 *
1141 * Returns 0 on success, !0 on failure.
1142 */
e54aafa0 1143int hwsampler_stop_all(void)
ec6a3df1
HG		 1144{
1145 int tmp_rc, rc, cpu;
1146 struct hws_cpu_buffer *cb;
1147
1148 mutex_lock(&hws_sem);
1149
1150 rc = 0;
1151 if (hws_state == HWS_INIT) {
1152 mutex_unlock(&hws_sem);
94414ca0 1153 return 0;
ec6a3df1
HG		 1154 }
1155 hws_state = HWS_STOPPING;
1156 mutex_unlock(&hws_sem);
1157
1158 for_each_online_cpu(cpu) {
1159 cb = &per_cpu(sampler_cpu_buffer, cpu);
1160 cb->stop_mode = 1;
1161 tmp_rc = stop_sampling(cpu);
1162 if (tmp_rc)
1163 rc = tmp_rc;
1164 }
1165
1166 if (hws_wq)
1167 flush_workqueue(hws_wq);
1168
1169 mutex_lock(&hws_sem);
1170 if (hws_oom) {
1171 unregister_oom_notifier(&hws_oom_notifier);
1172 hws_oom = 0;
1173 }
1174 hws_state = HWS_STOPPED;
1175 mutex_unlock(&hws_sem);
1176
1177 return rc;
1178}
Linux kernel - Deliverables
This page took 0.230541 seconds and 5 git commands to generate.