2 * Cell Broadband Engine OProfile Support
4 * (C) Copyright IBM Corporation 2006
6 * Author: David Erb (djerb@us.ibm.com)
8 * Carl Love <carll@us.ibm.com>
9 * Maynard Johnson <maynardj@us.ibm.com>
11 * This program is free software; you can redistribute it and/or
12 * modify it under the terms of the GNU General Public License
13 * as published by the Free Software Foundation; either version
14 * 2 of the License, or (at your option) any later version.
17 #include <linux/cpufreq.h>
18 #include <linux/delay.h>
19 #include <linux/init.h>
20 #include <linux/jiffies.h>
21 #include <linux/kthread.h>
22 #include <linux/oprofile.h>
23 #include <linux/percpu.h>
24 #include <linux/smp.h>
25 #include <linux/spinlock.h>
26 #include <linux/timer.h>
27 #include <asm/cell-pmu.h>
28 #include <asm/cputable.h>
29 #include <asm/firmware.h>
31 #include <asm/oprofile_impl.h>
32 #include <asm/processor.h>
34 #include <asm/ptrace.h>
37 #include <asm/system.h>
39 #include "../platforms/cell/interrupt.h"
41 #define PPU_CYCLES_EVENT_NUM 1 /* event number for CYCLES */
42 #define PPU_CYCLES_GRP_NUM 1 /* special group number for identifying
45 #define CBE_COUNT_ALL_CYCLES 0x42800000 /* PPU cycle event specifier */
47 #define NUM_THREADS 2 /* number of physical threads in
50 #define NUM_TRACE_BUS_WORDS 4
51 #define NUM_INPUT_BUS_WORDS 2
54 struct pmc_cntrl_data
{
58 unsigned long enabled
;
62 * ibm,cbe-perftools rtas parameters
66 u16 cpu
; /* Processor to modify */
67 u16 sub_unit
; /* hw subunit this applies to (if applicable) */
68 short int signal_group
; /* Signal Group to Enable/Disable */
69 u8 bus_word
; /* Enable/Disable on this Trace/Trigger/Event
70 * Bus Word(s) (bitmask)
72 u8 bit
; /* Trigger/Event bit (if applicable) */
81 SUBFUNC_DEACTIVATE
= 3,
98 u32 debug_bus_control
;
99 struct pm_cntrl pm_cntrl
;
100 u32 pm07_cntrl
[NR_PHYS_CTRS
];
103 #define GET_SUB_UNIT(x) ((x & 0x0000f000) >> 12)
104 #define GET_BUS_WORD(x) ((x & 0x000000f0) >> 4)
105 #define GET_BUS_TYPE(x) ((x & 0x00000300) >> 8)
106 #define GET_POLARITY(x) ((x & 0x00000002) >> 1)
107 #define GET_COUNT_CYCLES(x) (x & 0x00000001)
108 #define GET_INPUT_CONTROL(x) ((x & 0x00000004) >> 2)
110 static DEFINE_PER_CPU(unsigned long[NR_PHYS_CTRS
], pmc_values
);
112 static struct pmc_cntrl_data pmc_cntrl
[NUM_THREADS
][NR_PHYS_CTRS
];
114 /* Interpetation of hdw_thread:
115 * 0 - even virtual cpus 0, 2, 4,...
116 * 1 - odd virtual cpus 1, 3, 5, ...
118 static u32 hdw_thread
;
120 static u32 virt_cntr_inter_mask
;
121 static struct timer_list timer_virt_cntr
;
123 /* pm_signal needs to be global since it is initialized in
124 * cell_reg_setup at the time when the necessary information
127 static struct pm_signal pm_signal
[NR_PHYS_CTRS
];
128 static int pm_rtas_token
;
130 static u32 reset_value
[NR_PHYS_CTRS
];
131 static int num_counters
;
132 static int oprofile_running
;
133 static spinlock_t virt_cntr_lock
= SPIN_LOCK_UNLOCKED
;
135 static u32 ctr_enabled
;
137 static unsigned char trace_bus
[NUM_TRACE_BUS_WORDS
];
138 static unsigned char input_bus
[NUM_INPUT_BUS_WORDS
];
141 * Firmware interface functions
144 rtas_ibm_cbe_perftools(int subfunc
, int passthru
,
145 void *address
, unsigned long length
)
147 u64 paddr
= __pa(address
);
149 return rtas_call(pm_rtas_token
, 5, 1, NULL
, subfunc
, passthru
,
150 paddr
>> 32, paddr
& 0xffffffff, length
);
153 static void pm_rtas_reset_signals(u32 node
)
156 struct pm_signal pm_signal_local
;
158 /* The debug bus is being set to the passthru disable state.
159 * However, the FW still expects atleast one legal signal routing
160 * entry or it will return an error on the arguments. If we don't
161 * supply a valid entry, we must ignore all return values. Ignoring
162 * all return values means we might miss an error we should be
166 /* fw expects physical cpu #. */
167 pm_signal_local
.cpu
= node
;
168 pm_signal_local
.signal_group
= 21;
169 pm_signal_local
.bus_word
= 1;
170 pm_signal_local
.sub_unit
= 0;
171 pm_signal_local
.bit
= 0;
173 ret
= rtas_ibm_cbe_perftools(SUBFUNC_RESET
, PASSTHRU_DISABLE
,
175 sizeof(struct pm_signal
));
178 printk(KERN_WARNING
"%s: rtas returned: %d\n",
182 static void pm_rtas_activate_signals(u32 node
, u32 count
)
186 struct pm_signal pm_signal_local
[NR_PHYS_CTRS
];
188 /* There is no debug setup required for the cycles event.
189 * Note that only events in the same group can be used.
190 * Otherwise, there will be conflicts in correctly routing
191 * the signals on the debug bus. It is the responsiblity
192 * of the OProfile user tool to check the events are in
196 for (j
= 0; j
< count
; j
++) {
197 if (pm_signal
[j
].signal_group
!= PPU_CYCLES_GRP_NUM
) {
199 /* fw expects physical cpu # */
200 pm_signal_local
[i
].cpu
= node
;
201 pm_signal_local
[i
].signal_group
202 = pm_signal
[j
].signal_group
;
203 pm_signal_local
[i
].bus_word
= pm_signal
[j
].bus_word
;
204 pm_signal_local
[i
].sub_unit
= pm_signal
[j
].sub_unit
;
205 pm_signal_local
[i
].bit
= pm_signal
[j
].bit
;
211 ret
= rtas_ibm_cbe_perftools(SUBFUNC_ACTIVATE
, PASSTHRU_ENABLE
,
213 i
* sizeof(struct pm_signal
));
216 printk(KERN_WARNING
"%s: rtas returned: %d\n",
222 * PM Signal functions
224 static void set_pm_event(u32 ctr
, int event
, u32 unit_mask
)
228 u32 bus_word
, bus_type
, count_cycles
, polarity
, input_control
;
231 if (event
== PPU_CYCLES_EVENT_NUM
) {
232 /* Special Event: Count all cpu cycles */
233 pm_regs
.pm07_cntrl
[ctr
] = CBE_COUNT_ALL_CYCLES
;
234 p
= &(pm_signal
[ctr
]);
235 p
->signal_group
= PPU_CYCLES_GRP_NUM
;
241 pm_regs
.pm07_cntrl
[ctr
] = 0;
244 bus_word
= GET_BUS_WORD(unit_mask
);
245 bus_type
= GET_BUS_TYPE(unit_mask
);
246 count_cycles
= GET_COUNT_CYCLES(unit_mask
);
247 polarity
= GET_POLARITY(unit_mask
);
248 input_control
= GET_INPUT_CONTROL(unit_mask
);
249 signal_bit
= (event
% 100);
251 p
= &(pm_signal
[ctr
]);
253 p
->signal_group
= event
/ 100;
254 p
->bus_word
= bus_word
;
255 p
->sub_unit
= (unit_mask
& 0x0000f000) >> 12;
257 pm_regs
.pm07_cntrl
[ctr
] = 0;
258 pm_regs
.pm07_cntrl
[ctr
] |= PM07_CTR_COUNT_CYCLES(count_cycles
);
259 pm_regs
.pm07_cntrl
[ctr
] |= PM07_CTR_POLARITY(polarity
);
260 pm_regs
.pm07_cntrl
[ctr
] |= PM07_CTR_INPUT_CONTROL(input_control
);
262 /* Some of the islands signal selection is based on 64 bit words.
263 * The debug bus words are 32 bits, the input words to the performance
264 * counters are defined as 32 bits. Need to convert the 64 bit island
265 * specification to the appropriate 32 input bit and bus word for the
266 * performance counter event selection. See the CELL Performance
267 * monitoring signals manual and the Perf cntr hardware descriptions
270 if (input_control
== 0) {
271 if (signal_bit
> 31) {
275 else if (bus_word
== 0xc)
279 if ((bus_type
== 0) && p
->signal_group
>= 60)
281 if ((bus_type
== 1) && p
->signal_group
>= 50)
284 pm_regs
.pm07_cntrl
[ctr
] |= PM07_CTR_INPUT_MUX(signal_bit
);
286 pm_regs
.pm07_cntrl
[ctr
] = 0;
290 for (i
= 0; i
< NUM_TRACE_BUS_WORDS
; i
++) {
291 if (bus_word
& (1 << i
)) {
292 pm_regs
.debug_bus_control
|=
293 (bus_type
<< (31 - (2 * i
) + 1));
295 for (j
= 0; j
< NUM_INPUT_BUS_WORDS
; j
++) {
296 if (input_bus
[j
] == 0xff) {
298 pm_regs
.group_control
|=
309 static void write_pm_cntrl(int cpu
)
311 /* Oprofile will use 32 bit counters, set bits 7:10 to 0
312 * pmregs.pm_cntrl is a global
316 if (pm_regs
.pm_cntrl
.enable
== 1)
317 val
|= CBE_PM_ENABLE_PERF_MON
;
319 if (pm_regs
.pm_cntrl
.stop_at_max
== 1)
320 val
|= CBE_PM_STOP_AT_MAX
;
322 if (pm_regs
.pm_cntrl
.trace_mode
== 1)
323 val
|= CBE_PM_TRACE_MODE_SET(pm_regs
.pm_cntrl
.trace_mode
);
325 if (pm_regs
.pm_cntrl
.freeze
== 1)
326 val
|= CBE_PM_FREEZE_ALL_CTRS
;
328 /* Routine set_count_mode must be called previously to set
329 * the count mode based on the user selection of user and kernel.
331 val
|= CBE_PM_COUNT_MODE_SET(pm_regs
.pm_cntrl
.count_mode
);
332 cbe_write_pm(cpu
, pm_control
, val
);
336 set_count_mode(u32 kernel
, u32 user
)
338 /* The user must specify user and kernel if they want them. If
339 * neither is specified, OProfile will count in hypervisor mode.
340 * pm_regs.pm_cntrl is a global
344 pm_regs
.pm_cntrl
.count_mode
= CBE_COUNT_ALL_MODES
;
346 pm_regs
.pm_cntrl
.count_mode
=
347 CBE_COUNT_SUPERVISOR_MODE
;
350 pm_regs
.pm_cntrl
.count_mode
= CBE_COUNT_PROBLEM_MODE
;
352 pm_regs
.pm_cntrl
.count_mode
=
353 CBE_COUNT_HYPERVISOR_MODE
;
357 static inline void enable_ctr(u32 cpu
, u32 ctr
, u32
* pm07_cntrl
)
360 pm07_cntrl
[ctr
] |= CBE_PM_CTR_ENABLE
;
361 cbe_write_pm07_control(cpu
, ctr
, pm07_cntrl
[ctr
]);
365 * Oprofile is expected to collect data on all CPUs simultaneously.
366 * However, there is one set of performance counters per node. There are
367 * two hardware threads or virtual CPUs on each node. Hence, OProfile must
368 * multiplex in time the performance counter collection on the two virtual
369 * CPUs. The multiplexing of the performance counters is done by this
370 * virtual counter routine.
372 * The pmc_values used below is defined as 'per-cpu' but its use is
373 * more akin to 'per-node'. We need to store two sets of counter
374 * values per node -- one for the previous run and one for the next.
375 * The per-cpu[NR_PHYS_CTRS] gives us the storage we need. Each odd/even
376 * pair of per-cpu arrays is used for storing the previous and next
377 * pmc values for a given node.
378 * NOTE: We use the per-cpu variable to improve cache performance.
380 static void cell_virtual_cntr(unsigned long data
)
382 /* This routine will alternate loading the virtual counters for
385 int i
, prev_hdw_thread
, next_hdw_thread
;
389 /* Make sure that the interrupt_hander and
390 * the virt counter are not both playing with
391 * the counters on the same node.
394 spin_lock_irqsave(&virt_cntr_lock
, flags
);
396 prev_hdw_thread
= hdw_thread
;
398 /* switch the cpu handling the interrupts */
399 hdw_thread
= 1 ^ hdw_thread
;
400 next_hdw_thread
= hdw_thread
;
402 for (i
= 0; i
< num_counters
; i
++)
403 /* There are some per thread events. Must do the
404 * set event, for the thread that is being started
407 pmc_cntrl
[next_hdw_thread
][i
].evnts
,
408 pmc_cntrl
[next_hdw_thread
][i
].masks
);
410 /* The following is done only once per each node, but
411 * we need cpu #, not node #, to pass to the cbe_xxx functions.
413 for_each_online_cpu(cpu
) {
414 if (cbe_get_hw_thread_id(cpu
))
417 /* stop counters, save counter values, restore counts
418 * for previous thread
421 cbe_disable_pm_interrupts(cpu
);
422 for (i
= 0; i
< num_counters
; i
++) {
423 per_cpu(pmc_values
, cpu
+ prev_hdw_thread
)[i
]
424 = cbe_read_ctr(cpu
, i
);
426 if (per_cpu(pmc_values
, cpu
+ next_hdw_thread
)[i
]
428 /* If the cntr value is 0xffffffff, we must
429 * reset that to 0xfffffff0 when the current
430 * thread is restarted. This will generate a
431 * new interrupt and make sure that we never
432 * restore the counters to the max value. If
433 * the counters were restored to the max value,
434 * they do not increment and no interrupts are
435 * generated. Hence no more samples will be
436 * collected on that cpu.
438 cbe_write_ctr(cpu
, i
, 0xFFFFFFF0);
440 cbe_write_ctr(cpu
, i
,
443 next_hdw_thread
)[i
]);
446 /* Switch to the other thread. Change the interrupt
447 * and control regs to be scheduled on the CPU
448 * corresponding to the thread to execute.
450 for (i
= 0; i
< num_counters
; i
++) {
451 if (pmc_cntrl
[next_hdw_thread
][i
].enabled
) {
452 /* There are some per thread events.
453 * Must do the set event, enable_cntr
459 cbe_write_pm07_control(cpu
, i
, 0);
463 /* Enable interrupts on the CPU thread that is starting */
464 cbe_enable_pm_interrupts(cpu
, next_hdw_thread
,
465 virt_cntr_inter_mask
);
469 spin_unlock_irqrestore(&virt_cntr_lock
, flags
);
471 mod_timer(&timer_virt_cntr
, jiffies
+ HZ
/ 10);
474 static void start_virt_cntrs(void)
476 init_timer(&timer_virt_cntr
);
477 timer_virt_cntr
.function
= cell_virtual_cntr
;
478 timer_virt_cntr
.data
= 0UL;
479 timer_virt_cntr
.expires
= jiffies
+ HZ
/ 10;
480 add_timer(&timer_virt_cntr
);
483 /* This function is called once for all cpus combined */
485 cell_reg_setup(struct op_counter_config
*ctr
,
486 struct op_system_config
*sys
, int num_ctrs
)
490 pm_rtas_token
= rtas_token("ibm,cbe-perftools");
491 if (pm_rtas_token
== RTAS_UNKNOWN_SERVICE
) {
492 printk(KERN_WARNING
"%s: RTAS_UNKNOWN_SERVICE\n",
497 num_counters
= num_ctrs
;
499 pm_regs
.group_control
= 0;
500 pm_regs
.debug_bus_control
= 0;
502 /* setup the pm_control register */
503 memset(&pm_regs
.pm_cntrl
, 0, sizeof(struct pm_cntrl
));
504 pm_regs
.pm_cntrl
.stop_at_max
= 1;
505 pm_regs
.pm_cntrl
.trace_mode
= 0;
506 pm_regs
.pm_cntrl
.freeze
= 1;
508 set_count_mode(sys
->enable_kernel
, sys
->enable_user
);
510 /* Setup the thread 0 events */
511 for (i
= 0; i
< num_ctrs
; ++i
) {
513 pmc_cntrl
[0][i
].evnts
= ctr
[i
].event
;
514 pmc_cntrl
[0][i
].masks
= ctr
[i
].unit_mask
;
515 pmc_cntrl
[0][i
].enabled
= ctr
[i
].enabled
;
516 pmc_cntrl
[0][i
].vcntr
= i
;
518 for_each_possible_cpu(j
)
519 per_cpu(pmc_values
, j
)[i
] = 0;
522 /* Setup the thread 1 events, map the thread 0 event to the
523 * equivalent thread 1 event.
525 for (i
= 0; i
< num_ctrs
; ++i
) {
526 if ((ctr
[i
].event
>= 2100) && (ctr
[i
].event
<= 2111))
527 pmc_cntrl
[1][i
].evnts
= ctr
[i
].event
+ 19;
528 else if (ctr
[i
].event
== 2203)
529 pmc_cntrl
[1][i
].evnts
= ctr
[i
].event
;
530 else if ((ctr
[i
].event
>= 2200) && (ctr
[i
].event
<= 2215))
531 pmc_cntrl
[1][i
].evnts
= ctr
[i
].event
+ 16;
533 pmc_cntrl
[1][i
].evnts
= ctr
[i
].event
;
535 pmc_cntrl
[1][i
].masks
= ctr
[i
].unit_mask
;
536 pmc_cntrl
[1][i
].enabled
= ctr
[i
].enabled
;
537 pmc_cntrl
[1][i
].vcntr
= i
;
540 for (i
= 0; i
< NUM_TRACE_BUS_WORDS
; i
++)
543 for (i
= 0; i
< NUM_INPUT_BUS_WORDS
; i
++)
546 /* Our counters count up, and "count" refers to
547 * how much before the next interrupt, and we interrupt
548 * on overflow. So we calculate the starting value
549 * which will give us "count" until overflow.
550 * Then we set the events on the enabled counters.
552 for (i
= 0; i
< num_counters
; ++i
) {
553 /* start with virtual counter set 0 */
554 if (pmc_cntrl
[0][i
].enabled
) {
555 /* Using 32bit counters, reset max - count */
556 reset_value
[i
] = 0xFFFFFFFF - ctr
[i
].count
;
558 pmc_cntrl
[0][i
].evnts
,
559 pmc_cntrl
[0][i
].masks
);
561 /* global, used by cell_cpu_setup */
562 ctr_enabled
|= (1 << i
);
566 /* initialize the previous counts for the virtual cntrs */
567 for_each_online_cpu(cpu
)
568 for (i
= 0; i
< num_counters
; ++i
) {
569 per_cpu(pmc_values
, cpu
)[i
] = reset_value
[i
];
575 /* This function is called once for each cpu */
576 static void cell_cpu_setup(struct op_counter_config
*cntr
)
578 u32 cpu
= smp_processor_id();
582 /* There is one performance monitor per processor chip (i.e. node),
583 * so we only need to perform this function once per node.
585 if (cbe_get_hw_thread_id(cpu
))
588 if (pm_rtas_token
== RTAS_UNKNOWN_SERVICE
) {
589 printk(KERN_WARNING
"%s: RTAS_UNKNOWN_SERVICE\n",
594 /* Stop all counters */
596 cbe_disable_pm_interrupts(cpu
);
598 cbe_write_pm(cpu
, pm_interval
, 0);
599 cbe_write_pm(cpu
, pm_start_stop
, 0);
600 cbe_write_pm(cpu
, group_control
, pm_regs
.group_control
);
601 cbe_write_pm(cpu
, debug_bus_control
, pm_regs
.debug_bus_control
);
604 for (i
= 0; i
< num_counters
; ++i
) {
605 if (ctr_enabled
& (1 << i
)) {
606 pm_signal
[num_enabled
].cpu
= cbe_cpu_to_node(cpu
);
611 pm_rtas_activate_signals(cbe_cpu_to_node(cpu
), num_enabled
);
616 static void cell_global_start(struct op_counter_config
*ctr
)
619 u32 interrupt_mask
= 0;
622 /* This routine gets called once for the system.
623 * There is one performance monitor per node, so we
624 * only need to perform this function once per node.
626 for_each_online_cpu(cpu
) {
627 if (cbe_get_hw_thread_id(cpu
))
632 for (i
= 0; i
< num_counters
; ++i
) {
633 if (ctr_enabled
& (1 << i
)) {
634 cbe_write_ctr(cpu
, i
, reset_value
[i
]);
635 enable_ctr(cpu
, i
, pm_regs
.pm07_cntrl
);
637 CBE_PM_CTR_OVERFLOW_INTR(i
);
639 /* Disable counter */
640 cbe_write_pm07_control(cpu
, i
, 0);
644 cbe_get_and_clear_pm_interrupts(cpu
);
645 cbe_enable_pm_interrupts(cpu
, hdw_thread
, interrupt_mask
);
649 virt_cntr_inter_mask
= interrupt_mask
;
650 oprofile_running
= 1;
653 /* NOTE: start_virt_cntrs will result in cell_virtual_cntr() being
654 * executed which manipulates the PMU. We start the "virtual counter"
655 * here so that we do not need to synchronize access to the PMU in
656 * the above for-loop.
661 static void cell_global_stop(void)
665 /* This routine will be called once for the system.
666 * There is one performance monitor per node, so we
667 * only need to perform this function once per node.
669 del_timer_sync(&timer_virt_cntr
);
670 oprofile_running
= 0;
673 for_each_online_cpu(cpu
) {
674 if (cbe_get_hw_thread_id(cpu
))
677 cbe_sync_irq(cbe_cpu_to_node(cpu
));
678 /* Stop the counters */
681 /* Deactivate the signals */
682 pm_rtas_reset_signals(cbe_cpu_to_node(cpu
));
684 /* Deactivate interrupts */
685 cbe_disable_pm_interrupts(cpu
);
690 cell_handle_interrupt(struct pt_regs
*regs
, struct op_counter_config
*ctr
)
695 unsigned long flags
= 0;
699 cpu
= smp_processor_id();
701 /* Need to make sure the interrupt handler and the virt counter
702 * routine are not running at the same time. See the
703 * cell_virtual_cntr() routine for additional comments.
705 spin_lock_irqsave(&virt_cntr_lock
, flags
);
707 /* Need to disable and reenable the performance counters
708 * to get the desired behavior from the hardware. This
709 * is hardware specific.
714 interrupt_mask
= cbe_get_and_clear_pm_interrupts(cpu
);
716 /* If the interrupt mask has been cleared, then the virt cntr
717 * has cleared the interrupt. When the thread that generated
718 * the interrupt is restored, the data count will be restored to
719 * 0xffffff0 to cause the interrupt to be regenerated.
722 if ((oprofile_running
== 1) && (interrupt_mask
!= 0)) {
724 is_kernel
= is_kernel_addr(pc
);
726 for (i
= 0; i
< num_counters
; ++i
) {
727 if ((interrupt_mask
& CBE_PM_CTR_OVERFLOW_INTR(i
))
729 oprofile_add_pc(pc
, is_kernel
, i
);
730 cbe_write_ctr(cpu
, i
, reset_value
[i
]);
734 /* The counters were frozen by the interrupt.
735 * Reenable the interrupt and restart the counters.
736 * If there was a race between the interrupt handler and
737 * the virtual counter routine. The virutal counter
738 * routine may have cleared the interrupts. Hence must
739 * use the virt_cntr_inter_mask to re-enable the interrupts.
741 cbe_enable_pm_interrupts(cpu
, hdw_thread
,
742 virt_cntr_inter_mask
);
744 /* The writes to the various performance counters only writes
745 * to a latch. The new values (interrupt setting bits, reset
746 * counter value etc.) are not copied to the actual registers
747 * until the performance monitor is enabled. In order to get
748 * this to work as desired, the permormance monitor needs to
749 * be disabled while writting to the latches. This is a
754 spin_unlock_irqrestore(&virt_cntr_lock
, flags
);
757 struct op_powerpc_model op_model_cell
= {
758 .reg_setup
= cell_reg_setup
,
759 .cpu_setup
= cell_cpu_setup
,
760 .global_start
= cell_global_start
,
761 .global_stop
= cell_global_stop
,
762 .handle_interrupt
= cell_handle_interrupt
,