2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Copyright (C) 2004-2012 Cavium, Inc.
9 #include <linux/interrupt.h>
10 #include <linux/irqdomain.h>
11 #include <linux/bitops.h>
12 #include <linux/percpu.h>
13 #include <linux/slab.h>
14 #include <linux/irq.h>
15 #include <linux/smp.h>
18 #include <asm/octeon/octeon.h>
20 static DEFINE_RAW_SPINLOCK(octeon_irq_ciu0_lock
);
21 static DEFINE_RAW_SPINLOCK(octeon_irq_ciu1_lock
);
23 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu0_en_mirror
);
24 static DEFINE_PER_CPU(unsigned long, octeon_irq_ciu1_en_mirror
);
26 static __read_mostly u8 octeon_irq_ciu_to_irq
[8][64];
28 union octeon_ciu_chip_data
{
37 struct octeon_core_chip_data
{
38 struct mutex core_irq_mutex
;
44 #define MIPS_CORE_IRQ_LINES 8
46 static struct octeon_core_chip_data octeon_irq_core_chip_data
[MIPS_CORE_IRQ_LINES
];
48 static void octeon_irq_set_ciu_mapping(int irq
, int line
, int bit
,
49 struct irq_chip
*chip
,
50 irq_flow_handler_t handler
)
52 union octeon_ciu_chip_data cd
;
54 irq_set_chip_and_handler(irq
, chip
, handler
);
60 irq_set_chip_data(irq
, cd
.p
);
61 octeon_irq_ciu_to_irq
[line
][bit
] = irq
;
64 static int octeon_coreid_for_cpu(int cpu
)
67 return cpu_logical_map(cpu
);
69 return cvmx_get_core_num();
73 static int octeon_cpu_for_coreid(int coreid
)
76 return cpu_number_map(coreid
);
78 return smp_processor_id();
82 static void octeon_irq_core_ack(struct irq_data
*data
)
84 struct octeon_core_chip_data
*cd
= irq_data_get_irq_chip_data(data
);
85 unsigned int bit
= cd
->bit
;
88 * We don't need to disable IRQs to make these atomic since
89 * they are already disabled earlier in the low level
92 clear_c0_status(0x100 << bit
);
93 /* The two user interrupts must be cleared manually. */
95 clear_c0_cause(0x100 << bit
);
98 static void octeon_irq_core_eoi(struct irq_data
*data
)
100 struct octeon_core_chip_data
*cd
= irq_data_get_irq_chip_data(data
);
103 * We don't need to disable IRQs to make these atomic since
104 * they are already disabled earlier in the low level
107 set_c0_status(0x100 << cd
->bit
);
110 static void octeon_irq_core_set_enable_local(void *arg
)
112 struct irq_data
*data
= arg
;
113 struct octeon_core_chip_data
*cd
= irq_data_get_irq_chip_data(data
);
114 unsigned int mask
= 0x100 << cd
->bit
;
117 * Interrupts are already disabled, so these are atomic.
122 clear_c0_status(mask
);
126 static void octeon_irq_core_disable(struct irq_data
*data
)
128 struct octeon_core_chip_data
*cd
= irq_data_get_irq_chip_data(data
);
129 cd
->desired_en
= false;
132 static void octeon_irq_core_enable(struct irq_data
*data
)
134 struct octeon_core_chip_data
*cd
= irq_data_get_irq_chip_data(data
);
135 cd
->desired_en
= true;
138 static void octeon_irq_core_bus_lock(struct irq_data
*data
)
140 struct octeon_core_chip_data
*cd
= irq_data_get_irq_chip_data(data
);
142 mutex_lock(&cd
->core_irq_mutex
);
145 static void octeon_irq_core_bus_sync_unlock(struct irq_data
*data
)
147 struct octeon_core_chip_data
*cd
= irq_data_get_irq_chip_data(data
);
149 if (cd
->desired_en
!= cd
->current_en
) {
150 on_each_cpu(octeon_irq_core_set_enable_local
, data
, 1);
152 cd
->current_en
= cd
->desired_en
;
155 mutex_unlock(&cd
->core_irq_mutex
);
158 static struct irq_chip octeon_irq_chip_core
= {
160 .irq_enable
= octeon_irq_core_enable
,
161 .irq_disable
= octeon_irq_core_disable
,
162 .irq_ack
= octeon_irq_core_ack
,
163 .irq_eoi
= octeon_irq_core_eoi
,
164 .irq_bus_lock
= octeon_irq_core_bus_lock
,
165 .irq_bus_sync_unlock
= octeon_irq_core_bus_sync_unlock
,
167 .irq_cpu_online
= octeon_irq_core_eoi
,
168 .irq_cpu_offline
= octeon_irq_core_ack
,
169 .flags
= IRQCHIP_ONOFFLINE_ENABLED
,
172 static void __init
octeon_irq_init_core(void)
176 struct octeon_core_chip_data
*cd
;
178 for (i
= 0; i
< MIPS_CORE_IRQ_LINES
; i
++) {
179 cd
= &octeon_irq_core_chip_data
[i
];
180 cd
->current_en
= false;
181 cd
->desired_en
= false;
183 mutex_init(&cd
->core_irq_mutex
);
185 irq
= OCTEON_IRQ_SW0
+ i
;
187 case OCTEON_IRQ_TIMER
:
191 case OCTEON_IRQ_PERF
:
192 irq_set_chip_data(irq
, cd
);
193 irq_set_chip_and_handler(irq
, &octeon_irq_chip_core
,
202 static int next_cpu_for_irq(struct irq_data
*data
)
207 int weight
= cpumask_weight(data
->affinity
);
210 cpu
= smp_processor_id();
212 cpu
= cpumask_next(cpu
, data
->affinity
);
213 if (cpu
>= nr_cpu_ids
) {
216 } else if (cpumask_test_cpu(cpu
, cpu_online_mask
)) {
220 } else if (weight
== 1) {
221 cpu
= cpumask_first(data
->affinity
);
223 cpu
= smp_processor_id();
227 return smp_processor_id();
231 static void octeon_irq_ciu_enable(struct irq_data
*data
)
233 int cpu
= next_cpu_for_irq(data
);
234 int coreid
= octeon_coreid_for_cpu(cpu
);
237 union octeon_ciu_chip_data cd
;
239 cd
.p
= irq_data_get_irq_chip_data(data
);
241 if (cd
.s
.line
== 0) {
242 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock
, flags
);
243 pen
= &per_cpu(octeon_irq_ciu0_en_mirror
, cpu
);
244 set_bit(cd
.s
.bit
, pen
);
245 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid
* 2), *pen
);
246 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock
, flags
);
248 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock
, flags
);
249 pen
= &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
);
250 set_bit(cd
.s
.bit
, pen
);
251 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid
* 2 + 1), *pen
);
252 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock
, flags
);
256 static void octeon_irq_ciu_enable_local(struct irq_data
*data
)
260 union octeon_ciu_chip_data cd
;
262 cd
.p
= irq_data_get_irq_chip_data(data
);
264 if (cd
.s
.line
== 0) {
265 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock
, flags
);
266 pen
= &__get_cpu_var(octeon_irq_ciu0_en_mirror
);
267 set_bit(cd
.s
.bit
, pen
);
268 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen
);
269 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock
, flags
);
271 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock
, flags
);
272 pen
= &__get_cpu_var(octeon_irq_ciu1_en_mirror
);
273 set_bit(cd
.s
.bit
, pen
);
274 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen
);
275 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock
, flags
);
279 static void octeon_irq_ciu_disable_local(struct irq_data
*data
)
283 union octeon_ciu_chip_data cd
;
285 cd
.p
= irq_data_get_irq_chip_data(data
);
287 if (cd
.s
.line
== 0) {
288 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock
, flags
);
289 pen
= &__get_cpu_var(octeon_irq_ciu0_en_mirror
);
290 clear_bit(cd
.s
.bit
, pen
);
291 cvmx_write_csr(CVMX_CIU_INTX_EN0(cvmx_get_core_num() * 2), *pen
);
292 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock
, flags
);
294 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock
, flags
);
295 pen
= &__get_cpu_var(octeon_irq_ciu1_en_mirror
);
296 clear_bit(cd
.s
.bit
, pen
);
297 cvmx_write_csr(CVMX_CIU_INTX_EN1(cvmx_get_core_num() * 2 + 1), *pen
);
298 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock
, flags
);
302 static void octeon_irq_ciu_disable_all(struct irq_data
*data
)
307 union octeon_ciu_chip_data cd
;
309 wmb(); /* Make sure flag changes arrive before register updates. */
311 cd
.p
= irq_data_get_irq_chip_data(data
);
313 if (cd
.s
.line
== 0) {
314 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock
, flags
);
315 for_each_online_cpu(cpu
) {
316 int coreid
= octeon_coreid_for_cpu(cpu
);
317 pen
= &per_cpu(octeon_irq_ciu0_en_mirror
, cpu
);
318 clear_bit(cd
.s
.bit
, pen
);
319 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid
* 2), *pen
);
321 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock
, flags
);
323 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock
, flags
);
324 for_each_online_cpu(cpu
) {
325 int coreid
= octeon_coreid_for_cpu(cpu
);
326 pen
= &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
);
327 clear_bit(cd
.s
.bit
, pen
);
328 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid
* 2 + 1), *pen
);
330 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock
, flags
);
334 static void octeon_irq_ciu_enable_all(struct irq_data
*data
)
339 union octeon_ciu_chip_data cd
;
341 cd
.p
= irq_data_get_irq_chip_data(data
);
343 if (cd
.s
.line
== 0) {
344 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock
, flags
);
345 for_each_online_cpu(cpu
) {
346 int coreid
= octeon_coreid_for_cpu(cpu
);
347 pen
= &per_cpu(octeon_irq_ciu0_en_mirror
, cpu
);
348 set_bit(cd
.s
.bit
, pen
);
349 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid
* 2), *pen
);
351 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock
, flags
);
353 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock
, flags
);
354 for_each_online_cpu(cpu
) {
355 int coreid
= octeon_coreid_for_cpu(cpu
);
356 pen
= &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
);
357 set_bit(cd
.s
.bit
, pen
);
358 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid
* 2 + 1), *pen
);
360 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock
, flags
);
365 * Enable the irq on the next core in the affinity set for chips that
366 * have the EN*_W1{S,C} registers.
368 static void octeon_irq_ciu_enable_v2(struct irq_data
*data
)
371 int cpu
= next_cpu_for_irq(data
);
372 union octeon_ciu_chip_data cd
;
374 cd
.p
= irq_data_get_irq_chip_data(data
);
375 mask
= 1ull << (cd
.s
.bit
);
378 * Called under the desc lock, so these should never get out
381 if (cd
.s
.line
== 0) {
382 int index
= octeon_coreid_for_cpu(cpu
) * 2;
383 set_bit(cd
.s
.bit
, &per_cpu(octeon_irq_ciu0_en_mirror
, cpu
));
384 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index
), mask
);
386 int index
= octeon_coreid_for_cpu(cpu
) * 2 + 1;
387 set_bit(cd
.s
.bit
, &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
));
388 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index
), mask
);
393 * Enable the irq on the current CPU for chips that
394 * have the EN*_W1{S,C} registers.
396 static void octeon_irq_ciu_enable_local_v2(struct irq_data
*data
)
399 union octeon_ciu_chip_data cd
;
401 cd
.p
= irq_data_get_irq_chip_data(data
);
402 mask
= 1ull << (cd
.s
.bit
);
404 if (cd
.s
.line
== 0) {
405 int index
= cvmx_get_core_num() * 2;
406 set_bit(cd
.s
.bit
, &__get_cpu_var(octeon_irq_ciu0_en_mirror
));
407 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index
), mask
);
409 int index
= cvmx_get_core_num() * 2 + 1;
410 set_bit(cd
.s
.bit
, &__get_cpu_var(octeon_irq_ciu1_en_mirror
));
411 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index
), mask
);
415 static void octeon_irq_ciu_disable_local_v2(struct irq_data
*data
)
418 union octeon_ciu_chip_data cd
;
420 cd
.p
= irq_data_get_irq_chip_data(data
);
421 mask
= 1ull << (cd
.s
.bit
);
423 if (cd
.s
.line
== 0) {
424 int index
= cvmx_get_core_num() * 2;
425 clear_bit(cd
.s
.bit
, &__get_cpu_var(octeon_irq_ciu0_en_mirror
));
426 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index
), mask
);
428 int index
= cvmx_get_core_num() * 2 + 1;
429 clear_bit(cd
.s
.bit
, &__get_cpu_var(octeon_irq_ciu1_en_mirror
));
430 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index
), mask
);
435 * Write to the W1C bit in CVMX_CIU_INTX_SUM0 to clear the irq.
437 static void octeon_irq_ciu_ack(struct irq_data
*data
)
440 union octeon_ciu_chip_data cd
;
442 cd
.p
= data
->chip_data
;
443 mask
= 1ull << (cd
.s
.bit
);
445 if (cd
.s
.line
== 0) {
446 int index
= cvmx_get_core_num() * 2;
447 cvmx_write_csr(CVMX_CIU_INTX_SUM0(index
), mask
);
449 cvmx_write_csr(CVMX_CIU_INT_SUM1
, mask
);
454 * Disable the irq on the all cores for chips that have the EN*_W1{S,C}
457 static void octeon_irq_ciu_disable_all_v2(struct irq_data
*data
)
461 union octeon_ciu_chip_data cd
;
463 wmb(); /* Make sure flag changes arrive before register updates. */
465 cd
.p
= data
->chip_data
;
466 mask
= 1ull << (cd
.s
.bit
);
468 if (cd
.s
.line
== 0) {
469 for_each_online_cpu(cpu
) {
470 int index
= octeon_coreid_for_cpu(cpu
) * 2;
471 clear_bit(cd
.s
.bit
, &per_cpu(octeon_irq_ciu0_en_mirror
, cpu
));
472 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index
), mask
);
475 for_each_online_cpu(cpu
) {
476 int index
= octeon_coreid_for_cpu(cpu
) * 2 + 1;
477 clear_bit(cd
.s
.bit
, &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
));
478 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index
), mask
);
484 * Enable the irq on the all cores for chips that have the EN*_W1{S,C}
487 static void octeon_irq_ciu_enable_all_v2(struct irq_data
*data
)
491 union octeon_ciu_chip_data cd
;
493 cd
.p
= data
->chip_data
;
494 mask
= 1ull << (cd
.s
.bit
);
496 if (cd
.s
.line
== 0) {
497 for_each_online_cpu(cpu
) {
498 int index
= octeon_coreid_for_cpu(cpu
) * 2;
499 set_bit(cd
.s
.bit
, &per_cpu(octeon_irq_ciu0_en_mirror
, cpu
));
500 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index
), mask
);
503 for_each_online_cpu(cpu
) {
504 int index
= octeon_coreid_for_cpu(cpu
) * 2 + 1;
505 set_bit(cd
.s
.bit
, &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
));
506 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index
), mask
);
511 static void octeon_irq_gpio_setup(struct irq_data
*data
)
513 union cvmx_gpio_bit_cfgx cfg
;
514 union octeon_ciu_chip_data cd
;
515 u32 t
= irqd_get_trigger_type(data
);
517 cd
.p
= irq_data_get_irq_chip_data(data
);
521 cfg
.s
.int_type
= (t
& IRQ_TYPE_EDGE_BOTH
) != 0;
522 cfg
.s
.rx_xor
= (t
& (IRQ_TYPE_LEVEL_LOW
| IRQ_TYPE_EDGE_FALLING
)) != 0;
524 /* 140 nS glitch filter*/
528 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd
.s
.bit
- 16), cfg
.u64
);
531 static void octeon_irq_ciu_enable_gpio_v2(struct irq_data
*data
)
533 octeon_irq_gpio_setup(data
);
534 octeon_irq_ciu_enable_v2(data
);
537 static void octeon_irq_ciu_enable_gpio(struct irq_data
*data
)
539 octeon_irq_gpio_setup(data
);
540 octeon_irq_ciu_enable(data
);
543 static int octeon_irq_ciu_gpio_set_type(struct irq_data
*data
, unsigned int t
)
545 irqd_set_trigger_type(data
, t
);
546 octeon_irq_gpio_setup(data
);
548 return IRQ_SET_MASK_OK
;
551 static void octeon_irq_ciu_disable_gpio_v2(struct irq_data
*data
)
553 union octeon_ciu_chip_data cd
;
555 cd
.p
= irq_data_get_irq_chip_data(data
);
556 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd
.s
.bit
- 16), 0);
558 octeon_irq_ciu_disable_all_v2(data
);
561 static void octeon_irq_ciu_disable_gpio(struct irq_data
*data
)
563 union octeon_ciu_chip_data cd
;
565 cd
.p
= irq_data_get_irq_chip_data(data
);
566 cvmx_write_csr(CVMX_GPIO_BIT_CFGX(cd
.s
.bit
- 16), 0);
568 octeon_irq_ciu_disable_all(data
);
571 static void octeon_irq_ciu_gpio_ack(struct irq_data
*data
)
573 union octeon_ciu_chip_data cd
;
576 cd
.p
= irq_data_get_irq_chip_data(data
);
577 mask
= 1ull << (cd
.s
.bit
- 16);
579 cvmx_write_csr(CVMX_GPIO_INT_CLR
, mask
);
582 static void octeon_irq_handle_gpio(unsigned int irq
, struct irq_desc
*desc
)
584 if (irqd_get_trigger_type(irq_desc_get_irq_data(desc
)) & IRQ_TYPE_EDGE_BOTH
)
585 handle_edge_irq(irq
, desc
);
587 handle_level_irq(irq
, desc
);
592 static void octeon_irq_cpu_offline_ciu(struct irq_data
*data
)
594 int cpu
= smp_processor_id();
595 cpumask_t new_affinity
;
597 if (!cpumask_test_cpu(cpu
, data
->affinity
))
600 if (cpumask_weight(data
->affinity
) > 1) {
602 * It has multi CPU affinity, just remove this CPU
603 * from the affinity set.
605 cpumask_copy(&new_affinity
, data
->affinity
);
606 cpumask_clear_cpu(cpu
, &new_affinity
);
608 /* Otherwise, put it on lowest numbered online CPU. */
609 cpumask_clear(&new_affinity
);
610 cpumask_set_cpu(cpumask_first(cpu_online_mask
), &new_affinity
);
612 __irq_set_affinity_locked(data
, &new_affinity
);
615 static int octeon_irq_ciu_set_affinity(struct irq_data
*data
,
616 const struct cpumask
*dest
, bool force
)
619 bool enable_one
= !irqd_irq_disabled(data
) && !irqd_irq_masked(data
);
621 union octeon_ciu_chip_data cd
;
623 cd
.p
= data
->chip_data
;
626 * For non-v2 CIU, we will allow only single CPU affinity.
627 * This removes the need to do locking in the .ack/.eoi
630 if (cpumask_weight(dest
) != 1)
636 if (cd
.s
.line
== 0) {
637 raw_spin_lock_irqsave(&octeon_irq_ciu0_lock
, flags
);
638 for_each_online_cpu(cpu
) {
639 int coreid
= octeon_coreid_for_cpu(cpu
);
640 unsigned long *pen
= &per_cpu(octeon_irq_ciu0_en_mirror
, cpu
);
642 if (cpumask_test_cpu(cpu
, dest
) && enable_one
) {
644 set_bit(cd
.s
.bit
, pen
);
646 clear_bit(cd
.s
.bit
, pen
);
648 cvmx_write_csr(CVMX_CIU_INTX_EN0(coreid
* 2), *pen
);
650 raw_spin_unlock_irqrestore(&octeon_irq_ciu0_lock
, flags
);
652 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock
, flags
);
653 for_each_online_cpu(cpu
) {
654 int coreid
= octeon_coreid_for_cpu(cpu
);
655 unsigned long *pen
= &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
);
657 if (cpumask_test_cpu(cpu
, dest
) && enable_one
) {
659 set_bit(cd
.s
.bit
, pen
);
661 clear_bit(cd
.s
.bit
, pen
);
663 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid
* 2 + 1), *pen
);
665 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock
, flags
);
671 * Set affinity for the irq for chips that have the EN*_W1{S,C}
674 static int octeon_irq_ciu_set_affinity_v2(struct irq_data
*data
,
675 const struct cpumask
*dest
,
679 bool enable_one
= !irqd_irq_disabled(data
) && !irqd_irq_masked(data
);
681 union octeon_ciu_chip_data cd
;
686 cd
.p
= data
->chip_data
;
687 mask
= 1ull << cd
.s
.bit
;
689 if (cd
.s
.line
== 0) {
690 for_each_online_cpu(cpu
) {
691 unsigned long *pen
= &per_cpu(octeon_irq_ciu0_en_mirror
, cpu
);
692 int index
= octeon_coreid_for_cpu(cpu
) * 2;
693 if (cpumask_test_cpu(cpu
, dest
) && enable_one
) {
695 set_bit(cd
.s
.bit
, pen
);
696 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1S(index
), mask
);
698 clear_bit(cd
.s
.bit
, pen
);
699 cvmx_write_csr(CVMX_CIU_INTX_EN0_W1C(index
), mask
);
703 for_each_online_cpu(cpu
) {
704 unsigned long *pen
= &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
);
705 int index
= octeon_coreid_for_cpu(cpu
) * 2 + 1;
706 if (cpumask_test_cpu(cpu
, dest
) && enable_one
) {
708 set_bit(cd
.s
.bit
, pen
);
709 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(index
), mask
);
711 clear_bit(cd
.s
.bit
, pen
);
712 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1C(index
), mask
);
721 * The v1 CIU code already masks things, so supply a dummy version to
722 * the core chip code.
724 static void octeon_irq_dummy_mask(struct irq_data
*data
)
729 * Newer octeon chips have support for lockless CIU operation.
731 static struct irq_chip octeon_irq_chip_ciu_v2
= {
733 .irq_enable
= octeon_irq_ciu_enable_v2
,
734 .irq_disable
= octeon_irq_ciu_disable_all_v2
,
735 .irq_ack
= octeon_irq_ciu_ack
,
736 .irq_mask
= octeon_irq_ciu_disable_local_v2
,
737 .irq_unmask
= octeon_irq_ciu_enable_v2
,
739 .irq_set_affinity
= octeon_irq_ciu_set_affinity_v2
,
740 .irq_cpu_offline
= octeon_irq_cpu_offline_ciu
,
744 static struct irq_chip octeon_irq_chip_ciu
= {
746 .irq_enable
= octeon_irq_ciu_enable
,
747 .irq_disable
= octeon_irq_ciu_disable_all
,
748 .irq_ack
= octeon_irq_ciu_ack
,
749 .irq_mask
= octeon_irq_dummy_mask
,
751 .irq_set_affinity
= octeon_irq_ciu_set_affinity
,
752 .irq_cpu_offline
= octeon_irq_cpu_offline_ciu
,
756 /* The mbox versions don't do any affinity or round-robin. */
757 static struct irq_chip octeon_irq_chip_ciu_mbox_v2
= {
759 .irq_enable
= octeon_irq_ciu_enable_all_v2
,
760 .irq_disable
= octeon_irq_ciu_disable_all_v2
,
761 .irq_ack
= octeon_irq_ciu_disable_local_v2
,
762 .irq_eoi
= octeon_irq_ciu_enable_local_v2
,
764 .irq_cpu_online
= octeon_irq_ciu_enable_local_v2
,
765 .irq_cpu_offline
= octeon_irq_ciu_disable_local_v2
,
766 .flags
= IRQCHIP_ONOFFLINE_ENABLED
,
769 static struct irq_chip octeon_irq_chip_ciu_mbox
= {
771 .irq_enable
= octeon_irq_ciu_enable_all
,
772 .irq_disable
= octeon_irq_ciu_disable_all
,
774 .irq_cpu_online
= octeon_irq_ciu_enable_local
,
775 .irq_cpu_offline
= octeon_irq_ciu_disable_local
,
776 .flags
= IRQCHIP_ONOFFLINE_ENABLED
,
779 static struct irq_chip octeon_irq_chip_ciu_gpio_v2
= {
781 .irq_enable
= octeon_irq_ciu_enable_gpio_v2
,
782 .irq_disable
= octeon_irq_ciu_disable_gpio_v2
,
783 .irq_ack
= octeon_irq_ciu_gpio_ack
,
784 .irq_mask
= octeon_irq_ciu_disable_local_v2
,
785 .irq_unmask
= octeon_irq_ciu_enable_v2
,
786 .irq_set_type
= octeon_irq_ciu_gpio_set_type
,
788 .irq_set_affinity
= octeon_irq_ciu_set_affinity_v2
,
790 .flags
= IRQCHIP_SET_TYPE_MASKED
,
793 static struct irq_chip octeon_irq_chip_ciu_gpio
= {
795 .irq_enable
= octeon_irq_ciu_enable_gpio
,
796 .irq_disable
= octeon_irq_ciu_disable_gpio
,
797 .irq_mask
= octeon_irq_dummy_mask
,
798 .irq_ack
= octeon_irq_ciu_gpio_ack
,
799 .irq_set_type
= octeon_irq_ciu_gpio_set_type
,
801 .irq_set_affinity
= octeon_irq_ciu_set_affinity
,
803 .flags
= IRQCHIP_SET_TYPE_MASKED
,
807 * Watchdog interrupts are special. They are associated with a single
808 * core, so we hardwire the affinity to that core.
810 static void octeon_irq_ciu_wd_enable(struct irq_data
*data
)
814 int coreid
= data
->irq
- OCTEON_IRQ_WDOG0
; /* Bit 0-63 of EN1 */
815 int cpu
= octeon_cpu_for_coreid(coreid
);
817 raw_spin_lock_irqsave(&octeon_irq_ciu1_lock
, flags
);
818 pen
= &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
);
819 set_bit(coreid
, pen
);
820 cvmx_write_csr(CVMX_CIU_INTX_EN1(coreid
* 2 + 1), *pen
);
821 raw_spin_unlock_irqrestore(&octeon_irq_ciu1_lock
, flags
);
825 * Watchdog interrupts are special. They are associated with a single
826 * core, so we hardwire the affinity to that core.
828 static void octeon_irq_ciu1_wd_enable_v2(struct irq_data
*data
)
830 int coreid
= data
->irq
- OCTEON_IRQ_WDOG0
;
831 int cpu
= octeon_cpu_for_coreid(coreid
);
833 set_bit(coreid
, &per_cpu(octeon_irq_ciu1_en_mirror
, cpu
));
834 cvmx_write_csr(CVMX_CIU_INTX_EN1_W1S(coreid
* 2 + 1), 1ull << coreid
);
838 static struct irq_chip octeon_irq_chip_ciu_wd_v2
= {
840 .irq_enable
= octeon_irq_ciu1_wd_enable_v2
,
841 .irq_disable
= octeon_irq_ciu_disable_all_v2
,
842 .irq_mask
= octeon_irq_ciu_disable_local_v2
,
843 .irq_unmask
= octeon_irq_ciu_enable_local_v2
,
846 static struct irq_chip octeon_irq_chip_ciu_wd
= {
848 .irq_enable
= octeon_irq_ciu_wd_enable
,
849 .irq_disable
= octeon_irq_ciu_disable_all
,
850 .irq_mask
= octeon_irq_dummy_mask
,
853 static bool octeon_irq_ciu_is_edge(unsigned int line
, unsigned int bit
)
859 case 48 ... 49: /* GMX DRP */
860 case 50: /* IPD_DRP */
861 case 52 ... 55: /* Timers */
879 struct octeon_irq_gpio_domain_data
{
880 unsigned int base_hwirq
;
883 static int octeon_irq_gpio_xlat(struct irq_domain
*d
,
884 struct device_node
*node
,
886 unsigned int intsize
,
887 unsigned long *out_hwirq
,
888 unsigned int *out_type
)
892 unsigned int trigger
;
893 struct octeon_irq_gpio_domain_data
*gpiod
;
895 if (d
->of_node
!= node
)
905 trigger
= intspec
[1];
909 type
= IRQ_TYPE_EDGE_RISING
;
912 type
= IRQ_TYPE_EDGE_FALLING
;
915 type
= IRQ_TYPE_LEVEL_HIGH
;
918 type
= IRQ_TYPE_LEVEL_LOW
;
921 pr_err("Error: (%s) Invalid irq trigger specification: %x\n",
924 type
= IRQ_TYPE_LEVEL_LOW
;
928 gpiod
= d
->host_data
;
929 *out_hwirq
= gpiod
->base_hwirq
+ pin
;
934 static int octeon_irq_ciu_xlat(struct irq_domain
*d
,
935 struct device_node
*node
,
937 unsigned int intsize
,
938 unsigned long *out_hwirq
,
939 unsigned int *out_type
)
941 unsigned int ciu
, bit
;
946 if (ciu
> 1 || bit
> 63)
949 /* These are the GPIO lines */
950 if (ciu
== 0 && bit
>= 16 && bit
< 32)
953 *out_hwirq
= (ciu
<< 6) | bit
;
959 static struct irq_chip
*octeon_irq_ciu_chip
;
960 static struct irq_chip
*octeon_irq_gpio_chip
;
962 static bool octeon_irq_virq_in_range(unsigned int virq
)
964 /* We cannot let it overflow the mapping array. */
965 if (virq
< (1ul << 8 * sizeof(octeon_irq_ciu_to_irq
[0][0])))
968 WARN_ONCE(true, "virq out of range %u.\n", virq
);
972 static int octeon_irq_ciu_map(struct irq_domain
*d
,
973 unsigned int virq
, irq_hw_number_t hw
)
975 unsigned int line
= hw
>> 6;
976 unsigned int bit
= hw
& 63;
978 if (!octeon_irq_virq_in_range(virq
))
981 if (line
> 1 || octeon_irq_ciu_to_irq
[line
][bit
] != 0)
984 if (octeon_irq_ciu_is_edge(line
, bit
))
985 octeon_irq_set_ciu_mapping(virq
, line
, bit
,
989 octeon_irq_set_ciu_mapping(virq
, line
, bit
,
996 static int octeon_irq_gpio_map(struct irq_domain
*d
,
997 unsigned int virq
, irq_hw_number_t hw
)
999 unsigned int line
= hw
>> 6;
1000 unsigned int bit
= hw
& 63;
1002 if (!octeon_irq_virq_in_range(virq
))
1005 if (line
> 1 || octeon_irq_ciu_to_irq
[line
][bit
] != 0)
1008 octeon_irq_set_ciu_mapping(virq
, line
, bit
,
1009 octeon_irq_gpio_chip
,
1010 octeon_irq_handle_gpio
);
1015 static struct irq_domain_ops octeon_irq_domain_ciu_ops
= {
1016 .map
= octeon_irq_ciu_map
,
1017 .xlate
= octeon_irq_ciu_xlat
,
1020 static struct irq_domain_ops octeon_irq_domain_gpio_ops
= {
1021 .map
= octeon_irq_gpio_map
,
1022 .xlate
= octeon_irq_gpio_xlat
,
1025 static void octeon_irq_ip2_v1(void)
1027 const unsigned long core_id
= cvmx_get_core_num();
1028 u64 ciu_sum
= cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id
* 2));
1030 ciu_sum
&= __get_cpu_var(octeon_irq_ciu0_en_mirror
);
1031 clear_c0_status(STATUSF_IP2
);
1032 if (likely(ciu_sum
)) {
1033 int bit
= fls64(ciu_sum
) - 1;
1034 int irq
= octeon_irq_ciu_to_irq
[0][bit
];
1038 spurious_interrupt();
1040 spurious_interrupt();
1042 set_c0_status(STATUSF_IP2
);
1045 static void octeon_irq_ip2_v2(void)
1047 const unsigned long core_id
= cvmx_get_core_num();
1048 u64 ciu_sum
= cvmx_read_csr(CVMX_CIU_INTX_SUM0(core_id
* 2));
1050 ciu_sum
&= __get_cpu_var(octeon_irq_ciu0_en_mirror
);
1051 if (likely(ciu_sum
)) {
1052 int bit
= fls64(ciu_sum
) - 1;
1053 int irq
= octeon_irq_ciu_to_irq
[0][bit
];
1057 spurious_interrupt();
1059 spurious_interrupt();
1062 static void octeon_irq_ip3_v1(void)
1064 u64 ciu_sum
= cvmx_read_csr(CVMX_CIU_INT_SUM1
);
1066 ciu_sum
&= __get_cpu_var(octeon_irq_ciu1_en_mirror
);
1067 clear_c0_status(STATUSF_IP3
);
1068 if (likely(ciu_sum
)) {
1069 int bit
= fls64(ciu_sum
) - 1;
1070 int irq
= octeon_irq_ciu_to_irq
[1][bit
];
1074 spurious_interrupt();
1076 spurious_interrupt();
1078 set_c0_status(STATUSF_IP3
);
1081 static void octeon_irq_ip3_v2(void)
1083 u64 ciu_sum
= cvmx_read_csr(CVMX_CIU_INT_SUM1
);
1085 ciu_sum
&= __get_cpu_var(octeon_irq_ciu1_en_mirror
);
1086 if (likely(ciu_sum
)) {
1087 int bit
= fls64(ciu_sum
) - 1;
1088 int irq
= octeon_irq_ciu_to_irq
[1][bit
];
1092 spurious_interrupt();
1094 spurious_interrupt();
1098 static void octeon_irq_ip4_mask(void)
1100 clear_c0_status(STATUSF_IP4
);
1101 spurious_interrupt();
1104 static void (*octeon_irq_ip2
)(void);
1105 static void (*octeon_irq_ip3
)(void);
1106 static void (*octeon_irq_ip4
)(void);
1108 void __cpuinitdata (*octeon_irq_setup_secondary
)(void);
1110 static void __cpuinit
octeon_irq_percpu_enable(void)
1115 static void __cpuinit
octeon_irq_init_ciu_percpu(void)
1117 int coreid
= cvmx_get_core_num();
1119 * Disable All CIU Interrupts. The ones we need will be
1120 * enabled later. Read the SUM register so we know the write
1123 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid
* 2)), 0);
1124 cvmx_write_csr(CVMX_CIU_INTX_EN0((coreid
* 2 + 1)), 0);
1125 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid
* 2)), 0);
1126 cvmx_write_csr(CVMX_CIU_INTX_EN1((coreid
* 2 + 1)), 0);
1127 cvmx_read_csr(CVMX_CIU_INTX_SUM0((coreid
* 2)));
1130 static void __cpuinit
octeon_irq_setup_secondary_ciu(void)
1133 __get_cpu_var(octeon_irq_ciu0_en_mirror
) = 0;
1134 __get_cpu_var(octeon_irq_ciu1_en_mirror
) = 0;
1136 octeon_irq_init_ciu_percpu();
1137 octeon_irq_percpu_enable();
1139 /* Enable the CIU lines */
1140 set_c0_status(STATUSF_IP3
| STATUSF_IP2
);
1141 clear_c0_status(STATUSF_IP4
);
1144 static void __init
octeon_irq_init_ciu(void)
1147 struct irq_chip
*chip
;
1148 struct irq_chip
*chip_mbox
;
1149 struct irq_chip
*chip_wd
;
1150 struct device_node
*gpio_node
;
1151 struct device_node
*ciu_node
;
1153 octeon_irq_init_ciu_percpu();
1154 octeon_irq_setup_secondary
= octeon_irq_setup_secondary_ciu
;
1156 if (OCTEON_IS_MODEL(OCTEON_CN58XX_PASS2_X
) ||
1157 OCTEON_IS_MODEL(OCTEON_CN56XX_PASS2_X
) ||
1158 OCTEON_IS_MODEL(OCTEON_CN52XX_PASS2_X
) ||
1159 OCTEON_IS_MODEL(OCTEON_CN6XXX
)) {
1160 octeon_irq_ip2
= octeon_irq_ip2_v2
;
1161 octeon_irq_ip3
= octeon_irq_ip3_v2
;
1162 chip
= &octeon_irq_chip_ciu_v2
;
1163 chip_mbox
= &octeon_irq_chip_ciu_mbox_v2
;
1164 chip_wd
= &octeon_irq_chip_ciu_wd_v2
;
1165 octeon_irq_gpio_chip
= &octeon_irq_chip_ciu_gpio_v2
;
1167 octeon_irq_ip2
= octeon_irq_ip2_v1
;
1168 octeon_irq_ip3
= octeon_irq_ip3_v1
;
1169 chip
= &octeon_irq_chip_ciu
;
1170 chip_mbox
= &octeon_irq_chip_ciu_mbox
;
1171 chip_wd
= &octeon_irq_chip_ciu_wd
;
1172 octeon_irq_gpio_chip
= &octeon_irq_chip_ciu_gpio
;
1174 octeon_irq_ciu_chip
= chip
;
1175 octeon_irq_ip4
= octeon_irq_ip4_mask
;
1178 octeon_irq_init_core();
1181 for (i
= 0; i
< 16; i
++)
1182 octeon_irq_set_ciu_mapping(i
+ OCTEON_IRQ_WORKQ0
, 0, i
+ 0, chip
, handle_level_irq
);
1184 octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX0
, 0, 32, chip_mbox
, handle_percpu_irq
);
1185 octeon_irq_set_ciu_mapping(OCTEON_IRQ_MBOX1
, 0, 33, chip_mbox
, handle_percpu_irq
);
1187 for (i
= 0; i
< 4; i
++)
1188 octeon_irq_set_ciu_mapping(i
+ OCTEON_IRQ_PCI_INT0
, 0, i
+ 36, chip
, handle_level_irq
);
1189 for (i
= 0; i
< 4; i
++)
1190 octeon_irq_set_ciu_mapping(i
+ OCTEON_IRQ_PCI_MSI0
, 0, i
+ 40, chip
, handle_level_irq
);
1192 octeon_irq_set_ciu_mapping(OCTEON_IRQ_RML
, 0, 46, chip
, handle_level_irq
);
1193 for (i
= 0; i
< 4; i
++)
1194 octeon_irq_set_ciu_mapping(i
+ OCTEON_IRQ_TIMER0
, 0, i
+ 52, chip
, handle_edge_irq
);
1196 octeon_irq_set_ciu_mapping(OCTEON_IRQ_USB0
, 0, 56, chip
, handle_level_irq
);
1197 octeon_irq_set_ciu_mapping(OCTEON_IRQ_BOOTDMA
, 0, 63, chip
, handle_level_irq
);
1200 for (i
= 0; i
< 16; i
++)
1201 octeon_irq_set_ciu_mapping(i
+ OCTEON_IRQ_WDOG0
, 1, i
+ 0, chip_wd
, handle_level_irq
);
1203 octeon_irq_set_ciu_mapping(OCTEON_IRQ_USB1
, 1, 17, chip
, handle_level_irq
);
1205 gpio_node
= of_find_compatible_node(NULL
, NULL
, "cavium,octeon-3860-gpio");
1207 struct octeon_irq_gpio_domain_data
*gpiod
;
1209 gpiod
= kzalloc(sizeof(*gpiod
), GFP_KERNEL
);
1211 /* gpio domain host_data is the base hwirq number. */
1212 gpiod
->base_hwirq
= 16;
1213 irq_domain_add_linear(gpio_node
, 16, &octeon_irq_domain_gpio_ops
, gpiod
);
1214 of_node_put(gpio_node
);
1216 pr_warn("Cannot allocate memory for GPIO irq_domain.\n");
1218 pr_warn("Cannot find device node for cavium,octeon-3860-gpio.\n");
1220 ciu_node
= of_find_compatible_node(NULL
, NULL
, "cavium,octeon-3860-ciu");
1222 irq_domain_add_tree(ciu_node
, &octeon_irq_domain_ciu_ops
, NULL
);
1223 of_node_put(ciu_node
);
1225 pr_warn("Cannot find device node for cavium,octeon-3860-ciu.\n");
1227 /* Enable the CIU lines */
1228 set_c0_status(STATUSF_IP3
| STATUSF_IP2
);
1229 clear_c0_status(STATUSF_IP4
);
1232 void __init
arch_init_irq(void)
1235 /* Set the default affinity to the boot cpu. */
1236 cpumask_clear(irq_default_affinity
);
1237 cpumask_set_cpu(smp_processor_id(), irq_default_affinity
);
1239 octeon_irq_init_ciu();
1242 asmlinkage
void plat_irq_dispatch(void)
1244 unsigned long cop0_cause
;
1245 unsigned long cop0_status
;
1248 cop0_cause
= read_c0_cause();
1249 cop0_status
= read_c0_status();
1250 cop0_cause
&= cop0_status
;
1251 cop0_cause
&= ST0_IM
;
1253 if (unlikely(cop0_cause
& STATUSF_IP2
))
1255 else if (unlikely(cop0_cause
& STATUSF_IP3
))
1257 else if (unlikely(cop0_cause
& STATUSF_IP4
))
1259 else if (likely(cop0_cause
))
1260 do_IRQ(fls(cop0_cause
) - 9 + MIPS_CPU_IRQ_BASE
);
1266 #ifdef CONFIG_HOTPLUG_CPU
1268 void fixup_irqs(void)
1273 #endif /* CONFIG_HOTPLUG_CPU */