Commit | Line | Data |
---|---|---|
6a01f230 YL |
1 | /* -*- linux-c -*- |
2 | * linux/arch/blackfin/kernel/ipipe.c | |
3 | * | |
4 | * Copyright (C) 2005-2007 Philippe Gerum. | |
5 | * | |
6 | * This program is free software; you can redistribute it and/or modify | |
7 | * it under the terms of the GNU General Public License as published by | |
8 | * the Free Software Foundation, Inc., 675 Mass Ave, Cambridge MA 02139, | |
9 | * USA; either version 2 of the License, or (at your option) any later | |
10 | * version. | |
11 | * | |
12 | * This program is distributed in the hope that it will be useful, | |
13 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
14 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
15 | * GNU General Public License for more details. | |
16 | * | |
17 | * You should have received a copy of the GNU General Public License | |
18 | * along with this program; if not, write to the Free Software | |
19 | * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA. | |
20 | * | |
21 | * Architecture-dependent I-pipe support for the Blackfin. | |
22 | */ | |
23 | ||
24 | #include <linux/kernel.h> | |
25 | #include <linux/sched.h> | |
26 | #include <linux/module.h> | |
27 | #include <linux/interrupt.h> | |
28 | #include <linux/percpu.h> | |
29 | #include <linux/bitops.h> | |
30 | #include <linux/slab.h> | |
31 | #include <linux/errno.h> | |
32 | #include <linux/kthread.h> | |
33 | #include <asm/unistd.h> | |
34 | #include <asm/system.h> | |
35 | #include <asm/atomic.h> | |
36 | #include <asm/io.h> | |
37 | ||
6a01f230 YL |
38 | DEFINE_PER_CPU(struct pt_regs, __ipipe_tick_regs); |
39 | ||
6a01f230 YL |
40 | asmlinkage void asm_do_IRQ(unsigned int irq, struct pt_regs *regs); |
41 | ||
42 | static void __ipipe_no_irqtail(void); | |
43 | ||
44 | unsigned long __ipipe_irq_tail_hook = (unsigned long)&__ipipe_no_irqtail; | |
45 | EXPORT_SYMBOL(__ipipe_irq_tail_hook); | |
46 | ||
47 | unsigned long __ipipe_core_clock; | |
48 | EXPORT_SYMBOL(__ipipe_core_clock); | |
49 | ||
50 | unsigned long __ipipe_freq_scale; | |
51 | EXPORT_SYMBOL(__ipipe_freq_scale); | |
52 | ||
53 | atomic_t __ipipe_irq_lvdepth[IVG15 + 1]; | |
54 | ||
06ecc190 | 55 | unsigned long __ipipe_irq_lvmask = bfin_no_irqs; |
6a01f230 YL |
56 | EXPORT_SYMBOL(__ipipe_irq_lvmask); |
57 | ||
58 | static void __ipipe_ack_irq(unsigned irq, struct irq_desc *desc) | |
59 | { | |
60 | desc->ipipe_ack(irq, desc); | |
61 | } | |
62 | ||
63 | /* | |
64 | * __ipipe_enable_pipeline() -- We are running on the boot CPU, hw | |
65 | * interrupts are off, and secondary CPUs are still lost in space. | |
66 | */ | |
67 | void __ipipe_enable_pipeline(void) | |
68 | { | |
69 | unsigned irq; | |
70 | ||
71 | __ipipe_core_clock = get_cclk(); /* Fetch this once. */ | |
72 | __ipipe_freq_scale = 1000000000UL / __ipipe_core_clock; | |
73 | ||
74 | for (irq = 0; irq < NR_IRQS; ++irq) | |
75 | ipipe_virtualize_irq(ipipe_root_domain, | |
76 | irq, | |
77 | (ipipe_irq_handler_t)&asm_do_IRQ, | |
78 | NULL, | |
79 | &__ipipe_ack_irq, | |
80 | IPIPE_HANDLE_MASK | IPIPE_PASS_MASK); | |
81 | } | |
82 | ||
83 | /* | |
84 | * __ipipe_handle_irq() -- IPIPE's generic IRQ handler. An optimistic | |
85 | * interrupt protection log is maintained here for each domain. Hw | |
86 | * interrupts are masked on entry. | |
87 | */ | |
88 | void __ipipe_handle_irq(unsigned irq, struct pt_regs *regs) | |
89 | { | |
9bd50df6 | 90 | struct ipipe_percpu_domain_data *p = ipipe_root_cpudom_ptr(); |
6a01f230 YL |
91 | struct ipipe_domain *this_domain, *next_domain; |
92 | struct list_head *head, *pos; | |
93 | int m_ack, s = -1; | |
94 | ||
95 | /* | |
96 | * Software-triggered IRQs do not need any ack. The contents | |
97 | * of the register frame should only be used when processing | |
98 | * the timer interrupt, but not for handling any other | |
99 | * interrupt. | |
100 | */ | |
101 | m_ack = (regs == NULL || irq == IRQ_SYSTMR || irq == IRQ_CORETMR); | |
6640cfa8 | 102 | this_domain = __ipipe_current_domain; |
6a01f230 YL |
103 | |
104 | if (unlikely(test_bit(IPIPE_STICKY_FLAG, &this_domain->irqs[irq].control))) | |
105 | head = &this_domain->p_link; | |
106 | else { | |
107 | head = __ipipe_pipeline.next; | |
108 | next_domain = list_entry(head, struct ipipe_domain, p_link); | |
109 | if (likely(test_bit(IPIPE_WIRED_FLAG, &next_domain->irqs[irq].control))) { | |
110 | if (!m_ack && next_domain->irqs[irq].acknowledge != NULL) | |
9bd50df6 PG |
111 | next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq)); |
112 | if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status)) | |
113 | s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status); | |
6a01f230 | 114 | __ipipe_dispatch_wired(next_domain, irq); |
9bd50df6 | 115 | goto out; |
6a01f230 YL |
116 | } |
117 | } | |
118 | ||
119 | /* Ack the interrupt. */ | |
120 | ||
121 | pos = head; | |
6a01f230 YL |
122 | while (pos != &__ipipe_pipeline) { |
123 | next_domain = list_entry(pos, struct ipipe_domain, p_link); | |
6a01f230 | 124 | if (test_bit(IPIPE_HANDLE_FLAG, &next_domain->irqs[irq].control)) { |
6a01f230 | 125 | __ipipe_set_irq_pending(next_domain, irq); |
6a01f230 | 126 | if (!m_ack && next_domain->irqs[irq].acknowledge != NULL) { |
9bd50df6 | 127 | next_domain->irqs[irq].acknowledge(irq, irq_to_desc(irq)); |
6a01f230 YL |
128 | m_ack = 1; |
129 | } | |
130 | } | |
6a01f230 YL |
131 | if (!test_bit(IPIPE_PASS_FLAG, &next_domain->irqs[irq].control)) |
132 | break; | |
6a01f230 YL |
133 | pos = next_domain->p_link.next; |
134 | } | |
135 | ||
136 | /* | |
137 | * Now walk the pipeline, yielding control to the highest | |
138 | * priority domain that has pending interrupt(s) or | |
139 | * immediately to the current domain if the interrupt has been | |
140 | * marked as 'sticky'. This search does not go beyond the | |
141 | * current domain in the pipeline. We also enforce the | |
9bd50df6 PG |
142 | * additional root stage lock (blackfin-specific). |
143 | */ | |
144 | if (test_bit(IPIPE_SYNCDEFER_FLAG, &p->status)) | |
145 | s = __test_and_set_bit(IPIPE_STALL_FLAG, &p->status); | |
6a01f230 | 146 | |
9bd50df6 PG |
147 | /* |
148 | * If the interrupt preempted the head domain, then do not | |
149 | * even try to walk the pipeline, unless an interrupt is | |
150 | * pending for it. | |
151 | */ | |
152 | if (test_bit(IPIPE_AHEAD_FLAG, &this_domain->flags) && | |
153 | ipipe_head_cpudom_var(irqpend_himask) == 0) | |
154 | goto out; | |
6a01f230 YL |
155 | |
156 | __ipipe_walk_pipeline(head); | |
9bd50df6 | 157 | out: |
6a01f230 | 158 | if (!s) |
9bd50df6 | 159 | __clear_bit(IPIPE_STALL_FLAG, &p->status); |
6a01f230 YL |
160 | } |
161 | ||
162 | int __ipipe_check_root(void) | |
163 | { | |
164 | return ipipe_root_domain_p; | |
165 | } | |
166 | ||
167 | void __ipipe_enable_irqdesc(struct ipipe_domain *ipd, unsigned irq) | |
168 | { | |
9bd50df6 | 169 | struct irq_desc *desc = irq_to_desc(irq); |
51387009 | 170 | int prio = __ipipe_get_irq_priority(irq); |
6a01f230 YL |
171 | |
172 | desc->depth = 0; | |
173 | if (ipd != &ipipe_root && | |
174 | atomic_inc_return(&__ipipe_irq_lvdepth[prio]) == 1) | |
175 | __set_bit(prio, &__ipipe_irq_lvmask); | |
176 | } | |
177 | EXPORT_SYMBOL(__ipipe_enable_irqdesc); | |
178 | ||
179 | void __ipipe_disable_irqdesc(struct ipipe_domain *ipd, unsigned irq) | |
180 | { | |
51387009 | 181 | int prio = __ipipe_get_irq_priority(irq); |
6a01f230 YL |
182 | |
183 | if (ipd != &ipipe_root && | |
184 | atomic_dec_and_test(&__ipipe_irq_lvdepth[prio])) | |
185 | __clear_bit(prio, &__ipipe_irq_lvmask); | |
186 | } | |
187 | EXPORT_SYMBOL(__ipipe_disable_irqdesc); | |
188 | ||
189 | void __ipipe_stall_root_raw(void) | |
190 | { | |
191 | /* | |
192 | * This code is called by the ins{bwl} routines (see | |
193 | * arch/blackfin/lib/ins.S), which are heavily used by the | |
194 | * network stack. It masks all interrupts but those handled by | |
195 | * non-root domains, so that we keep decent network transfer | |
196 | * rates for Linux without inducing pathological jitter for | |
197 | * the real-time domain. | |
198 | */ | |
199 | __asm__ __volatile__ ("sti %0;" : : "d"(__ipipe_irq_lvmask)); | |
200 | ||
201 | __set_bit(IPIPE_STALL_FLAG, | |
202 | &ipipe_root_cpudom_var(status)); | |
203 | } | |
204 | ||
205 | void __ipipe_unstall_root_raw(void) | |
206 | { | |
207 | __clear_bit(IPIPE_STALL_FLAG, | |
208 | &ipipe_root_cpudom_var(status)); | |
209 | ||
210 | __asm__ __volatile__ ("sti %0;" : : "d"(bfin_irq_flags)); | |
211 | } | |
212 | ||
213 | int __ipipe_syscall_root(struct pt_regs *regs) | |
214 | { | |
6640cfa8 | 215 | struct ipipe_percpu_domain_data *p; |
6a01f230 | 216 | unsigned long flags; |
6640cfa8 | 217 | int ret; |
6a01f230 | 218 | |
9bd50df6 PG |
219 | /* |
220 | * We need to run the IRQ tail hook whenever we don't | |
6a01f230 YL |
221 | * propagate a syscall to higher domains, because we know that |
222 | * important operations might be pending there (e.g. Xenomai | |
9bd50df6 PG |
223 | * deferred rescheduling). |
224 | */ | |
6a01f230 | 225 | |
9bd50df6 | 226 | if (regs->orig_p0 < NR_syscalls) { |
6a01f230 YL |
227 | void (*hook)(void) = (void (*)(void))__ipipe_irq_tail_hook; |
228 | hook(); | |
9bd50df6 PG |
229 | if ((current->flags & PF_EVNOTIFY) == 0) |
230 | return 0; | |
6a01f230 YL |
231 | } |
232 | ||
233 | /* | |
234 | * This routine either returns: | |
235 | * 0 -- if the syscall is to be passed to Linux; | |
6640cfa8 | 236 | * >0 -- if the syscall should not be passed to Linux, and no |
6a01f230 | 237 | * tail work should be performed; |
6640cfa8 | 238 | * <0 -- if the syscall should not be passed to Linux but the |
6a01f230 YL |
239 | * tail work has to be performed (for handling signals etc). |
240 | */ | |
241 | ||
6640cfa8 YL |
242 | if (!__ipipe_event_monitored_p(IPIPE_EVENT_SYSCALL)) |
243 | return 0; | |
244 | ||
245 | ret = __ipipe_dispatch_event(IPIPE_EVENT_SYSCALL, regs); | |
246 | ||
247 | local_irq_save_hw(flags); | |
248 | ||
249 | if (!__ipipe_root_domain_p) { | |
250 | local_irq_restore_hw(flags); | |
6a01f230 YL |
251 | return 1; |
252 | } | |
253 | ||
6640cfa8 YL |
254 | p = ipipe_root_cpudom_ptr(); |
255 | if ((p->irqpend_himask & IPIPE_IRQMASK_VIRT) != 0) | |
256 | __ipipe_sync_pipeline(IPIPE_IRQMASK_VIRT); | |
257 | ||
258 | local_irq_restore_hw(flags); | |
259 | ||
260 | return -ret; | |
6a01f230 YL |
261 | } |
262 | ||
263 | unsigned long ipipe_critical_enter(void (*syncfn) (void)) | |
264 | { | |
265 | unsigned long flags; | |
266 | ||
267 | local_irq_save_hw(flags); | |
268 | ||
269 | return flags; | |
270 | } | |
271 | ||
272 | void ipipe_critical_exit(unsigned long flags) | |
273 | { | |
274 | local_irq_restore_hw(flags); | |
275 | } | |
276 | ||
277 | static void __ipipe_no_irqtail(void) | |
278 | { | |
279 | } | |
280 | ||
281 | int ipipe_get_sysinfo(struct ipipe_sysinfo *info) | |
282 | { | |
283 | info->ncpus = num_online_cpus(); | |
284 | info->cpufreq = ipipe_cpu_freq(); | |
285 | info->archdep.tmirq = IPIPE_TIMER_IRQ; | |
286 | info->archdep.tmfreq = info->cpufreq; | |
287 | ||
288 | return 0; | |
289 | } | |
290 | ||
291 | /* | |
292 | * ipipe_trigger_irq() -- Push the interrupt at front of the pipeline | |
293 | * just like if it has been actually received from a hw source. Also | |
294 | * works for virtual interrupts. | |
295 | */ | |
296 | int ipipe_trigger_irq(unsigned irq) | |
297 | { | |
298 | unsigned long flags; | |
299 | ||
9bd50df6 | 300 | #ifdef CONFIG_IPIPE_DEBUG |
6a01f230 YL |
301 | if (irq >= IPIPE_NR_IRQS || |
302 | (ipipe_virtual_irq_p(irq) | |
303 | && !test_bit(irq - IPIPE_VIRQ_BASE, &__ipipe_virtual_irq_map))) | |
304 | return -EINVAL; | |
9bd50df6 | 305 | #endif |
6a01f230 YL |
306 | |
307 | local_irq_save_hw(flags); | |
6a01f230 | 308 | __ipipe_handle_irq(irq, NULL); |
6a01f230 YL |
309 | local_irq_restore_hw(flags); |
310 | ||
311 | return 1; | |
312 | } | |
313 | ||
9bd50df6 | 314 | asmlinkage void __ipipe_sync_root(void) |
6a01f230 | 315 | { |
51387009 | 316 | void (*irq_tail_hook)(void) = (void (*)(void))__ipipe_irq_tail_hook; |
9bd50df6 | 317 | unsigned long flags; |
6a01f230 | 318 | |
9bd50df6 | 319 | BUG_ON(irqs_disabled()); |
6a01f230 | 320 | |
9bd50df6 | 321 | local_irq_save_hw(flags); |
6a01f230 | 322 | |
51387009 PG |
323 | if (irq_tail_hook) |
324 | irq_tail_hook(); | |
325 | ||
9bd50df6 | 326 | clear_thread_flag(TIF_IRQ_SYNC); |
6a01f230 | 327 | |
9bd50df6 PG |
328 | if (ipipe_root_cpudom_var(irqpend_himask) != 0) |
329 | __ipipe_sync_pipeline(IPIPE_IRQMASK_ANY); | |
6a01f230 | 330 | |
9bd50df6 | 331 | local_irq_restore_hw(flags); |
6a01f230 YL |
332 | } |
333 | ||
9bd50df6 | 334 | void ___ipipe_sync_pipeline(unsigned long syncmask) |
6a01f230 | 335 | { |
6640cfa8 | 336 | if (__ipipe_root_domain_p) { |
9bd50df6 PG |
337 | if (test_bit(IPIPE_SYNCDEFER_FLAG, &ipipe_root_cpudom_var(status))) |
338 | return; | |
6a01f230 | 339 | } |
9bd50df6 PG |
340 | |
341 | __ipipe_sync_stage(syncmask); | |
6a01f230 YL |
342 | } |
343 | ||
344 | EXPORT_SYMBOL(show_stack); | |
345 | ||
346 | #ifdef CONFIG_IPIPE_TRACE_MCOUNT | |
347 | void notrace _mcount(void); | |
348 | EXPORT_SYMBOL(_mcount); | |
349 | #endif /* CONFIG_IPIPE_TRACE_MCOUNT */ |