Commit | Line | Data |
---|---|---|
18aecc2b CM |
1 | /* |
2 | * Copyright 2011 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
13 | * | |
14 | * Linux interrupt vectors. | |
15 | */ | |
16 | ||
17 | #include <linux/linkage.h> | |
18 | #include <linux/errno.h> | |
19 | #include <linux/unistd.h> | |
2f9ac29e | 20 | #include <linux/init.h> |
18aecc2b CM |
21 | #include <asm/ptrace.h> |
22 | #include <asm/thread_info.h> | |
23 | #include <asm/irqflags.h> | |
24 | #include <asm/asm-offsets.h> | |
25 | #include <asm/types.h> | |
2f9ac29e | 26 | #include <asm/traps.h> |
2858f856 | 27 | #include <asm/signal.h> |
18aecc2b CM |
28 | #include <hv/hypervisor.h> |
29 | #include <arch/abi.h> | |
30 | #include <arch/interrupts.h> | |
31 | #include <arch/spr_def.h> | |
32 | ||
18aecc2b CM |
33 | #define PTREGS_PTR(reg, ptreg) addli reg, sp, C_ABI_SAVE_AREA_SIZE + (ptreg) |
34 | ||
35 | #define PTREGS_OFFSET_SYSCALL PTREGS_OFFSET_REG(TREG_SYSCALL_NR) | |
36 | ||
051168df CM |
37 | #if CONFIG_KERNEL_PL == 1 || CONFIG_KERNEL_PL == 2 |
38 | /* | |
39 | * Set "result" non-zero if ex1 holds the PL of the kernel | |
40 | * (with or without ICS being set). Note this works only | |
41 | * because we never find the PL at level 3. | |
42 | */ | |
43 | # define IS_KERNEL_EX1(result, ex1) andi result, ex1, CONFIG_KERNEL_PL | |
44 | #else | |
45 | # error Recode IS_KERNEL_EX1 for CONFIG_KERNEL_PL | |
46 | #endif | |
18aecc2b CM |
47 | |
48 | .macro push_reg reg, ptr=sp, delta=-8 | |
49 | { | |
50 | st \ptr, \reg | |
51 | addli \ptr, \ptr, \delta | |
52 | } | |
53 | .endm | |
54 | ||
55 | .macro pop_reg reg, ptr=sp, delta=8 | |
56 | { | |
57 | ld \reg, \ptr | |
58 | addli \ptr, \ptr, \delta | |
59 | } | |
60 | .endm | |
61 | ||
62 | .macro pop_reg_zero reg, zreg, ptr=sp, delta=8 | |
63 | { | |
64 | move \zreg, zero | |
65 | ld \reg, \ptr | |
66 | addi \ptr, \ptr, \delta | |
67 | } | |
68 | .endm | |
69 | ||
70 | .macro push_extra_callee_saves reg | |
71 | PTREGS_PTR(\reg, PTREGS_OFFSET_REG(51)) | |
72 | push_reg r51, \reg | |
73 | push_reg r50, \reg | |
74 | push_reg r49, \reg | |
75 | push_reg r48, \reg | |
76 | push_reg r47, \reg | |
77 | push_reg r46, \reg | |
78 | push_reg r45, \reg | |
79 | push_reg r44, \reg | |
80 | push_reg r43, \reg | |
81 | push_reg r42, \reg | |
82 | push_reg r41, \reg | |
83 | push_reg r40, \reg | |
84 | push_reg r39, \reg | |
85 | push_reg r38, \reg | |
86 | push_reg r37, \reg | |
87 | push_reg r36, \reg | |
88 | push_reg r35, \reg | |
89 | push_reg r34, \reg, PTREGS_OFFSET_BASE - PTREGS_OFFSET_REG(34) | |
90 | .endm | |
91 | ||
92 | .macro panic str | |
93 | .pushsection .rodata, "a" | |
94 | 1: | |
95 | .asciz "\str" | |
96 | .popsection | |
97 | { | |
98 | moveli r0, hw2_last(1b) | |
99 | } | |
100 | { | |
101 | shl16insli r0, r0, hw1(1b) | |
102 | } | |
103 | { | |
104 | shl16insli r0, r0, hw0(1b) | |
105 | jal panic | |
106 | } | |
107 | .endm | |
108 | ||
2f9ac29e CM |
109 | /* |
110 | * Unalign data exception fast handling: In order to handle | |
111 | * unaligned data access, a fast JIT version is generated and stored | |
112 | * in a specific area in user space. We first need to do a quick poke | |
113 | * to see if the JIT is available. We use certain bits in the fault | |
114 | * PC (3 to 9 is used for 16KB page size) as index to address the JIT | |
115 | * code area. The first 64bit word is the fault PC, and the 2nd one is | |
116 | * the fault bundle itself. If these 2 words both match, then we | |
117 | * directly "iret" to JIT code. If not, a slow path is invoked to | |
118 | * generate new JIT code. Note: the current JIT code WILL be | |
119 | * overwritten if it existed. So, ideally we can handle 128 unalign | |
120 | * fixups via JIT. For lookup efficiency and to effectively support | |
121 | * tight loops with multiple unaligned reference, a simple | |
122 | * direct-mapped cache is used. | |
123 | * | |
124 | * SPR_EX_CONTEXT_K_0 is modified to return to JIT code. | |
125 | * SPR_EX_CONTEXT_K_1 has ICS set. | |
126 | * SPR_EX_CONTEXT_0_0 is setup to user program's next PC. | |
127 | * SPR_EX_CONTEXT_0_1 = 0. | |
128 | */ | |
129 | .macro int_hand_unalign_fast vecnum, vecname | |
130 | .org (\vecnum << 8) | |
131 | intvec_\vecname: | |
132 | /* Put r3 in SPR_SYSTEM_SAVE_K_1. */ | |
133 | mtspr SPR_SYSTEM_SAVE_K_1, r3 | |
134 | ||
135 | mfspr r3, SPR_EX_CONTEXT_K_1 | |
136 | /* | |
137 | * Examine if exception comes from user without ICS set. | |
138 | * If not, just go directly to the slow path. | |
139 | */ | |
140 | bnez r3, hand_unalign_slow_nonuser | |
141 | ||
142 | mfspr r3, SPR_SYSTEM_SAVE_K_0 | |
143 | ||
144 | /* Get &thread_info->unalign_jit_tmp[0] in r3. */ | |
35f05976 | 145 | bfexts r3, r3, 0, CPU_SHIFT-1 |
2f9ac29e | 146 | mm r3, zero, LOG2_THREAD_SIZE, 63 |
35f05976 | 147 | addli r3, r3, THREAD_INFO_UNALIGN_JIT_TMP_OFFSET |
2f9ac29e CM |
148 | |
149 | /* | |
150 | * Save r0, r1, r2 into thread_info array r3 points to | |
151 | * from low to high memory in order. | |
152 | */ | |
153 | st_add r3, r0, 8 | |
154 | st_add r3, r1, 8 | |
155 | { | |
156 | st_add r3, r2, 8 | |
157 | andi r2, sp, 7 | |
158 | } | |
159 | ||
160 | /* Save stored r3 value so we can revert it on a page fault. */ | |
161 | mfspr r1, SPR_SYSTEM_SAVE_K_1 | |
162 | st r3, r1 | |
163 | ||
164 | { | |
165 | /* Generate a SIGBUS if sp is not 8-byte aligned. */ | |
166 | bnez r2, hand_unalign_slow_badsp | |
167 | } | |
168 | ||
169 | /* | |
170 | * Get the thread_info in r0; load r1 with pc. Set the low bit of sp | |
171 | * as an indicator to the page fault code in case we fault. | |
172 | */ | |
173 | { | |
174 | ori sp, sp, 1 | |
175 | mfspr r1, SPR_EX_CONTEXT_K_0 | |
176 | } | |
177 | ||
178 | /* Add the jit_info offset in thread_info; extract r1 [3:9] into r2. */ | |
179 | { | |
180 | addli r0, r3, THREAD_INFO_UNALIGN_JIT_BASE_OFFSET - \ | |
181 | (THREAD_INFO_UNALIGN_JIT_TMP_OFFSET + (3 * 8)) | |
182 | bfextu r2, r1, 3, (2 + PAGE_SHIFT - UNALIGN_JIT_SHIFT) | |
183 | } | |
184 | ||
185 | /* Load the jit_info; multiply r2 by 128. */ | |
186 | { | |
187 | ld r0, r0 | |
188 | shli r2, r2, UNALIGN_JIT_SHIFT | |
189 | } | |
190 | ||
191 | /* | |
192 | * If r0 is NULL, the JIT page is not mapped, so go to slow path; | |
193 | * add offset r2 to r0 at the same time. | |
194 | */ | |
195 | { | |
196 | beqz r0, hand_unalign_slow | |
197 | add r2, r0, r2 | |
198 | } | |
199 | ||
200 | /* | |
201 | * We are loading from userspace (both the JIT info PC and | |
202 | * instruction word, and the instruction word we executed) | |
203 | * and since either could fault while holding the interrupt | |
204 | * critical section, we must tag this region and check it in | |
205 | * do_page_fault() to handle it properly. | |
206 | */ | |
207 | ENTRY(__start_unalign_asm_code) | |
208 | ||
209 | /* Load first word of JIT in r0 and increment r2 by 8. */ | |
210 | ld_add r0, r2, 8 | |
211 | ||
212 | /* | |
213 | * Compare the PC with the 1st word in JIT; load the fault bundle | |
214 | * into r1. | |
215 | */ | |
216 | { | |
217 | cmpeq r0, r0, r1 | |
218 | ld r1, r1 | |
219 | } | |
220 | ||
221 | /* Go to slow path if PC doesn't match. */ | |
222 | beqz r0, hand_unalign_slow | |
223 | ||
224 | /* | |
225 | * Load the 2nd word of JIT, which is supposed to be the fault | |
226 | * bundle for a cache hit. Increment r2; after this bundle r2 will | |
227 | * point to the potential start of the JIT code we want to run. | |
228 | */ | |
229 | ld_add r0, r2, 8 | |
230 | ||
231 | /* No further accesses to userspace are done after this point. */ | |
232 | ENTRY(__end_unalign_asm_code) | |
233 | ||
234 | /* Compare the real bundle with what is saved in the JIT area. */ | |
235 | { | |
236 | cmpeq r0, r1, r0 | |
237 | mtspr SPR_EX_CONTEXT_0_1, zero | |
238 | } | |
239 | ||
240 | /* Go to slow path if the fault bundle does not match. */ | |
241 | beqz r0, hand_unalign_slow | |
242 | ||
243 | /* | |
244 | * A cache hit is found. | |
245 | * r2 points to start of JIT code (3rd word). | |
246 | * r0 is the fault pc. | |
247 | * r1 is the fault bundle. | |
248 | * Reset the low bit of sp. | |
249 | */ | |
250 | { | |
251 | mfspr r0, SPR_EX_CONTEXT_K_0 | |
252 | andi sp, sp, ~1 | |
253 | } | |
254 | ||
255 | /* Write r2 into EX_CONTEXT_K_0 and increment PC. */ | |
256 | { | |
257 | mtspr SPR_EX_CONTEXT_K_0, r2 | |
258 | addi r0, r0, 8 | |
259 | } | |
260 | ||
261 | /* | |
262 | * Set ICS on kernel EX_CONTEXT_K_1 in order to "iret" to | |
263 | * user with ICS set. This way, if the JIT fixup causes another | |
264 | * unalign exception (which shouldn't be possible) the user | |
265 | * process will be terminated with SIGBUS. Also, our fixup will | |
266 | * run without interleaving with external interrupts. | |
267 | * Each fixup is at most 14 bundles, so it won't hold ICS for long. | |
268 | */ | |
269 | { | |
270 | movei r1, PL_ICS_EX1(USER_PL, 1) | |
271 | mtspr SPR_EX_CONTEXT_0_0, r0 | |
272 | } | |
273 | ||
274 | { | |
275 | mtspr SPR_EX_CONTEXT_K_1, r1 | |
276 | addi r3, r3, -(3 * 8) | |
277 | } | |
278 | ||
279 | /* Restore r0..r3. */ | |
280 | ld_add r0, r3, 8 | |
281 | ld_add r1, r3, 8 | |
282 | ld_add r2, r3, 8 | |
283 | ld r3, r3 | |
284 | ||
285 | iret | |
286 | ENDPROC(intvec_\vecname) | |
287 | .endm | |
18aecc2b CM |
288 | |
289 | #ifdef __COLLECT_LINKER_FEEDBACK__ | |
290 | .pushsection .text.intvec_feedback,"ax" | |
291 | intvec_feedback: | |
292 | .popsection | |
293 | #endif | |
294 | ||
295 | /* | |
296 | * Default interrupt handler. | |
297 | * | |
298 | * vecnum is where we'll put this code. | |
299 | * c_routine is the C routine we'll call. | |
300 | * | |
301 | * The C routine is passed two arguments: | |
302 | * - A pointer to the pt_regs state. | |
303 | * - The interrupt vector number. | |
304 | * | |
305 | * The "processing" argument specifies the code for processing | |
306 | * the interrupt. Defaults to "handle_interrupt". | |
307 | */ | |
2f9ac29e | 308 | .macro __int_hand vecnum, vecname, c_routine,processing=handle_interrupt |
18aecc2b CM |
309 | intvec_\vecname: |
310 | /* Temporarily save a register so we have somewhere to work. */ | |
311 | ||
312 | mtspr SPR_SYSTEM_SAVE_K_1, r0 | |
313 | mfspr r0, SPR_EX_CONTEXT_K_1 | |
314 | ||
2f9ac29e CM |
315 | /* |
316 | * The unalign data fastpath code sets the low bit in sp to | |
317 | * force us to reset it here on fault. | |
318 | */ | |
319 | { | |
320 | blbs sp, 2f | |
051168df | 321 | IS_KERNEL_EX1(r0, r0) |
2f9ac29e | 322 | } |
18aecc2b CM |
323 | |
324 | .ifc \vecnum, INT_DOUBLE_FAULT | |
325 | /* | |
326 | * For double-faults from user-space, fall through to the normal | |
327 | * register save and stack setup path. Otherwise, it's the | |
328 | * hypervisor giving us one last chance to dump diagnostics, and we | |
329 | * branch to the kernel_double_fault routine to do so. | |
330 | */ | |
331 | beqz r0, 1f | |
332 | j _kernel_double_fault | |
333 | 1: | |
334 | .else | |
335 | /* | |
336 | * If we're coming from user-space, then set sp to the top of | |
337 | * the kernel stack. Otherwise, assume sp is already valid. | |
338 | */ | |
339 | { | |
340 | bnez r0, 0f | |
341 | move r0, sp | |
342 | } | |
343 | .endif | |
344 | ||
345 | .ifc \c_routine, do_page_fault | |
346 | /* | |
347 | * The page_fault handler may be downcalled directly by the | |
348 | * hypervisor even when Linux is running and has ICS set. | |
349 | * | |
350 | * In this case the contents of EX_CONTEXT_K_1 reflect the | |
351 | * previous fault and can't be relied on to choose whether or | |
352 | * not to reinitialize the stack pointer. So we add a test | |
353 | * to see whether SYSTEM_SAVE_K_2 has the high bit set, | |
354 | * and if so we don't reinitialize sp, since we must be coming | |
355 | * from Linux. (In fact the precise case is !(val & ~1), | |
356 | * but any Linux PC has to have the high bit set.) | |
357 | * | |
358 | * Note that the hypervisor *always* sets SYSTEM_SAVE_K_2 for | |
359 | * any path that turns into a downcall to one of our TLB handlers. | |
360 | * | |
361 | * FIXME: if we end up never using this path, perhaps we should | |
362 | * prevent the hypervisor from generating downcalls in this case. | |
363 | * The advantage of getting a downcall is we can panic in Linux. | |
364 | */ | |
365 | mfspr r0, SPR_SYSTEM_SAVE_K_2 | |
366 | { | |
367 | bltz r0, 0f /* high bit in S_S_1_2 is for a PC to use */ | |
368 | move r0, sp | |
369 | } | |
370 | .endif | |
371 | ||
2f9ac29e | 372 | 2: |
18aecc2b | 373 | /* |
35f05976 CM |
374 | * SYSTEM_SAVE_K_0 holds the cpu number in the high bits, and |
375 | * the current stack top in the lower bits. So we recover | |
376 | * our starting stack value by sign-extending the low bits, then | |
18aecc2b CM |
377 | * point sp at the top aligned address on the actual stack page. |
378 | */ | |
379 | mfspr r0, SPR_SYSTEM_SAVE_K_0 | |
35f05976 | 380 | bfexts r0, r0, 0, CPU_SHIFT-1 |
18aecc2b CM |
381 | |
382 | 0: | |
383 | /* | |
384 | * Align the stack mod 64 so we can properly predict what | |
385 | * cache lines we need to write-hint to reduce memory fetch | |
386 | * latency as we enter the kernel. The layout of memory is | |
387 | * as follows, with cache line 0 at the lowest VA, and cache | |
388 | * line 8 just below the r0 value this "andi" computes. | |
389 | * Note that we never write to cache line 8, and we skip | |
390 | * cache lines 1-3 for syscalls. | |
391 | * | |
392 | * cache line 8: ptregs padding (two words) | |
393 | * cache line 7: sp, lr, pc, ex1, faultnum, orig_r0, flags, cmpexch | |
394 | * cache line 6: r46...r53 (tp) | |
395 | * cache line 5: r38...r45 | |
396 | * cache line 4: r30...r37 | |
397 | * cache line 3: r22...r29 | |
398 | * cache line 2: r14...r21 | |
399 | * cache line 1: r6...r13 | |
400 | * cache line 0: 2 x frame, r0..r5 | |
401 | */ | |
35f05976 CM |
402 | #if STACK_TOP_DELTA != 64 |
403 | #error STACK_TOP_DELTA must be 64 for assumptions here and in task_pt_regs() | |
404 | #endif | |
18aecc2b CM |
405 | andi r0, r0, -64 |
406 | ||
407 | /* | |
408 | * Push the first four registers on the stack, so that we can set | |
409 | * them to vector-unique values before we jump to the common code. | |
410 | * | |
411 | * Registers are pushed on the stack as a struct pt_regs, | |
412 | * with the sp initially just above the struct, and when we're | |
413 | * done, sp points to the base of the struct, minus | |
414 | * C_ABI_SAVE_AREA_SIZE, so we can directly jal to C code. | |
415 | * | |
416 | * This routine saves just the first four registers, plus the | |
417 | * stack context so we can do proper backtracing right away, | |
418 | * and defers to handle_interrupt to save the rest. | |
51007004 CM |
419 | * The backtracer needs pc, ex1, lr, sp, r52, and faultnum, |
420 | * and needs sp set to its final location at the bottom of | |
421 | * the stack frame. | |
18aecc2b CM |
422 | */ |
423 | addli r0, r0, PTREGS_OFFSET_LR - (PTREGS_SIZE + KSTK_PTREGS_GAP) | |
424 | wh64 r0 /* cache line 7 */ | |
425 | { | |
426 | st r0, lr | |
427 | addli r0, r0, PTREGS_OFFSET_SP - PTREGS_OFFSET_LR | |
428 | } | |
429 | { | |
430 | st r0, sp | |
431 | addli sp, r0, PTREGS_OFFSET_REG(52) - PTREGS_OFFSET_SP | |
432 | } | |
433 | wh64 sp /* cache line 6 */ | |
434 | { | |
435 | st sp, r52 | |
436 | addli sp, sp, PTREGS_OFFSET_REG(1) - PTREGS_OFFSET_REG(52) | |
437 | } | |
438 | wh64 sp /* cache line 0 */ | |
439 | { | |
440 | st sp, r1 | |
441 | addli sp, sp, PTREGS_OFFSET_REG(2) - PTREGS_OFFSET_REG(1) | |
442 | } | |
443 | { | |
444 | st sp, r2 | |
445 | addli sp, sp, PTREGS_OFFSET_REG(3) - PTREGS_OFFSET_REG(2) | |
446 | } | |
447 | { | |
448 | st sp, r3 | |
449 | addli sp, sp, PTREGS_OFFSET_PC - PTREGS_OFFSET_REG(3) | |
450 | } | |
451 | mfspr r0, SPR_EX_CONTEXT_K_0 | |
452 | .ifc \processing,handle_syscall | |
453 | /* | |
454 | * Bump the saved PC by one bundle so that when we return, we won't | |
455 | * execute the same swint instruction again. We need to do this while | |
456 | * we're in the critical section. | |
457 | */ | |
458 | addi r0, r0, 8 | |
459 | .endif | |
460 | { | |
461 | st sp, r0 | |
462 | addli sp, sp, PTREGS_OFFSET_EX1 - PTREGS_OFFSET_PC | |
463 | } | |
464 | mfspr r0, SPR_EX_CONTEXT_K_1 | |
465 | { | |
466 | st sp, r0 | |
467 | addi sp, sp, PTREGS_OFFSET_FAULTNUM - PTREGS_OFFSET_EX1 | |
468 | /* | |
469 | * Use r0 for syscalls so it's a temporary; use r1 for interrupts | |
470 | * so that it gets passed through unchanged to the handler routine. | |
471 | * Note that the .if conditional confusingly spans bundles. | |
472 | */ | |
473 | .ifc \processing,handle_syscall | |
474 | movei r0, \vecnum | |
475 | } | |
476 | { | |
477 | st sp, r0 | |
478 | .else | |
479 | movei r1, \vecnum | |
480 | } | |
481 | { | |
482 | st sp, r1 | |
483 | .endif | |
484 | addli sp, sp, PTREGS_OFFSET_REG(0) - PTREGS_OFFSET_FAULTNUM | |
485 | } | |
486 | mfspr r0, SPR_SYSTEM_SAVE_K_1 /* Original r0 */ | |
487 | { | |
488 | st sp, r0 | |
489 | addi sp, sp, -PTREGS_OFFSET_REG(0) - 8 | |
490 | } | |
491 | { | |
492 | st sp, zero /* write zero into "Next SP" frame pointer */ | |
493 | addi sp, sp, -8 /* leave SP pointing at bottom of frame */ | |
494 | } | |
495 | .ifc \processing,handle_syscall | |
496 | j handle_syscall | |
497 | .else | |
498 | /* Capture per-interrupt SPR context to registers. */ | |
499 | .ifc \c_routine, do_page_fault | |
500 | mfspr r2, SPR_SYSTEM_SAVE_K_3 /* address of page fault */ | |
501 | mfspr r3, SPR_SYSTEM_SAVE_K_2 /* info about page fault */ | |
502 | .else | |
503 | .ifc \vecnum, INT_ILL_TRANS | |
70d2b595 | 504 | mfspr r2, ILL_VA_PC |
18aecc2b CM |
505 | .else |
506 | .ifc \vecnum, INT_DOUBLE_FAULT | |
507 | mfspr r2, SPR_SYSTEM_SAVE_K_2 /* double fault info from HV */ | |
508 | .else | |
509 | .ifc \c_routine, do_trap | |
510 | mfspr r2, GPV_REASON | |
511 | .else | |
8e3441eb | 512 | .ifc \c_routine, handle_perf_interrupt |
18aecc2b | 513 | mfspr r2, PERF_COUNT_STS |
18aecc2b | 514 | .else |
8e3441eb | 515 | .ifc \c_routine, handle_perf_interrupt |
18aecc2b CM |
516 | mfspr r2, AUX_PERF_COUNT_STS |
517 | .endif | |
18aecc2b CM |
518 | .endif |
519 | .endif | |
520 | .endif | |
521 | .endif | |
522 | .endif | |
523 | /* Put function pointer in r0 */ | |
524 | moveli r0, hw2_last(\c_routine) | |
525 | shl16insli r0, r0, hw1(\c_routine) | |
526 | { | |
527 | shl16insli r0, r0, hw0(\c_routine) | |
528 | j \processing | |
529 | } | |
530 | .endif | |
531 | ENDPROC(intvec_\vecname) | |
532 | ||
533 | #ifdef __COLLECT_LINKER_FEEDBACK__ | |
534 | .pushsection .text.intvec_feedback,"ax" | |
535 | .org (\vecnum << 5) | |
acbde1db | 536 | FEEDBACK_ENTER_EXPLICIT(intvec_\vecname, .intrpt, 1 << 8) |
18aecc2b CM |
537 | jrp lr |
538 | .popsection | |
539 | #endif | |
540 | ||
541 | .endm | |
542 | ||
543 | ||
544 | /* | |
545 | * Save the rest of the registers that we didn't save in the actual | |
546 | * vector itself. We can't use r0-r10 inclusive here. | |
547 | */ | |
548 | .macro finish_interrupt_save, function | |
549 | ||
550 | /* If it's a syscall, save a proper orig_r0, otherwise just zero. */ | |
551 | PTREGS_PTR(r52, PTREGS_OFFSET_ORIG_R0) | |
552 | { | |
553 | .ifc \function,handle_syscall | |
554 | st r52, r0 | |
555 | .else | |
556 | st r52, zero | |
557 | .endif | |
558 | PTREGS_PTR(r52, PTREGS_OFFSET_TP) | |
559 | } | |
560 | st r52, tp | |
561 | { | |
562 | mfspr tp, CMPEXCH_VALUE | |
563 | PTREGS_PTR(r52, PTREGS_OFFSET_CMPEXCH) | |
564 | } | |
565 | ||
566 | /* | |
567 | * For ordinary syscalls, we save neither caller- nor callee- | |
568 | * save registers, since the syscall invoker doesn't expect the | |
569 | * caller-saves to be saved, and the called kernel functions will | |
570 | * take care of saving the callee-saves for us. | |
571 | * | |
572 | * For interrupts we save just the caller-save registers. Saving | |
573 | * them is required (since the "caller" can't save them). Again, | |
574 | * the called kernel functions will restore the callee-save | |
575 | * registers for us appropriately. | |
576 | * | |
577 | * On return, we normally restore nothing special for syscalls, | |
578 | * and just the caller-save registers for interrupts. | |
579 | * | |
580 | * However, there are some important caveats to all this: | |
581 | * | |
582 | * - We always save a few callee-save registers to give us | |
583 | * some scratchpad registers to carry across function calls. | |
584 | * | |
585 | * - fork/vfork/etc require us to save all the callee-save | |
586 | * registers, which we do in PTREGS_SYSCALL_ALL_REGS, below. | |
587 | * | |
588 | * - We always save r0..r5 and r10 for syscalls, since we need | |
589 | * to reload them a bit later for the actual kernel call, and | |
590 | * since we might need them for -ERESTARTNOINTR, etc. | |
591 | * | |
592 | * - Before invoking a signal handler, we save the unsaved | |
593 | * callee-save registers so they are visible to the | |
594 | * signal handler or any ptracer. | |
595 | * | |
596 | * - If the unsaved callee-save registers are modified, we set | |
597 | * a bit in pt_regs so we know to reload them from pt_regs | |
598 | * and not just rely on the kernel function unwinding. | |
599 | * (Done for ptrace register writes and SA_SIGINFO handler.) | |
600 | */ | |
601 | { | |
602 | st r52, tp | |
603 | PTREGS_PTR(r52, PTREGS_OFFSET_REG(33)) | |
604 | } | |
605 | wh64 r52 /* cache line 4 */ | |
606 | push_reg r33, r52 | |
607 | push_reg r32, r52 | |
608 | push_reg r31, r52 | |
609 | .ifc \function,handle_syscall | |
610 | push_reg r30, r52, PTREGS_OFFSET_SYSCALL - PTREGS_OFFSET_REG(30) | |
611 | push_reg TREG_SYSCALL_NR_NAME, r52, \ | |
612 | PTREGS_OFFSET_REG(5) - PTREGS_OFFSET_SYSCALL | |
613 | .else | |
614 | ||
615 | push_reg r30, r52, PTREGS_OFFSET_REG(29) - PTREGS_OFFSET_REG(30) | |
616 | wh64 r52 /* cache line 3 */ | |
617 | push_reg r29, r52 | |
618 | push_reg r28, r52 | |
619 | push_reg r27, r52 | |
620 | push_reg r26, r52 | |
621 | push_reg r25, r52 | |
622 | push_reg r24, r52 | |
623 | push_reg r23, r52 | |
624 | push_reg r22, r52 | |
625 | wh64 r52 /* cache line 2 */ | |
626 | push_reg r21, r52 | |
627 | push_reg r20, r52 | |
628 | push_reg r19, r52 | |
629 | push_reg r18, r52 | |
630 | push_reg r17, r52 | |
631 | push_reg r16, r52 | |
632 | push_reg r15, r52 | |
633 | push_reg r14, r52 | |
634 | wh64 r52 /* cache line 1 */ | |
635 | push_reg r13, r52 | |
636 | push_reg r12, r52 | |
637 | push_reg r11, r52 | |
638 | push_reg r10, r52 | |
639 | push_reg r9, r52 | |
640 | push_reg r8, r52 | |
641 | push_reg r7, r52 | |
642 | push_reg r6, r52 | |
643 | ||
644 | .endif | |
645 | ||
646 | push_reg r5, r52 | |
647 | st r52, r4 | |
648 | ||
18aecc2b CM |
649 | /* |
650 | * If we will be returning to the kernel, we will need to | |
651 | * reset the interrupt masks to the state they had before. | |
051168df CM |
652 | * Set DISABLE_IRQ in flags iff we came from kernel pl with |
653 | * irqs disabled. | |
18aecc2b CM |
654 | */ |
655 | mfspr r32, SPR_EX_CONTEXT_K_1 | |
656 | { | |
051168df | 657 | IS_KERNEL_EX1(r22, r22) |
18aecc2b CM |
658 | PTREGS_PTR(r21, PTREGS_OFFSET_FLAGS) |
659 | } | |
660 | beqzt r32, 1f /* zero if from user space */ | |
661 | IRQS_DISABLED(r32) /* zero if irqs enabled */ | |
662 | #if PT_FLAGS_DISABLE_IRQ != 1 | |
663 | # error Value of IRQS_DISABLED used to set PT_FLAGS_DISABLE_IRQ; fix | |
664 | #endif | |
665 | 1: | |
666 | .ifnc \function,handle_syscall | |
667 | /* Record the fact that we saved the caller-save registers above. */ | |
668 | ori r32, r32, PT_FLAGS_CALLER_SAVES | |
669 | .endif | |
670 | st r21, r32 | |
671 | ||
51007004 CM |
672 | /* |
673 | * we've captured enough state to the stack (including in | |
674 | * particular our EX_CONTEXT state) that we can now release | |
675 | * the interrupt critical section and replace it with our | |
676 | * standard "interrupts disabled" mask value. This allows | |
677 | * synchronous interrupts (and profile interrupts) to punch | |
678 | * through from this point onwards. | |
679 | * | |
680 | * It's important that no code before this point touch memory | |
681 | * other than our own stack (to keep the invariant that this | |
682 | * is all that gets touched under ICS), and that no code after | |
683 | * this point reference any interrupt-specific SPR, in particular | |
684 | * the EX_CONTEXT_K_ values. | |
685 | */ | |
686 | .ifc \function,handle_nmi | |
687 | IRQ_DISABLE_ALL(r20) | |
688 | .else | |
689 | IRQ_DISABLE(r20, r21) | |
690 | .endif | |
691 | mtspr INTERRUPT_CRITICAL_SECTION, zero | |
692 | ||
693 | /* Load tp with our per-cpu offset. */ | |
694 | #ifdef CONFIG_SMP | |
695 | { | |
696 | mfspr r20, SPR_SYSTEM_SAVE_K_0 | |
697 | moveli r21, hw2_last(__per_cpu_offset) | |
698 | } | |
699 | { | |
700 | shl16insli r21, r21, hw1(__per_cpu_offset) | |
35f05976 | 701 | bfextu r20, r20, CPU_SHIFT, 63 |
51007004 CM |
702 | } |
703 | shl16insli r21, r21, hw0(__per_cpu_offset) | |
704 | shl3add r20, r20, r21 | |
705 | ld tp, r20 | |
706 | #else | |
707 | move tp, zero | |
708 | #endif | |
709 | ||
18aecc2b CM |
710 | #ifdef __COLLECT_LINKER_FEEDBACK__ |
711 | /* | |
712 | * Notify the feedback routines that we were in the | |
713 | * appropriate fixed interrupt vector area. Note that we | |
714 | * still have ICS set at this point, so we can't invoke any | |
715 | * atomic operations or we will panic. The feedback | |
716 | * routines internally preserve r0..r10 and r30 up. | |
717 | */ | |
718 | .ifnc \function,handle_syscall | |
719 | shli r20, r1, 5 | |
720 | .else | |
721 | moveli r20, INT_SWINT_1 << 5 | |
722 | .endif | |
723 | moveli r21, hw2_last(intvec_feedback) | |
724 | shl16insli r21, r21, hw1(intvec_feedback) | |
725 | shl16insli r21, r21, hw0(intvec_feedback) | |
726 | add r20, r20, r21 | |
727 | jalr r20 | |
728 | ||
729 | /* And now notify the feedback routines that we are here. */ | |
730 | FEEDBACK_ENTER(\function) | |
731 | #endif | |
732 | ||
18aecc2b CM |
733 | /* |
734 | * Prepare the first 256 stack bytes to be rapidly accessible | |
735 | * without having to fetch the background data. | |
736 | */ | |
737 | addi r52, sp, -64 | |
738 | { | |
739 | wh64 r52 | |
740 | addi r52, r52, -64 | |
741 | } | |
742 | { | |
743 | wh64 r52 | |
744 | addi r52, r52, -64 | |
745 | } | |
746 | { | |
747 | wh64 r52 | |
748 | addi r52, r52, -64 | |
749 | } | |
750 | wh64 r52 | |
751 | ||
752 | #ifdef CONFIG_TRACE_IRQFLAGS | |
753 | .ifnc \function,handle_nmi | |
754 | /* | |
755 | * We finally have enough state set up to notify the irq | |
756 | * tracing code that irqs were disabled on entry to the handler. | |
757 | * The TRACE_IRQS_OFF call clobbers registers r0-r29. | |
758 | * For syscalls, we already have the register state saved away | |
759 | * on the stack, so we don't bother to do any register saves here, | |
760 | * and later we pop the registers back off the kernel stack. | |
761 | * For interrupt handlers, save r0-r3 in callee-saved registers. | |
762 | */ | |
763 | .ifnc \function,handle_syscall | |
764 | { move r30, r0; move r31, r1 } | |
765 | { move r32, r2; move r33, r3 } | |
766 | .endif | |
767 | TRACE_IRQS_OFF | |
768 | .ifnc \function,handle_syscall | |
769 | { move r0, r30; move r1, r31 } | |
770 | { move r2, r32; move r3, r33 } | |
771 | .endif | |
772 | .endif | |
773 | #endif | |
774 | ||
775 | .endm | |
776 | ||
777 | /* | |
778 | * Redispatch a downcall. | |
779 | */ | |
780 | .macro dc_dispatch vecnum, vecname | |
781 | .org (\vecnum << 8) | |
782 | intvec_\vecname: | |
9ae09838 | 783 | j _hv_downcall_dispatch |
18aecc2b CM |
784 | ENDPROC(intvec_\vecname) |
785 | .endm | |
786 | ||
787 | /* | |
788 | * Common code for most interrupts. The C function we're eventually | |
789 | * going to is in r0, and the faultnum is in r1; the original | |
790 | * values for those registers are on the stack. | |
791 | */ | |
792 | .pushsection .text.handle_interrupt,"ax" | |
793 | handle_interrupt: | |
794 | finish_interrupt_save handle_interrupt | |
795 | ||
796 | /* Jump to the C routine; it should enable irqs as soon as possible. */ | |
797 | { | |
798 | jalr r0 | |
799 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
800 | } | |
801 | FEEDBACK_REENTER(handle_interrupt) | |
802 | { | |
803 | movei r30, 0 /* not an NMI */ | |
804 | j interrupt_return | |
805 | } | |
806 | STD_ENDPROC(handle_interrupt) | |
807 | ||
808 | /* | |
809 | * This routine takes a boolean in r30 indicating if this is an NMI. | |
810 | * If so, we also expect a boolean in r31 indicating whether to | |
811 | * re-enable the oprofile interrupts. | |
e1d5c019 CM |
812 | * |
813 | * Note that .Lresume_userspace is jumped to directly in several | |
814 | * places, and we need to make sure r30 is set correctly in those | |
815 | * callers as well. | |
18aecc2b CM |
816 | */ |
817 | STD_ENTRY(interrupt_return) | |
818 | /* If we're resuming to kernel space, don't check thread flags. */ | |
819 | { | |
820 | bnez r30, .Lrestore_all /* NMIs don't special-case user-space */ | |
821 | PTREGS_PTR(r29, PTREGS_OFFSET_EX1) | |
822 | } | |
823 | ld r29, r29 | |
051168df | 824 | IS_KERNEL_EX1(r29, r29) |
18aecc2b CM |
825 | { |
826 | beqzt r29, .Lresume_userspace | |
bc1a298f | 827 | move r29, sp |
18aecc2b CM |
828 | } |
829 | ||
bc1a298f CM |
830 | #ifdef CONFIG_PREEMPT |
831 | /* Returning to kernel space. Check if we need preemption. */ | |
832 | EXTRACT_THREAD_INFO(r29) | |
833 | addli r28, r29, THREAD_INFO_FLAGS_OFFSET | |
834 | { | |
835 | ld r28, r28 | |
836 | addli r29, r29, THREAD_INFO_PREEMPT_COUNT_OFFSET | |
837 | } | |
838 | { | |
839 | andi r28, r28, _TIF_NEED_RESCHED | |
840 | ld4s r29, r29 | |
841 | } | |
842 | beqzt r28, 1f | |
843 | bnez r29, 1f | |
3f725c5b CM |
844 | /* Disable interrupts explicitly for preemption. */ |
845 | IRQ_DISABLE(r20,r21) | |
846 | TRACE_IRQS_OFF | |
bc1a298f CM |
847 | jal preempt_schedule_irq |
848 | FEEDBACK_REENTER(interrupt_return) | |
849 | 1: | |
850 | #endif | |
851 | ||
18aecc2b | 852 | /* If we're resuming to _cpu_idle_nap, bump PC forward by 8. */ |
bc1a298f CM |
853 | { |
854 | moveli r27, hw2_last(_cpu_idle_nap) | |
855 | PTREGS_PTR(r29, PTREGS_OFFSET_PC) | |
856 | } | |
18aecc2b CM |
857 | { |
858 | ld r28, r29 | |
859 | shl16insli r27, r27, hw1(_cpu_idle_nap) | |
860 | } | |
861 | { | |
862 | shl16insli r27, r27, hw0(_cpu_idle_nap) | |
863 | } | |
864 | { | |
865 | cmpeq r27, r27, r28 | |
866 | } | |
867 | { | |
868 | blbc r27, .Lrestore_all | |
869 | addi r28, r28, 8 | |
870 | } | |
871 | st r29, r28 | |
872 | j .Lrestore_all | |
873 | ||
874 | .Lresume_userspace: | |
875 | FEEDBACK_REENTER(interrupt_return) | |
876 | ||
fc327e26 CM |
877 | /* |
878 | * Use r33 to hold whether we have already loaded the callee-saves | |
879 | * into ptregs. We don't want to do it twice in this loop, since | |
880 | * then we'd clobber whatever changes are made by ptrace, etc. | |
881 | */ | |
882 | { | |
883 | movei r33, 0 | |
884 | move r32, sp | |
885 | } | |
886 | ||
887 | /* Get base of stack in r32. */ | |
888 | EXTRACT_THREAD_INFO(r32) | |
889 | ||
890 | .Lretry_work_pending: | |
18aecc2b CM |
891 | /* |
892 | * Disable interrupts so as to make sure we don't | |
893 | * miss an interrupt that sets any of the thread flags (like | |
894 | * need_resched or sigpending) between sampling and the iret. | |
895 | * Routines like schedule() or do_signal() may re-enable | |
896 | * interrupts before returning. | |
897 | */ | |
898 | IRQ_DISABLE(r20, r21) | |
899 | TRACE_IRQS_OFF /* Note: clobbers registers r0-r29 */ | |
900 | ||
18aecc2b CM |
901 | |
902 | /* Check to see if there is any work to do before returning to user. */ | |
903 | { | |
904 | addi r29, r32, THREAD_INFO_FLAGS_OFFSET | |
905 | moveli r1, hw1_last(_TIF_ALLWORK_MASK) | |
906 | } | |
907 | { | |
908 | ld r29, r29 | |
909 | shl16insli r1, r1, hw0(_TIF_ALLWORK_MASK) | |
910 | } | |
911 | and r1, r29, r1 | |
912 | beqzt r1, .Lrestore_all | |
913 | ||
914 | /* | |
915 | * Make sure we have all the registers saved for signal | |
fc327e26 | 916 | * handling or notify-resume. Call out to C code to figure out |
18aecc2b CM |
917 | * exactly what we need to do for each flag bit, then if |
918 | * necessary, reload the flags and recheck. | |
919 | */ | |
18aecc2b CM |
920 | { |
921 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
fc327e26 | 922 | bnez r33, 1f |
18aecc2b | 923 | } |
fc327e26 CM |
924 | push_extra_callee_saves r0 |
925 | movei r33, 1 | |
926 | 1: jal do_work_pending | |
927 | bnez r0, .Lretry_work_pending | |
18aecc2b CM |
928 | |
929 | /* | |
930 | * In the NMI case we | |
931 | * omit the call to single_process_check_nohz, which normally checks | |
932 | * to see if we should start or stop the scheduler tick, because | |
933 | * we can't call arbitrary Linux code from an NMI context. | |
934 | * We always call the homecache TLB deferral code to re-trigger | |
935 | * the deferral mechanism. | |
936 | * | |
937 | * The other chunk of responsibility this code has is to reset the | |
938 | * interrupt masks appropriately to reset irqs and NMIs. We have | |
939 | * to call TRACE_IRQS_OFF and TRACE_IRQS_ON to support all the | |
940 | * lockdep-type stuff, but we can't set ICS until afterwards, since | |
941 | * ICS can only be used in very tight chunks of code to avoid | |
942 | * tripping over various assertions that it is off. | |
943 | */ | |
944 | .Lrestore_all: | |
945 | PTREGS_PTR(r0, PTREGS_OFFSET_EX1) | |
946 | { | |
947 | ld r0, r0 | |
948 | PTREGS_PTR(r32, PTREGS_OFFSET_FLAGS) | |
949 | } | |
950 | { | |
051168df | 951 | IS_KERNEL_EX1(r0, r0) |
18aecc2b CM |
952 | ld r32, r32 |
953 | } | |
954 | bnez r0, 1f | |
955 | j 2f | |
956 | #if PT_FLAGS_DISABLE_IRQ != 1 | |
957 | # error Assuming PT_FLAGS_DISABLE_IRQ == 1 so we can use blbct below | |
958 | #endif | |
959 | 1: blbct r32, 2f | |
960 | IRQ_DISABLE(r20,r21) | |
961 | TRACE_IRQS_OFF | |
962 | movei r0, 1 | |
963 | mtspr INTERRUPT_CRITICAL_SECTION, r0 | |
964 | beqzt r30, .Lrestore_regs | |
965 | j 3f | |
966 | 2: TRACE_IRQS_ON | |
51007004 | 967 | IRQ_ENABLE_LOAD(r20, r21) |
18aecc2b CM |
968 | movei r0, 1 |
969 | mtspr INTERRUPT_CRITICAL_SECTION, r0 | |
51007004 | 970 | IRQ_ENABLE_APPLY(r20, r21) |
18aecc2b CM |
971 | beqzt r30, .Lrestore_regs |
972 | 3: | |
973 | ||
ba678231 ZL |
974 | #if INT_PERF_COUNT + 1 != INT_AUX_PERF_COUNT |
975 | # error Bad interrupt assumption | |
976 | #endif | |
977 | { | |
978 | movei r0, 3 /* two adjacent bits for the PERF_COUNT mask */ | |
979 | beqz r31, .Lrestore_regs | |
980 | } | |
981 | shli r0, r0, INT_PERF_COUNT | |
982 | mtspr SPR_INTERRUPT_MASK_RESET_K, r0 | |
18aecc2b CM |
983 | |
984 | /* | |
985 | * We now commit to returning from this interrupt, since we will be | |
986 | * doing things like setting EX_CONTEXT SPRs and unwinding the stack | |
987 | * frame. No calls should be made to any other code after this point. | |
988 | * This code should only be entered with ICS set. | |
989 | * r32 must still be set to ptregs.flags. | |
990 | * We launch loads to each cache line separately first, so we can | |
991 | * get some parallelism out of the memory subsystem. | |
992 | * We start zeroing caller-saved registers throughout, since | |
993 | * that will save some cycles if this turns out to be a syscall. | |
994 | */ | |
995 | .Lrestore_regs: | |
18aecc2b CM |
996 | |
997 | /* | |
998 | * Rotate so we have one high bit and one low bit to test. | |
999 | * - low bit says whether to restore all the callee-saved registers, | |
1000 | * or just r30-r33, and r52 up. | |
1001 | * - high bit (i.e. sign bit) says whether to restore all the | |
1002 | * caller-saved registers, or just r0. | |
1003 | */ | |
1004 | #if PT_FLAGS_CALLER_SAVES != 2 || PT_FLAGS_RESTORE_REGS != 4 | |
1005 | # error Rotate trick does not work :-) | |
1006 | #endif | |
1007 | { | |
1008 | rotli r20, r32, 62 | |
1009 | PTREGS_PTR(sp, PTREGS_OFFSET_REG(0)) | |
1010 | } | |
1011 | ||
1012 | /* | |
1013 | * Load cache lines 0, 4, 6 and 7, in that order, then use | |
1014 | * the last loaded value, which makes it likely that the other | |
1015 | * cache lines have also loaded, at which point we should be | |
1016 | * able to safely read all the remaining words on those cache | |
1017 | * lines without waiting for the memory subsystem. | |
1018 | */ | |
1019 | pop_reg r0, sp, PTREGS_OFFSET_REG(30) - PTREGS_OFFSET_REG(0) | |
1020 | pop_reg r30, sp, PTREGS_OFFSET_REG(52) - PTREGS_OFFSET_REG(30) | |
1021 | pop_reg_zero r52, r3, sp, PTREGS_OFFSET_CMPEXCH - PTREGS_OFFSET_REG(52) | |
1022 | pop_reg_zero r21, r27, sp, PTREGS_OFFSET_EX1 - PTREGS_OFFSET_CMPEXCH | |
1023 | pop_reg_zero lr, r2, sp, PTREGS_OFFSET_PC - PTREGS_OFFSET_EX1 | |
1024 | { | |
1025 | mtspr CMPEXCH_VALUE, r21 | |
1026 | move r4, zero | |
1027 | } | |
1028 | pop_reg r21, sp, PTREGS_OFFSET_REG(31) - PTREGS_OFFSET_PC | |
1029 | { | |
1030 | mtspr SPR_EX_CONTEXT_K_1, lr | |
051168df | 1031 | IS_KERNEL_EX1(lr, lr) |
18aecc2b CM |
1032 | } |
1033 | { | |
1034 | mtspr SPR_EX_CONTEXT_K_0, r21 | |
1035 | move r5, zero | |
1036 | } | |
1037 | ||
1038 | /* Restore callee-saveds that we actually use. */ | |
1039 | pop_reg_zero r31, r6 | |
1040 | pop_reg_zero r32, r7 | |
1041 | pop_reg_zero r33, r8, sp, PTREGS_OFFSET_REG(29) - PTREGS_OFFSET_REG(33) | |
1042 | ||
1043 | /* | |
1044 | * If we modified other callee-saveds, restore them now. | |
1045 | * This is rare, but could be via ptrace or signal handler. | |
1046 | */ | |
1047 | { | |
1048 | move r9, zero | |
1049 | blbs r20, .Lrestore_callees | |
1050 | } | |
1051 | .Lcontinue_restore_regs: | |
1052 | ||
1053 | /* Check if we're returning from a syscall. */ | |
1054 | { | |
1055 | move r10, zero | |
1056 | bltzt r20, 1f /* no, so go restore callee-save registers */ | |
1057 | } | |
1058 | ||
1059 | /* | |
1060 | * Check if we're returning to userspace. | |
1061 | * Note that if we're not, we don't worry about zeroing everything. | |
1062 | */ | |
1063 | { | |
1064 | addli sp, sp, PTREGS_OFFSET_LR - PTREGS_OFFSET_REG(29) | |
1065 | bnez lr, .Lkernel_return | |
1066 | } | |
1067 | ||
1068 | /* | |
1069 | * On return from syscall, we've restored r0 from pt_regs, but we | |
1070 | * clear the remainder of the caller-saved registers. We could | |
1071 | * restore the syscall arguments, but there's not much point, | |
1072 | * and it ensures user programs aren't trying to use the | |
1073 | * caller-saves if we clear them, as well as avoiding leaking | |
1074 | * kernel pointers into userspace. | |
1075 | */ | |
1076 | pop_reg_zero lr, r11, sp, PTREGS_OFFSET_TP - PTREGS_OFFSET_LR | |
1077 | pop_reg_zero tp, r12, sp, PTREGS_OFFSET_SP - PTREGS_OFFSET_TP | |
1078 | { | |
1079 | ld sp, sp | |
1080 | move r13, zero | |
1081 | move r14, zero | |
1082 | } | |
1083 | { move r15, zero; move r16, zero } | |
1084 | { move r17, zero; move r18, zero } | |
1085 | { move r19, zero; move r20, zero } | |
1086 | { move r21, zero; move r22, zero } | |
1087 | { move r23, zero; move r24, zero } | |
1088 | { move r25, zero; move r26, zero } | |
1089 | ||
1090 | /* Set r1 to errno if we are returning an error, otherwise zero. */ | |
1091 | { | |
1092 | moveli r29, 4096 | |
1093 | sub r1, zero, r0 | |
1094 | } | |
1095 | { | |
1096 | move r28, zero | |
1097 | cmpltu r29, r1, r29 | |
1098 | } | |
1099 | { | |
1100 | mnz r1, r29, r1 | |
1101 | move r29, zero | |
1102 | } | |
1103 | iret | |
1104 | ||
1105 | /* | |
1106 | * Not a syscall, so restore caller-saved registers. | |
1107 | * First kick off loads for cache lines 1-3, which we're touching | |
1108 | * for the first time here. | |
1109 | */ | |
1110 | .align 64 | |
1111 | 1: pop_reg r29, sp, PTREGS_OFFSET_REG(21) - PTREGS_OFFSET_REG(29) | |
1112 | pop_reg r21, sp, PTREGS_OFFSET_REG(13) - PTREGS_OFFSET_REG(21) | |
1113 | pop_reg r13, sp, PTREGS_OFFSET_REG(1) - PTREGS_OFFSET_REG(13) | |
1114 | pop_reg r1 | |
1115 | pop_reg r2 | |
1116 | pop_reg r3 | |
1117 | pop_reg r4 | |
1118 | pop_reg r5 | |
1119 | pop_reg r6 | |
1120 | pop_reg r7 | |
1121 | pop_reg r8 | |
1122 | pop_reg r9 | |
1123 | pop_reg r10 | |
1124 | pop_reg r11 | |
1125 | pop_reg r12, sp, 16 | |
1126 | /* r13 already restored above */ | |
1127 | pop_reg r14 | |
1128 | pop_reg r15 | |
1129 | pop_reg r16 | |
1130 | pop_reg r17 | |
1131 | pop_reg r18 | |
1132 | pop_reg r19 | |
1133 | pop_reg r20, sp, 16 | |
1134 | /* r21 already restored above */ | |
1135 | pop_reg r22 | |
1136 | pop_reg r23 | |
1137 | pop_reg r24 | |
1138 | pop_reg r25 | |
1139 | pop_reg r26 | |
1140 | pop_reg r27 | |
1141 | pop_reg r28, sp, PTREGS_OFFSET_LR - PTREGS_OFFSET_REG(28) | |
1142 | /* r29 already restored above */ | |
1143 | bnez lr, .Lkernel_return | |
1144 | pop_reg lr, sp, PTREGS_OFFSET_TP - PTREGS_OFFSET_LR | |
1145 | pop_reg tp, sp, PTREGS_OFFSET_SP - PTREGS_OFFSET_TP | |
1146 | ld sp, sp | |
1147 | iret | |
1148 | ||
1149 | /* | |
1150 | * We can't restore tp when in kernel mode, since a thread might | |
1151 | * have migrated from another cpu and brought a stale tp value. | |
1152 | */ | |
1153 | .Lkernel_return: | |
1154 | pop_reg lr, sp, PTREGS_OFFSET_SP - PTREGS_OFFSET_LR | |
1155 | ld sp, sp | |
1156 | iret | |
1157 | ||
1158 | /* Restore callee-saved registers from r34 to r51. */ | |
1159 | .Lrestore_callees: | |
1160 | addli sp, sp, PTREGS_OFFSET_REG(34) - PTREGS_OFFSET_REG(29) | |
1161 | pop_reg r34 | |
1162 | pop_reg r35 | |
1163 | pop_reg r36 | |
1164 | pop_reg r37 | |
1165 | pop_reg r38 | |
1166 | pop_reg r39 | |
1167 | pop_reg r40 | |
1168 | pop_reg r41 | |
1169 | pop_reg r42 | |
1170 | pop_reg r43 | |
1171 | pop_reg r44 | |
1172 | pop_reg r45 | |
1173 | pop_reg r46 | |
1174 | pop_reg r47 | |
1175 | pop_reg r48 | |
1176 | pop_reg r49 | |
1177 | pop_reg r50 | |
1178 | pop_reg r51, sp, PTREGS_OFFSET_REG(29) - PTREGS_OFFSET_REG(51) | |
1179 | j .Lcontinue_restore_regs | |
1180 | STD_ENDPROC(interrupt_return) | |
1181 | ||
1182 | /* | |
1183 | * "NMI" interrupts mask ALL interrupts before calling the | |
1184 | * handler, and don't check thread flags, etc., on the way | |
1185 | * back out. In general, the only things we do here for NMIs | |
1186 | * are register save/restore and dataplane kernel-TLB management. | |
1187 | * We don't (for example) deal with start/stop of the sched tick. | |
1188 | */ | |
1189 | .pushsection .text.handle_nmi,"ax" | |
1190 | handle_nmi: | |
1191 | finish_interrupt_save handle_nmi | |
1192 | { | |
1193 | jalr r0 | |
1194 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
1195 | } | |
1196 | FEEDBACK_REENTER(handle_nmi) | |
1197 | { | |
1198 | movei r30, 1 | |
ba678231 | 1199 | cmpeq r31, r0, zero |
18aecc2b CM |
1200 | } |
1201 | j interrupt_return | |
1202 | STD_ENDPROC(handle_nmi) | |
1203 | ||
1204 | /* | |
1205 | * Parallel code for syscalls to handle_interrupt. | |
1206 | */ | |
1207 | .pushsection .text.handle_syscall,"ax" | |
1208 | handle_syscall: | |
1209 | finish_interrupt_save handle_syscall | |
1210 | ||
1211 | /* Enable irqs. */ | |
1212 | TRACE_IRQS_ON | |
1213 | IRQ_ENABLE(r20, r21) | |
1214 | ||
1215 | /* Bump the counter for syscalls made on this tile. */ | |
1216 | moveli r20, hw2_last(irq_stat + IRQ_CPUSTAT_SYSCALL_COUNT_OFFSET) | |
1217 | shl16insli r20, r20, hw1(irq_stat + IRQ_CPUSTAT_SYSCALL_COUNT_OFFSET) | |
1218 | shl16insli r20, r20, hw0(irq_stat + IRQ_CPUSTAT_SYSCALL_COUNT_OFFSET) | |
1219 | add r20, r20, tp | |
1220 | ld4s r21, r20 | |
fc327e26 CM |
1221 | { |
1222 | addi r21, r21, 1 | |
1223 | move r31, sp | |
1224 | } | |
1225 | { | |
1226 | st4 r20, r21 | |
1227 | EXTRACT_THREAD_INFO(r31) | |
1228 | } | |
18aecc2b CM |
1229 | |
1230 | /* Trace syscalls, if requested. */ | |
18aecc2b | 1231 | addi r31, r31, THREAD_INFO_FLAGS_OFFSET |
ef567f25 SM |
1232 | { |
1233 | ld r30, r31 | |
1234 | moveli r32, _TIF_SYSCALL_ENTRY_WORK | |
1235 | } | |
1236 | and r30, r30, r32 | |
18aecc2b CM |
1237 | { |
1238 | addi r30, r31, THREAD_INFO_STATUS_OFFSET - THREAD_INFO_FLAGS_OFFSET | |
1239 | beqzt r30, .Lrestore_syscall_regs | |
1240 | } | |
ef182724 SM |
1241 | { |
1242 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
1243 | jal do_syscall_trace_enter | |
1244 | } | |
18aecc2b CM |
1245 | FEEDBACK_REENTER(handle_syscall) |
1246 | ||
1247 | /* | |
1248 | * We always reload our registers from the stack at this | |
1249 | * point. They might be valid, if we didn't build with | |
1250 | * TRACE_IRQFLAGS, and this isn't a dataplane tile, and we're not | |
1251 | * doing syscall tracing, but there are enough cases now that it | |
1252 | * seems simplest just to do the reload unconditionally. | |
1253 | */ | |
1254 | .Lrestore_syscall_regs: | |
1255 | { | |
1256 | ld r30, r30 | |
1257 | PTREGS_PTR(r11, PTREGS_OFFSET_REG(0)) | |
1258 | } | |
1259 | pop_reg r0, r11 | |
1260 | pop_reg r1, r11 | |
1261 | pop_reg r2, r11 | |
1262 | pop_reg r3, r11 | |
1263 | pop_reg r4, r11 | |
1264 | pop_reg r5, r11, PTREGS_OFFSET_SYSCALL - PTREGS_OFFSET_REG(5) | |
1265 | { | |
1266 | ld TREG_SYSCALL_NR_NAME, r11 | |
1267 | moveli r21, __NR_syscalls | |
1268 | } | |
1269 | ||
1270 | /* Ensure that the syscall number is within the legal range. */ | |
1271 | { | |
1272 | moveli r20, hw2(sys_call_table) | |
570fd501 | 1273 | #ifdef CONFIG_COMPAT |
18aecc2b | 1274 | blbs r30, .Lcompat_syscall |
570fd501 | 1275 | #endif |
18aecc2b CM |
1276 | } |
1277 | { | |
1278 | cmpltu r21, TREG_SYSCALL_NR_NAME, r21 | |
1279 | shl16insli r20, r20, hw1(sys_call_table) | |
1280 | } | |
1281 | { | |
1282 | blbc r21, .Linvalid_syscall | |
1283 | shl16insli r20, r20, hw0(sys_call_table) | |
1284 | } | |
1285 | .Lload_syscall_pointer: | |
1286 | shl3add r20, TREG_SYSCALL_NR_NAME, r20 | |
1287 | ld r20, r20 | |
1288 | ||
1289 | /* Jump to syscall handler. */ | |
1290 | jalr r20 | |
1291 | .Lhandle_syscall_link: /* value of "lr" after "jalr r20" above */ | |
1292 | ||
1293 | /* | |
1294 | * Write our r0 onto the stack so it gets restored instead | |
1295 | * of whatever the user had there before. | |
1296 | * In compat mode, sign-extend r0 before storing it. | |
1297 | */ | |
1298 | { | |
1299 | PTREGS_PTR(r29, PTREGS_OFFSET_REG(0)) | |
1300 | blbct r30, 1f | |
1301 | } | |
1302 | addxi r0, r0, 0 | |
1303 | 1: st r29, r0 | |
1304 | ||
1305 | .Lsyscall_sigreturn_skip: | |
1306 | FEEDBACK_REENTER(handle_syscall) | |
1307 | ||
1308 | /* Do syscall trace again, if requested. */ | |
ef567f25 SM |
1309 | { |
1310 | ld r30, r31 | |
1311 | moveli r32, _TIF_SYSCALL_EXIT_WORK | |
1312 | } | |
1313 | and r0, r30, r32 | |
2858f856 CM |
1314 | { |
1315 | andi r0, r30, _TIF_SINGLESTEP | |
1316 | beqzt r0, 1f | |
1317 | } | |
ef182724 SM |
1318 | { |
1319 | PTREGS_PTR(r0, PTREGS_OFFSET_BASE) | |
1320 | jal do_syscall_trace_exit | |
1321 | } | |
18aecc2b | 1322 | FEEDBACK_REENTER(handle_syscall) |
2858f856 CM |
1323 | andi r0, r30, _TIF_SINGLESTEP |
1324 | ||
1325 | 1: beqzt r0, 2f | |
1326 | ||
1327 | /* Single stepping -- notify ptrace. */ | |
1328 | { | |
1329 | movei r0, SIGTRAP | |
1330 | jal ptrace_notify | |
1331 | } | |
1332 | FEEDBACK_REENTER(handle_syscall) | |
1333 | ||
e1d5c019 CM |
1334 | 2: { |
1335 | movei r30, 0 /* not an NMI */ | |
1336 | j .Lresume_userspace /* jump into middle of interrupt_return */ | |
1337 | } | |
18aecc2b | 1338 | |
570fd501 | 1339 | #ifdef CONFIG_COMPAT |
18aecc2b CM |
1340 | .Lcompat_syscall: |
1341 | /* | |
1342 | * Load the base of the compat syscall table in r20, and | |
1343 | * range-check the syscall number (duplicated from 64-bit path). | |
1344 | * Sign-extend all the user's passed arguments to make them consistent. | |
1345 | * Also save the original "r(n)" values away in "r(11+n)" in | |
1346 | * case the syscall table entry wants to validate them. | |
1347 | */ | |
1348 | moveli r20, hw2(compat_sys_call_table) | |
1349 | { | |
1350 | cmpltu r21, TREG_SYSCALL_NR_NAME, r21 | |
1351 | shl16insli r20, r20, hw1(compat_sys_call_table) | |
1352 | } | |
1353 | { | |
1354 | blbc r21, .Linvalid_syscall | |
1355 | shl16insli r20, r20, hw0(compat_sys_call_table) | |
1356 | } | |
1357 | { move r11, r0; addxi r0, r0, 0 } | |
1358 | { move r12, r1; addxi r1, r1, 0 } | |
1359 | { move r13, r2; addxi r2, r2, 0 } | |
1360 | { move r14, r3; addxi r3, r3, 0 } | |
1361 | { move r15, r4; addxi r4, r4, 0 } | |
1362 | { move r16, r5; addxi r5, r5, 0 } | |
1363 | j .Lload_syscall_pointer | |
570fd501 | 1364 | #endif |
18aecc2b CM |
1365 | |
1366 | .Linvalid_syscall: | |
1367 | /* Report an invalid syscall back to the user program */ | |
1368 | { | |
1369 | PTREGS_PTR(r29, PTREGS_OFFSET_REG(0)) | |
1370 | movei r28, -ENOSYS | |
1371 | } | |
1372 | st r29, r28 | |
e1d5c019 CM |
1373 | { |
1374 | movei r30, 0 /* not an NMI */ | |
1375 | j .Lresume_userspace /* jump into middle of interrupt_return */ | |
1376 | } | |
18aecc2b CM |
1377 | STD_ENDPROC(handle_syscall) |
1378 | ||
1379 | /* Return the address for oprofile to suppress in backtraces. */ | |
1380 | STD_ENTRY_SECTION(handle_syscall_link_address, .text.handle_syscall) | |
1381 | lnk r0 | |
1382 | { | |
1383 | addli r0, r0, .Lhandle_syscall_link - . | |
1384 | jrp lr | |
1385 | } | |
1386 | STD_ENDPROC(handle_syscall_link_address) | |
1387 | ||
1388 | STD_ENTRY(ret_from_fork) | |
1389 | jal sim_notify_fork | |
1390 | jal schedule_tail | |
1391 | FEEDBACK_REENTER(ret_from_fork) | |
e1d5c019 CM |
1392 | { |
1393 | movei r30, 0 /* not an NMI */ | |
1394 | j .Lresume_userspace /* jump into middle of interrupt_return */ | |
1395 | } | |
18aecc2b CM |
1396 | STD_ENDPROC(ret_from_fork) |
1397 | ||
0f8b9838 CM |
1398 | STD_ENTRY(ret_from_kernel_thread) |
1399 | jal sim_notify_fork | |
1400 | jal schedule_tail | |
1401 | FEEDBACK_REENTER(ret_from_fork) | |
1402 | { | |
1403 | move r0, r31 | |
1404 | jalr r30 | |
1405 | } | |
1406 | FEEDBACK_REENTER(ret_from_kernel_thread) | |
1407 | { | |
1408 | movei r30, 0 /* not an NMI */ | |
1409 | j .Lresume_userspace /* jump into middle of interrupt_return */ | |
1410 | } | |
1411 | STD_ENDPROC(ret_from_kernel_thread) | |
1412 | ||
18aecc2b CM |
1413 | /* Various stub interrupt handlers and syscall handlers */ |
1414 | ||
1415 | STD_ENTRY_LOCAL(_kernel_double_fault) | |
1416 | mfspr r1, SPR_EX_CONTEXT_K_0 | |
1417 | move r2, lr | |
1418 | move r3, sp | |
1419 | move r4, r52 | |
1420 | addi sp, sp, -C_ABI_SAVE_AREA_SIZE | |
1421 | j kernel_double_fault | |
1422 | STD_ENDPROC(_kernel_double_fault) | |
1423 | ||
1424 | STD_ENTRY_LOCAL(bad_intr) | |
1425 | mfspr r2, SPR_EX_CONTEXT_K_0 | |
1426 | panic "Unhandled interrupt %#x: PC %#lx" | |
1427 | STD_ENDPROC(bad_intr) | |
1428 | ||
18aecc2b CM |
1429 | /* |
1430 | * Special-case sigreturn to not write r0 to the stack on return. | |
1431 | * This is technically more efficient, but it also avoids difficulties | |
1432 | * in the 64-bit OS when handling 32-bit compat code, since we must not | |
1433 | * sign-extend r0 for the sigreturn return-value case. | |
1434 | */ | |
1435 | #define PTREGS_SYSCALL_SIGRETURN(x, reg) \ | |
1436 | STD_ENTRY(_##x); \ | |
1437 | addli lr, lr, .Lsyscall_sigreturn_skip - .Lhandle_syscall_link; \ | |
1438 | { \ | |
1439 | PTREGS_PTR(reg, PTREGS_OFFSET_BASE); \ | |
1440 | j x \ | |
1441 | }; \ | |
1442 | STD_ENDPROC(_##x) | |
1443 | ||
18aecc2b CM |
1444 | PTREGS_SYSCALL_SIGRETURN(sys_rt_sigreturn, r0) |
1445 | #ifdef CONFIG_COMPAT | |
18aecc2b CM |
1446 | PTREGS_SYSCALL_SIGRETURN(compat_sys_rt_sigreturn, r0) |
1447 | #endif | |
1448 | ||
6b14e419 | 1449 | /* Save additional callee-saves to pt_regs and jump to standard function. */ |
18aecc2b CM |
1450 | STD_ENTRY(_sys_clone) |
1451 | push_extra_callee_saves r4 | |
1452 | j sys_clone | |
1453 | STD_ENDPROC(_sys_clone) | |
1454 | ||
2f9ac29e CM |
1455 | /* |
1456 | * Recover r3, r2, r1 and r0 here saved by unalign fast vector. | |
1457 | * The vector area limit is 32 bundles, so we handle the reload here. | |
1458 | * r0, r1, r2 are in thread_info from low to high memory in order. | |
1459 | * r3 points to location the original r3 was saved. | |
1460 | * We put this code in the __HEAD section so it can be reached | |
1461 | * via a conditional branch from the fast path. | |
1462 | */ | |
1463 | __HEAD | |
1464 | hand_unalign_slow: | |
1465 | andi sp, sp, ~1 | |
1466 | hand_unalign_slow_badsp: | |
1467 | addi r3, r3, -(3 * 8) | |
1468 | ld_add r0, r3, 8 | |
1469 | ld_add r1, r3, 8 | |
1470 | ld r2, r3 | |
1471 | hand_unalign_slow_nonuser: | |
1472 | mfspr r3, SPR_SYSTEM_SAVE_K_1 | |
1473 | __int_hand INT_UNALIGN_DATA, UNALIGN_DATA_SLOW, int_unalign | |
1474 | ||
1475 | /* The unaligned data support needs to read all the registers. */ | |
18aecc2b CM |
1476 | int_unalign: |
1477 | push_extra_callee_saves r0 | |
2f9ac29e CM |
1478 | j do_unaligned |
1479 | ENDPROC(hand_unalign_slow) | |
18aecc2b | 1480 | |
e1723538 CM |
1481 | /* Fill the return address stack with nonzero entries. */ |
1482 | STD_ENTRY(fill_ra_stack) | |
1483 | { | |
1484 | move r0, lr | |
1485 | jal 1f | |
1486 | } | |
1487 | 1: jal 2f | |
1488 | 2: jal 3f | |
1489 | 3: jal 4f | |
1490 | 4: jrp r0 | |
1491 | STD_ENDPROC(fill_ra_stack) | |
1492 | ||
2f9ac29e CM |
1493 | .macro int_hand vecnum, vecname, c_routine, processing=handle_interrupt |
1494 | .org (\vecnum << 8) | |
1495 | __int_hand \vecnum, \vecname, \c_routine, \processing | |
1496 | .endm | |
1497 | ||
acbde1db CM |
1498 | /* Include .intrpt array of interrupt vectors */ |
1499 | .section ".intrpt", "ax" | |
1500 | .global intrpt_start | |
1501 | intrpt_start: | |
18aecc2b | 1502 | |
8e3441eb ZL |
1503 | #ifndef CONFIG_USE_PMC |
1504 | #define handle_perf_interrupt bad_intr | |
1505 | #endif | |
18aecc2b CM |
1506 | |
1507 | #ifndef CONFIG_HARDWALL | |
1508 | #define do_hardwall_trap bad_intr | |
1509 | #endif | |
1510 | ||
a714ffff | 1511 | int_hand INT_MEM_ERROR, MEM_ERROR, do_trap |
18aecc2b CM |
1512 | int_hand INT_SINGLE_STEP_3, SINGLE_STEP_3, bad_intr |
1513 | #if CONFIG_KERNEL_PL == 2 | |
1514 | int_hand INT_SINGLE_STEP_2, SINGLE_STEP_2, gx_singlestep_handle | |
1515 | int_hand INT_SINGLE_STEP_1, SINGLE_STEP_1, bad_intr | |
1516 | #else | |
1517 | int_hand INT_SINGLE_STEP_2, SINGLE_STEP_2, bad_intr | |
1518 | int_hand INT_SINGLE_STEP_1, SINGLE_STEP_1, gx_singlestep_handle | |
1519 | #endif | |
1520 | int_hand INT_SINGLE_STEP_0, SINGLE_STEP_0, bad_intr | |
1521 | int_hand INT_IDN_COMPLETE, IDN_COMPLETE, bad_intr | |
1522 | int_hand INT_UDN_COMPLETE, UDN_COMPLETE, bad_intr | |
1523 | int_hand INT_ITLB_MISS, ITLB_MISS, do_page_fault | |
1524 | int_hand INT_ILL, ILL, do_trap | |
1525 | int_hand INT_GPV, GPV, do_trap | |
1526 | int_hand INT_IDN_ACCESS, IDN_ACCESS, do_trap | |
1527 | int_hand INT_UDN_ACCESS, UDN_ACCESS, do_trap | |
1528 | int_hand INT_SWINT_3, SWINT_3, do_trap | |
1529 | int_hand INT_SWINT_2, SWINT_2, do_trap | |
1530 | int_hand INT_SWINT_1, SWINT_1, SYSCALL, handle_syscall | |
1531 | int_hand INT_SWINT_0, SWINT_0, do_trap | |
1532 | int_hand INT_ILL_TRANS, ILL_TRANS, do_trap | |
2f9ac29e | 1533 | int_hand_unalign_fast INT_UNALIGN_DATA, UNALIGN_DATA |
18aecc2b CM |
1534 | int_hand INT_DTLB_MISS, DTLB_MISS, do_page_fault |
1535 | int_hand INT_DTLB_ACCESS, DTLB_ACCESS, do_page_fault | |
b8ace083 | 1536 | int_hand INT_IDN_FIREWALL, IDN_FIREWALL, do_hardwall_trap |
18aecc2b CM |
1537 | int_hand INT_UDN_FIREWALL, UDN_FIREWALL, do_hardwall_trap |
1538 | int_hand INT_TILE_TIMER, TILE_TIMER, do_timer_interrupt | |
1539 | int_hand INT_IDN_TIMER, IDN_TIMER, bad_intr | |
1540 | int_hand INT_UDN_TIMER, UDN_TIMER, bad_intr | |
1541 | int_hand INT_IDN_AVAIL, IDN_AVAIL, bad_intr | |
1542 | int_hand INT_UDN_AVAIL, UDN_AVAIL, bad_intr | |
1543 | int_hand INT_IPI_3, IPI_3, bad_intr | |
1544 | #if CONFIG_KERNEL_PL == 2 | |
1545 | int_hand INT_IPI_2, IPI_2, tile_dev_intr | |
1546 | int_hand INT_IPI_1, IPI_1, bad_intr | |
1547 | #else | |
1548 | int_hand INT_IPI_2, IPI_2, bad_intr | |
1549 | int_hand INT_IPI_1, IPI_1, tile_dev_intr | |
1550 | #endif | |
1551 | int_hand INT_IPI_0, IPI_0, bad_intr | |
1552 | int_hand INT_PERF_COUNT, PERF_COUNT, \ | |
8e3441eb | 1553 | handle_perf_interrupt, handle_nmi |
18aecc2b | 1554 | int_hand INT_AUX_PERF_COUNT, AUX_PERF_COUNT, \ |
8e3441eb | 1555 | handle_perf_interrupt, handle_nmi |
18aecc2b CM |
1556 | int_hand INT_INTCTRL_3, INTCTRL_3, bad_intr |
1557 | #if CONFIG_KERNEL_PL == 2 | |
1558 | dc_dispatch INT_INTCTRL_2, INTCTRL_2 | |
1559 | int_hand INT_INTCTRL_1, INTCTRL_1, bad_intr | |
1560 | #else | |
1561 | int_hand INT_INTCTRL_2, INTCTRL_2, bad_intr | |
1562 | dc_dispatch INT_INTCTRL_1, INTCTRL_1 | |
1563 | #endif | |
1564 | int_hand INT_INTCTRL_0, INTCTRL_0, bad_intr | |
1565 | int_hand INT_MESSAGE_RCV_DWNCL, MESSAGE_RCV_DWNCL, \ | |
1566 | hv_message_intr | |
1567 | int_hand INT_DEV_INTR_DWNCL, DEV_INTR_DWNCL, bad_intr | |
1568 | int_hand INT_I_ASID, I_ASID, bad_intr | |
1569 | int_hand INT_D_ASID, D_ASID, bad_intr | |
1570 | int_hand INT_DOUBLE_FAULT, DOUBLE_FAULT, do_trap | |
1571 | ||
1572 | /* Synthetic interrupt delivered only by the simulator */ | |
1573 | int_hand INT_BREAKPOINT, BREAKPOINT, do_breakpoint |