projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge git://git.kernel.org/pub/scm/linux/kernel/git/davem/net
[deliverable/linux.git] / arch / powerpc / kernel / entry_64.S
1 /*
2 * PowerPC version
3 * Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
4 * Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
5 * Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
6 * Adapted for Power Macintosh by Paul Mackerras.
7 * Low-level exception handlers and MMU support
8 * rewritten by Paul Mackerras.
9 * Copyright (C) 1996 Paul Mackerras.
10 * MPC8xx modifications Copyright (C) 1997 Dan Malek (dmalek@jlc.net).
11 *
12 * This file contains the system call entry code, context switch
13 * code, and exception/interrupt return code for PowerPC.
14 *
15 * This program is free software; you can redistribute it and/or
16 * modify it under the terms of the GNU General Public License
17 * as published by the Free Software Foundation; either version
18 * 2 of the License, or (at your option) any later version.
19 */
20
21 #include <linux/errno.h>
22 #include <linux/err.h>
23 #include <linux/magic.h>
24 #include <asm/unistd.h>
25 #include <asm/processor.h>
26 #include <asm/page.h>
27 #include <asm/mmu.h>
28 #include <asm/thread_info.h>
29 #include <asm/ppc_asm.h>
30 #include <asm/asm-offsets.h>
31 #include <asm/cputable.h>
32 #include <asm/firmware.h>
33 #include <asm/bug.h>
34 #include <asm/ptrace.h>
35 #include <asm/irqflags.h>
36 #include <asm/ftrace.h>
37 #include <asm/hw_irq.h>
38 #include <asm/context_tracking.h>
39 #include <asm/tm.h>
40 #include <asm/ppc-opcode.h>
41
42 /*
43 * System calls.
44 */
45 .section ".toc","aw"
46 SYS_CALL_TABLE:
47 .tc sys_call_table[TC],sys_call_table
48
49 /* This value is used to mark exception frames on the stack. */
50 exception_marker:
51 .tc ID_EXC_MARKER[TC],STACK_FRAME_REGS_MARKER
52
53 .section ".text"
54 .align 7
55
56 .globl system_call_common
57 system_call_common:
58 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
59 BEGIN_FTR_SECTION
60 extrdi. r10, r12, 1, (63-MSR_TS_T_LG) /* transaction active? */
61 bne tabort_syscall
62 END_FTR_SECTION_IFSET(CPU_FTR_TM)
63 #endif
64 andi. r10,r12,MSR_PR
65 mr r10,r1
66 addi r1,r1,-INT_FRAME_SIZE
67 beq- 1f
68 ld r1,PACAKSAVE(r13)
69 1: std r10,0(r1)
70 std r11,_NIP(r1)
71 std r12,_MSR(r1)
72 std r0,GPR0(r1)
73 std r10,GPR1(r1)
74 beq 2f /* if from kernel mode */
75 ACCOUNT_CPU_USER_ENTRY(r13, r10, r11)
76 2: std r2,GPR2(r1)
77 std r3,GPR3(r1)
78 mfcr r2
79 std r4,GPR4(r1)
80 std r5,GPR5(r1)
81 std r6,GPR6(r1)
82 std r7,GPR7(r1)
83 std r8,GPR8(r1)
84 li r11,0
85 std r11,GPR9(r1)
86 std r11,GPR10(r1)
87 std r11,GPR11(r1)
88 std r11,GPR12(r1)
89 std r11,_XER(r1)
90 std r11,_CTR(r1)
91 std r9,GPR13(r1)
92 mflr r10
93 /*
94 * This clears CR0.SO (bit 28), which is the error indication on
95 * return from this system call.
96 */
97 rldimi r2,r11,28,(63-28)
98 li r11,0xc01
99 std r10,_LINK(r1)
100 std r11,_TRAP(r1)
101 std r3,ORIG_GPR3(r1)
102 std r2,_CCR(r1)
103 ld r2,PACATOC(r13)
104 addi r9,r1,STACK_FRAME_OVERHEAD
105 ld r11,exception_marker@toc(r2)
106 std r11,-16(r9) /* "regshere" marker */
107 #if defined(CONFIG_VIRT_CPU_ACCOUNTING_NATIVE) && defined(CONFIG_PPC_SPLPAR)
108 BEGIN_FW_FTR_SECTION
109 beq 33f
110 /* if from user, see if there are any DTL entries to process */
111 ld r10,PACALPPACAPTR(r13) /* get ptr to VPA */
112 ld r11,PACA_DTL_RIDX(r13) /* get log read index */
113 addi r10,r10,LPPACA_DTLIDX
114 LDX_BE r10,0,r10 /* get log write index */
115 cmpd cr1,r11,r10
116 beq+ cr1,33f
117 bl accumulate_stolen_time
118 REST_GPR(0,r1)
119 REST_4GPRS(3,r1)
120 REST_2GPRS(7,r1)
121 addi r9,r1,STACK_FRAME_OVERHEAD
122 33:
123 END_FW_FTR_SECTION_IFSET(FW_FEATURE_SPLPAR)
124 #endif /* CONFIG_VIRT_CPU_ACCOUNTING_NATIVE && CONFIG_PPC_SPLPAR */
125
126 /*
127 * A syscall should always be called with interrupts enabled
128 * so we just unconditionally hard-enable here. When some kind
129 * of irq tracing is used, we additionally check that condition
130 * is correct
131 */
132 #if defined(CONFIG_TRACE_IRQFLAGS) && defined(CONFIG_BUG)
133 lbz r10,PACASOFTIRQEN(r13)
134 xori r10,r10,1
135 1: tdnei r10,0
136 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
137 #endif
138
139 #ifdef CONFIG_PPC_BOOK3E
140 wrteei 1
141 #else
142 ld r11,PACAKMSR(r13)
143 ori r11,r11,MSR_EE
144 mtmsrd r11,1
145 #endif /* CONFIG_PPC_BOOK3E */
146
147 /* We do need to set SOFTE in the stack frame or the return
148 * from interrupt will be painful
149 */
150 li r10,1
151 std r10,SOFTE(r1)
152
153 CURRENT_THREAD_INFO(r11, r1)
154 ld r10,TI_FLAGS(r11)
155 andi. r11,r10,_TIF_SYSCALL_DOTRACE
156 bne syscall_dotrace /* does not return */
157 cmpldi 0,r0,NR_syscalls
158 bge- syscall_enosys
159
160 system_call: /* label this so stack traces look sane */
161 /*
162 * Need to vector to 32 Bit or default sys_call_table here,
163 * based on caller's run-mode / personality.
164 */
165 ld r11,SYS_CALL_TABLE@toc(2)
166 andi. r10,r10,_TIF_32BIT
167 beq 15f
168 addi r11,r11,8 /* use 32-bit syscall entries */
169 clrldi r3,r3,32
170 clrldi r4,r4,32
171 clrldi r5,r5,32
172 clrldi r6,r6,32
173 clrldi r7,r7,32
174 clrldi r8,r8,32
175 15:
176 slwi r0,r0,4
177 ldx r12,r11,r0 /* Fetch system call handler [ptr] */
178 mtctr r12
179 bctrl /* Call handler */
180
181 .Lsyscall_exit:
182 std r3,RESULT(r1)
183 CURRENT_THREAD_INFO(r12, r1)
184
185 ld r8,_MSR(r1)
186 #ifdef CONFIG_PPC_BOOK3S
187 /* No MSR:RI on BookE */
188 andi. r10,r8,MSR_RI
189 beq- unrecov_restore
190 #endif
191 /*
192 * Disable interrupts so current_thread_info()->flags can't change,
193 * and so that we don't get interrupted after loading SRR0/1.
194 */
195 #ifdef CONFIG_PPC_BOOK3E
196 wrteei 0
197 #else
198 ld r10,PACAKMSR(r13)
199 /*
200 * For performance reasons we clear RI the same time that we
201 * clear EE. We only need to clear RI just before we restore r13
202 * below, but batching it with EE saves us one expensive mtmsrd call.
203 * We have to be careful to restore RI if we branch anywhere from
204 * here (eg syscall_exit_work).
205 */
206 li r9,MSR_RI
207 andc r11,r10,r9
208 mtmsrd r11,1
209 #endif /* CONFIG_PPC_BOOK3E */
210
211 ld r9,TI_FLAGS(r12)
212 li r11,-MAX_ERRNO
213 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP|_TIF_USER_WORK_MASK|_TIF_PERSYSCALL_MASK)
214 bne- syscall_exit_work
215
216 andi. r0,r8,MSR_FP
217 beq 2f
218 #ifdef CONFIG_ALTIVEC
219 andis. r0,r8,MSR_VEC@h
220 bne 3f
221 #endif
222 2: addi r3,r1,STACK_FRAME_OVERHEAD
223 #ifdef CONFIG_PPC_BOOK3S
224 mtmsrd r10,1 /* Restore RI */
225 #endif
226 bl restore_math
227 #ifdef CONFIG_PPC_BOOK3S
228 ld r10,PACAKMSR(r13)
229 li r9,MSR_RI
230 andc r11,r10,r9 /* Re-clear RI */
231 mtmsrd r11,1
232 #endif
233 ld r8,_MSR(r1)
234 ld r3,RESULT(r1)
235 li r11,-MAX_ERRNO
236
237 3: cmpld r3,r11
238 ld r5,_CCR(r1)
239 bge- syscall_error
240 .Lsyscall_error_cont:
241 ld r7,_NIP(r1)
242 BEGIN_FTR_SECTION
243 stdcx. r0,0,r1 /* to clear the reservation */
244 END_FTR_SECTION_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
245 andi. r6,r8,MSR_PR
246 ld r4,_LINK(r1)
247
248 beq- 1f
249 ACCOUNT_CPU_USER_EXIT(r13, r11, r12)
250
251 BEGIN_FTR_SECTION
252 HMT_MEDIUM_LOW
253 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
254
255 ld r13,GPR13(r1) /* only restore r13 if returning to usermode */
256 1: ld r2,GPR2(r1)
257 ld r1,GPR1(r1)
258 mtlr r4
259 mtcr r5
260 mtspr SPRN_SRR0,r7
261 mtspr SPRN_SRR1,r8
262 RFI
263 b . /* prevent speculative execution */
264
265 syscall_error:
266 oris r5,r5,0x1000 /* Set SO bit in CR */
267 neg r3,r3
268 std r5,_CCR(r1)
269 b .Lsyscall_error_cont
270
271 /* Traced system call support */
272 syscall_dotrace:
273 bl save_nvgprs
274 addi r3,r1,STACK_FRAME_OVERHEAD
275 bl do_syscall_trace_enter
276
277 /*
278 * We use the return value of do_syscall_trace_enter() as the syscall
279 * number. If the syscall was rejected for any reason do_syscall_trace_enter()
280 * returns an invalid syscall number and the test below against
281 * NR_syscalls will fail.
282 */
283 mr r0,r3
284
285 /* Restore argument registers just clobbered and/or possibly changed. */
286 ld r3,GPR3(r1)
287 ld r4,GPR4(r1)
288 ld r5,GPR5(r1)
289 ld r6,GPR6(r1)
290 ld r7,GPR7(r1)
291 ld r8,GPR8(r1)
292
293 /* Repopulate r9 and r10 for the system_call path */
294 addi r9,r1,STACK_FRAME_OVERHEAD
295 CURRENT_THREAD_INFO(r10, r1)
296 ld r10,TI_FLAGS(r10)
297
298 cmpldi r0,NR_syscalls
299 blt+ system_call
300
301 /* Return code is already in r3 thanks to do_syscall_trace_enter() */
302 b .Lsyscall_exit
303
304
305 syscall_enosys:
306 li r3,-ENOSYS
307 b .Lsyscall_exit
308
309 syscall_exit_work:
310 #ifdef CONFIG_PPC_BOOK3S
311 mtmsrd r10,1 /* Restore RI */
312 #endif
313 /* If TIF_RESTOREALL is set, don't scribble on either r3 or ccr.
314 If TIF_NOERROR is set, just save r3 as it is. */
315
316 andi. r0,r9,_TIF_RESTOREALL
317 beq+ 0f
318 REST_NVGPRS(r1)
319 b 2f
320 0: cmpld r3,r11 /* r11 is -MAX_ERRNO */
321 blt+ 1f
322 andi. r0,r9,_TIF_NOERROR
323 bne- 1f
324 ld r5,_CCR(r1)
325 neg r3,r3
326 oris r5,r5,0x1000 /* Set SO bit in CR */
327 std r5,_CCR(r1)
328 1: std r3,GPR3(r1)
329 2: andi. r0,r9,(_TIF_PERSYSCALL_MASK)
330 beq 4f
331
332 /* Clear per-syscall TIF flags if any are set. */
333
334 li r11,_TIF_PERSYSCALL_MASK
335 addi r12,r12,TI_FLAGS
336 3: ldarx r10,0,r12
337 andc r10,r10,r11
338 stdcx. r10,0,r12
339 bne- 3b
340 subi r12,r12,TI_FLAGS
341
342 4: /* Anything else left to do? */
343 BEGIN_FTR_SECTION
344 lis r3,INIT_PPR@highest /* Set thread.ppr = 3 */
345 ld r10,PACACURRENT(r13)
346 sldi r3,r3,32 /* bits 11-13 are used for ppr */
347 std r3,TASKTHREADPPR(r10)
348 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
349
350 andi. r0,r9,(_TIF_SYSCALL_DOTRACE|_TIF_SINGLESTEP)
351 beq ret_from_except_lite
352
353 /* Re-enable interrupts */
354 #ifdef CONFIG_PPC_BOOK3E
355 wrteei 1
356 #else
357 ld r10,PACAKMSR(r13)
358 ori r10,r10,MSR_EE
359 mtmsrd r10,1
360 #endif /* CONFIG_PPC_BOOK3E */
361
362 bl save_nvgprs
363 addi r3,r1,STACK_FRAME_OVERHEAD
364 bl do_syscall_trace_leave
365 b ret_from_except
366
367 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
368 tabort_syscall:
369 /* Firstly we need to enable TM in the kernel */
370 mfmsr r10
371 li r13, 1
372 rldimi r10, r13, MSR_TM_LG, 63-MSR_TM_LG
373 mtmsrd r10, 0
374
375 /* tabort, this dooms the transaction, nothing else */
376 li r13, (TM_CAUSE_SYSCALL|TM_CAUSE_PERSISTENT)
377 TABORT(R13)
378
379 /*
380 * Return directly to userspace. We have corrupted user register state,
381 * but userspace will never see that register state. Execution will
382 * resume after the tbegin of the aborted transaction with the
383 * checkpointed register state.
384 */
385 li r13, MSR_RI
386 andc r10, r10, r13
387 mtmsrd r10, 1
388 mtspr SPRN_SRR0, r11
389 mtspr SPRN_SRR1, r12
390
391 rfid
392 b . /* prevent speculative execution */
393 #endif
394
395 /* Save non-volatile GPRs, if not already saved. */
396 _GLOBAL(save_nvgprs)
397 ld r11,_TRAP(r1)
398 andi. r0,r11,1
399 beqlr-
400 SAVE_NVGPRS(r1)
401 clrrdi r0,r11,1
402 std r0,_TRAP(r1)
403 blr
404
405
406 /*
407 * The sigsuspend and rt_sigsuspend system calls can call do_signal
408 * and thus put the process into the stopped state where we might
409 * want to examine its user state with ptrace. Therefore we need
410 * to save all the nonvolatile registers (r14 - r31) before calling
411 * the C code. Similarly, fork, vfork and clone need the full
412 * register state on the stack so that it can be copied to the child.
413 */
414
415 _GLOBAL(ppc_fork)
416 bl save_nvgprs
417 bl sys_fork
418 b .Lsyscall_exit
419
420 _GLOBAL(ppc_vfork)
421 bl save_nvgprs
422 bl sys_vfork
423 b .Lsyscall_exit
424
425 _GLOBAL(ppc_clone)
426 bl save_nvgprs
427 bl sys_clone
428 b .Lsyscall_exit
429
430 _GLOBAL(ppc32_swapcontext)
431 bl save_nvgprs
432 bl compat_sys_swapcontext
433 b .Lsyscall_exit
434
435 _GLOBAL(ppc64_swapcontext)
436 bl save_nvgprs
437 bl sys_swapcontext
438 b .Lsyscall_exit
439
440 _GLOBAL(ppc_switch_endian)
441 bl save_nvgprs
442 bl sys_switch_endian
443 b .Lsyscall_exit
444
445 _GLOBAL(ret_from_fork)
446 bl schedule_tail
447 REST_NVGPRS(r1)
448 li r3,0
449 b .Lsyscall_exit
450
451 _GLOBAL(ret_from_kernel_thread)
452 bl schedule_tail
453 REST_NVGPRS(r1)
454 mtlr r14
455 mr r3,r15
456 #ifdef PPC64_ELF_ABI_v2
457 mr r12,r14
458 #endif
459 blrl
460 li r3,0
461 b .Lsyscall_exit
462
463 /*
464 * This routine switches between two different tasks. The process
465 * state of one is saved on its kernel stack. Then the state
466 * of the other is restored from its kernel stack. The memory
467 * management hardware is updated to the second process's state.
468 * Finally, we can return to the second process, via ret_from_except.
469 * On entry, r3 points to the THREAD for the current task, r4
470 * points to the THREAD for the new task.
471 *
472 * Note: there are two ways to get to the "going out" portion
473 * of this code; either by coming in via the entry (_switch)
474 * or via "fork" which must set up an environment equivalent
475 * to the "_switch" path. If you change this you'll have to change
476 * the fork code also.
477 *
478 * The code which creates the new task context is in 'copy_thread'
479 * in arch/powerpc/kernel/process.c
480 */
481 .align 7
482 _GLOBAL(_switch)
483 mflr r0
484 std r0,16(r1)
485 stdu r1,-SWITCH_FRAME_SIZE(r1)
486 /* r3-r13 are caller saved -- Cort */
487 SAVE_8GPRS(14, r1)
488 SAVE_10GPRS(22, r1)
489 std r0,_NIP(r1) /* Return to switch caller */
490 mfcr r23
491 std r23,_CCR(r1)
492 std r1,KSP(r3) /* Set old stack pointer */
493
494 #ifdef CONFIG_SMP
495 /* We need a sync somewhere here to make sure that if the
496 * previous task gets rescheduled on another CPU, it sees all
497 * stores it has performed on this one.
498 */
499 sync
500 #endif /* CONFIG_SMP */
501
502 /*
503 * If we optimise away the clear of the reservation in system
504 * calls because we know the CPU tracks the address of the
505 * reservation, then we need to clear it here to cover the
506 * case that the kernel context switch path has no larx
507 * instructions.
508 */
509 BEGIN_FTR_SECTION
510 ldarx r6,0,r1
511 END_FTR_SECTION_IFSET(CPU_FTR_STCX_CHECKS_ADDRESS)
512
513 BEGIN_FTR_SECTION
514 /*
515 * A cp_abort (copy paste abort) here ensures that when context switching, a
516 * copy from one process can't leak into the paste of another.
517 */
518 PPC_CP_ABORT
519 END_FTR_SECTION_IFSET(CPU_FTR_ARCH_300)
520
521 #ifdef CONFIG_PPC_BOOK3S
522 /* Cancel all explict user streams as they will have no use after context
523 * switch and will stop the HW from creating streams itself
524 */
525 DCBT_STOP_ALL_STREAM_IDS(r6)
526 #endif
527
528 addi r6,r4,-THREAD /* Convert THREAD to 'current' */
529 std r6,PACACURRENT(r13) /* Set new 'current' */
530
531 ld r8,KSP(r4) /* new stack pointer */
532 #ifdef CONFIG_PPC_STD_MMU_64
533 BEGIN_MMU_FTR_SECTION
534 b 2f
535 END_MMU_FTR_SECTION_IFSET(MMU_FTR_RADIX)
536 BEGIN_FTR_SECTION
537 clrrdi r6,r8,28 /* get its ESID */
538 clrrdi r9,r1,28 /* get current sp ESID */
539 FTR_SECTION_ELSE
540 clrrdi r6,r8,40 /* get its 1T ESID */
541 clrrdi r9,r1,40 /* get current sp 1T ESID */
542 ALT_MMU_FTR_SECTION_END_IFCLR(MMU_FTR_1T_SEGMENT)
543 clrldi. r0,r6,2 /* is new ESID c00000000? */
544 cmpd cr1,r6,r9 /* or is new ESID the same as current ESID? */
545 cror eq,4*cr1+eq,eq
546 beq 2f /* if yes, don't slbie it */
547
548 /* Bolt in the new stack SLB entry */
549 ld r7,KSP_VSID(r4) /* Get new stack's VSID */
550 oris r0,r6,(SLB_ESID_V)@h
551 ori r0,r0,(SLB_NUM_BOLTED-1)@l
552 BEGIN_FTR_SECTION
553 li r9,MMU_SEGSIZE_1T /* insert B field */
554 oris r6,r6,(MMU_SEGSIZE_1T << SLBIE_SSIZE_SHIFT)@h
555 rldimi r7,r9,SLB_VSID_SSIZE_SHIFT,0
556 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
557
558 /* Update the last bolted SLB. No write barriers are needed
559 * here, provided we only update the current CPU's SLB shadow
560 * buffer.
561 */
562 ld r9,PACA_SLBSHADOWPTR(r13)
563 li r12,0
564 std r12,SLBSHADOW_STACKESID(r9) /* Clear ESID */
565 li r12,SLBSHADOW_STACKVSID
566 STDX_BE r7,r12,r9 /* Save VSID */
567 li r12,SLBSHADOW_STACKESID
568 STDX_BE r0,r12,r9 /* Save ESID */
569
570 /* No need to check for MMU_FTR_NO_SLBIE_B here, since when
571 * we have 1TB segments, the only CPUs known to have the errata
572 * only support less than 1TB of system memory and we'll never
573 * actually hit this code path.
574 */
575
576 slbie r6
577 slbie r6 /* Workaround POWER5 < DD2.1 issue */
578 slbmte r7,r0
579 isync
580 2:
581 #endif /* CONFIG_PPC_STD_MMU_64 */
582
583 CURRENT_THREAD_INFO(r7, r8) /* base of new stack */
584 /* Note: this uses SWITCH_FRAME_SIZE rather than INT_FRAME_SIZE
585 because we don't need to leave the 288-byte ABI gap at the
586 top of the kernel stack. */
587 addi r7,r7,THREAD_SIZE-SWITCH_FRAME_SIZE
588
589 mr r1,r8 /* start using new stack pointer */
590 std r7,PACAKSAVE(r13)
591
592 ld r6,_CCR(r1)
593 mtcrf 0xFF,r6
594
595 /* r3-r13 are destroyed -- Cort */
596 REST_8GPRS(14, r1)
597 REST_10GPRS(22, r1)
598
599 /* convert old thread to its task_struct for return value */
600 addi r3,r3,-THREAD
601 ld r7,_NIP(r1) /* Return to _switch caller in new task */
602 mtlr r7
603 addi r1,r1,SWITCH_FRAME_SIZE
604 blr
605
606 .align 7
607 _GLOBAL(ret_from_except)
608 ld r11,_TRAP(r1)
609 andi. r0,r11,1
610 bne ret_from_except_lite
611 REST_NVGPRS(r1)
612
613 _GLOBAL(ret_from_except_lite)
614 /*
615 * Disable interrupts so that current_thread_info()->flags
616 * can't change between when we test it and when we return
617 * from the interrupt.
618 */
619 #ifdef CONFIG_PPC_BOOK3E
620 wrteei 0
621 #else
622 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
623 mtmsrd r10,1 /* Update machine state */
624 #endif /* CONFIG_PPC_BOOK3E */
625
626 CURRENT_THREAD_INFO(r9, r1)
627 ld r3,_MSR(r1)
628 #ifdef CONFIG_PPC_BOOK3E
629 ld r10,PACACURRENT(r13)
630 #endif /* CONFIG_PPC_BOOK3E */
631 ld r4,TI_FLAGS(r9)
632 andi. r3,r3,MSR_PR
633 beq resume_kernel
634 #ifdef CONFIG_PPC_BOOK3E
635 lwz r3,(THREAD+THREAD_DBCR0)(r10)
636 #endif /* CONFIG_PPC_BOOK3E */
637
638 /* Check current_thread_info()->flags */
639 andi. r0,r4,_TIF_USER_WORK_MASK
640 bne 1f
641 #ifdef CONFIG_PPC_BOOK3E
642 /*
643 * Check to see if the dbcr0 register is set up to debug.
644 * Use the internal debug mode bit to do this.
645 */
646 andis. r0,r3,DBCR0_IDM@h
647 beq restore
648 mfmsr r0
649 rlwinm r0,r0,0,~MSR_DE /* Clear MSR.DE */
650 mtmsr r0
651 mtspr SPRN_DBCR0,r3
652 li r10, -1
653 mtspr SPRN_DBSR,r10
654 b restore
655 #else
656 addi r3,r1,STACK_FRAME_OVERHEAD
657 bl restore_math
658 b restore
659 #endif
660 1: andi. r0,r4,_TIF_NEED_RESCHED
661 beq 2f
662 bl restore_interrupts
663 SCHEDULE_USER
664 b ret_from_except_lite
665 2:
666 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
667 andi. r0,r4,_TIF_USER_WORK_MASK & ~_TIF_RESTORE_TM
668 bne 3f /* only restore TM if nothing else to do */
669 addi r3,r1,STACK_FRAME_OVERHEAD
670 bl restore_tm_state
671 b restore
672 3:
673 #endif
674 bl save_nvgprs
675 /*
676 * Use a non volatile GPR to save and restore our thread_info flags
677 * across the call to restore_interrupts.
678 */
679 mr r30,r4
680 bl restore_interrupts
681 mr r4,r30
682 addi r3,r1,STACK_FRAME_OVERHEAD
683 bl do_notify_resume
684 b ret_from_except
685
686 resume_kernel:
687 /* check current_thread_info, _TIF_EMULATE_STACK_STORE */
688 andis. r8,r4,_TIF_EMULATE_STACK_STORE@h
689 beq+ 1f
690
691 addi r8,r1,INT_FRAME_SIZE /* Get the kprobed function entry */
692
693 lwz r3,GPR1(r1)
694 subi r3,r3,INT_FRAME_SIZE /* dst: Allocate a trampoline exception frame */
695 mr r4,r1 /* src: current exception frame */
696 mr r1,r3 /* Reroute the trampoline frame to r1 */
697
698 /* Copy from the original to the trampoline. */
699 li r5,INT_FRAME_SIZE/8 /* size: INT_FRAME_SIZE */
700 li r6,0 /* start offset: 0 */
701 mtctr r5
702 2: ldx r0,r6,r4
703 stdx r0,r6,r3
704 addi r6,r6,8
705 bdnz 2b
706
707 /* Do real store operation to complete stwu */
708 lwz r5,GPR1(r1)
709 std r8,0(r5)
710
711 /* Clear _TIF_EMULATE_STACK_STORE flag */
712 lis r11,_TIF_EMULATE_STACK_STORE@h
713 addi r5,r9,TI_FLAGS
714 0: ldarx r4,0,r5
715 andc r4,r4,r11
716 stdcx. r4,0,r5
717 bne- 0b
718 1:
719
720 #ifdef CONFIG_PREEMPT
721 /* Check if we need to preempt */
722 andi. r0,r4,_TIF_NEED_RESCHED
723 beq+ restore
724 /* Check that preempt_count() == 0 and interrupts are enabled */
725 lwz r8,TI_PREEMPT(r9)
726 cmpwi cr1,r8,0
727 ld r0,SOFTE(r1)
728 cmpdi r0,0
729 crandc eq,cr1*4+eq,eq
730 bne restore
731
732 /*
733 * Here we are preempting the current task. We want to make
734 * sure we are soft-disabled first and reconcile irq state.
735 */
736 RECONCILE_IRQ_STATE(r3,r4)
737 1: bl preempt_schedule_irq
738
739 /* Re-test flags and eventually loop */
740 CURRENT_THREAD_INFO(r9, r1)
741 ld r4,TI_FLAGS(r9)
742 andi. r0,r4,_TIF_NEED_RESCHED
743 bne 1b
744
745 /*
746 * arch_local_irq_restore() from preempt_schedule_irq above may
747 * enable hard interrupt but we really should disable interrupts
748 * when we return from the interrupt, and so that we don't get
749 * interrupted after loading SRR0/1.
750 */
751 #ifdef CONFIG_PPC_BOOK3E
752 wrteei 0
753 #else
754 ld r10,PACAKMSR(r13) /* Get kernel MSR without EE */
755 mtmsrd r10,1 /* Update machine state */
756 #endif /* CONFIG_PPC_BOOK3E */
757 #endif /* CONFIG_PREEMPT */
758
759 .globl fast_exc_return_irq
760 fast_exc_return_irq:
761 restore:
762 /*
763 * This is the main kernel exit path. First we check if we
764 * are about to re-enable interrupts
765 */
766 ld r5,SOFTE(r1)
767 lbz r6,PACASOFTIRQEN(r13)
768 cmpwi cr0,r5,0
769 beq restore_irq_off
770
771 /* We are enabling, were we already enabled ? Yes, just return */
772 cmpwi cr0,r6,1
773 beq cr0,do_restore
774
775 /*
776 * We are about to soft-enable interrupts (we are hard disabled
777 * at this point). We check if there's anything that needs to
778 * be replayed first.
779 */
780 lbz r0,PACAIRQHAPPENED(r13)
781 cmpwi cr0,r0,0
782 bne- restore_check_irq_replay
783
784 /*
785 * Get here when nothing happened while soft-disabled, just
786 * soft-enable and move-on. We will hard-enable as a side
787 * effect of rfi
788 */
789 restore_no_replay:
790 TRACE_ENABLE_INTS
791 li r0,1
792 stb r0,PACASOFTIRQEN(r13);
793
794 /*
795 * Final return path. BookE is handled in a different file
796 */
797 do_restore:
798 #ifdef CONFIG_PPC_BOOK3E
799 b exception_return_book3e
800 #else
801 /*
802 * Clear the reservation. If we know the CPU tracks the address of
803 * the reservation then we can potentially save some cycles and use
804 * a larx. On POWER6 and POWER7 this is significantly faster.
805 */
806 BEGIN_FTR_SECTION
807 stdcx. r0,0,r1 /* to clear the reservation */
808 FTR_SECTION_ELSE
809 ldarx r4,0,r1
810 ALT_FTR_SECTION_END_IFCLR(CPU_FTR_STCX_CHECKS_ADDRESS)
811
812 /*
813 * Some code path such as load_up_fpu or altivec return directly
814 * here. They run entirely hard disabled and do not alter the
815 * interrupt state. They also don't use lwarx/stwcx. and thus
816 * are known not to leave dangling reservations.
817 */
818 .globl fast_exception_return
819 fast_exception_return:
820 ld r3,_MSR(r1)
821 ld r4,_CTR(r1)
822 ld r0,_LINK(r1)
823 mtctr r4
824 mtlr r0
825 ld r4,_XER(r1)
826 mtspr SPRN_XER,r4
827
828 REST_8GPRS(5, r1)
829
830 andi. r0,r3,MSR_RI
831 beq- unrecov_restore
832
833 /* Load PPR from thread struct before we clear MSR:RI */
834 BEGIN_FTR_SECTION
835 ld r2,PACACURRENT(r13)
836 ld r2,TASKTHREADPPR(r2)
837 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
838
839 /*
840 * Clear RI before restoring r13. If we are returning to
841 * userspace and we take an exception after restoring r13,
842 * we end up corrupting the userspace r13 value.
843 */
844 ld r4,PACAKMSR(r13) /* Get kernel MSR without EE */
845 andc r4,r4,r0 /* r0 contains MSR_RI here */
846 mtmsrd r4,1
847
848 #ifdef CONFIG_PPC_TRANSACTIONAL_MEM
849 /* TM debug */
850 std r3, PACATMSCRATCH(r13) /* Stash returned-to MSR */
851 #endif
852 /*
853 * r13 is our per cpu area, only restore it if we are returning to
854 * userspace the value stored in the stack frame may belong to
855 * another CPU.
856 */
857 andi. r0,r3,MSR_PR
858 beq 1f
859 BEGIN_FTR_SECTION
860 mtspr SPRN_PPR,r2 /* Restore PPR */
861 END_FTR_SECTION_IFSET(CPU_FTR_HAS_PPR)
862 ACCOUNT_CPU_USER_EXIT(r13, r2, r4)
863 REST_GPR(13, r1)
864 1:
865 mtspr SPRN_SRR1,r3
866
867 ld r2,_CCR(r1)
868 mtcrf 0xFF,r2
869 ld r2,_NIP(r1)
870 mtspr SPRN_SRR0,r2
871
872 ld r0,GPR0(r1)
873 ld r2,GPR2(r1)
874 ld r3,GPR3(r1)
875 ld r4,GPR4(r1)
876 ld r1,GPR1(r1)
877
878 rfid
879 b . /* prevent speculative execution */
880
881 #endif /* CONFIG_PPC_BOOK3E */
882
883 /*
884 * We are returning to a context with interrupts soft disabled.
885 *
886 * However, we may also about to hard enable, so we need to
887 * make sure that in this case, we also clear PACA_IRQ_HARD_DIS
888 * or that bit can get out of sync and bad things will happen
889 */
890 restore_irq_off:
891 ld r3,_MSR(r1)
892 lbz r7,PACAIRQHAPPENED(r13)
893 andi. r0,r3,MSR_EE
894 beq 1f
895 rlwinm r7,r7,0,~PACA_IRQ_HARD_DIS
896 stb r7,PACAIRQHAPPENED(r13)
897 1: li r0,0
898 stb r0,PACASOFTIRQEN(r13);
899 TRACE_DISABLE_INTS
900 b do_restore
901
902 /*
903 * Something did happen, check if a re-emit is needed
904 * (this also clears paca->irq_happened)
905 */
906 restore_check_irq_replay:
907 /* XXX: We could implement a fast path here where we check
908 * for irq_happened being just 0x01, in which case we can
909 * clear it and return. That means that we would potentially
910 * miss a decrementer having wrapped all the way around.
911 *
912 * Still, this might be useful for things like hash_page
913 */
914 bl __check_irq_replay
915 cmpwi cr0,r3,0
916 beq restore_no_replay
917
918 /*
919 * We need to re-emit an interrupt. We do so by re-using our
920 * existing exception frame. We first change the trap value,
921 * but we need to ensure we preserve the low nibble of it
922 */
923 ld r4,_TRAP(r1)
924 clrldi r4,r4,60
925 or r4,r4,r3
926 std r4,_TRAP(r1)
927
928 /*
929 * Then find the right handler and call it. Interrupts are
930 * still soft-disabled and we keep them that way.
931 */
932 cmpwi cr0,r3,0x500
933 bne 1f
934 addi r3,r1,STACK_FRAME_OVERHEAD;
935 bl do_IRQ
936 b ret_from_except
937 1: cmpwi cr0,r3,0xe60
938 bne 1f
939 addi r3,r1,STACK_FRAME_OVERHEAD;
940 bl handle_hmi_exception
941 b ret_from_except
942 1: cmpwi cr0,r3,0x900
943 bne 1f
944 addi r3,r1,STACK_FRAME_OVERHEAD;
945 bl timer_interrupt
946 b ret_from_except
947 #ifdef CONFIG_PPC_DOORBELL
948 1:
949 #ifdef CONFIG_PPC_BOOK3E
950 cmpwi cr0,r3,0x280
951 #else
952 BEGIN_FTR_SECTION
953 cmpwi cr0,r3,0xe80
954 FTR_SECTION_ELSE
955 cmpwi cr0,r3,0xa00
956 ALT_FTR_SECTION_END_IFSET(CPU_FTR_HVMODE)
957 #endif /* CONFIG_PPC_BOOK3E */
958 bne 1f
959 addi r3,r1,STACK_FRAME_OVERHEAD;
960 bl doorbell_exception
961 b ret_from_except
962 #endif /* CONFIG_PPC_DOORBELL */
963 1: b ret_from_except /* What else to do here ? */
964
965 unrecov_restore:
966 addi r3,r1,STACK_FRAME_OVERHEAD
967 bl unrecoverable_exception
968 b unrecov_restore
969
970 #ifdef CONFIG_PPC_RTAS
971 /*
972 * On CHRP, the Run-Time Abstraction Services (RTAS) have to be
973 * called with the MMU off.
974 *
975 * In addition, we need to be in 32b mode, at least for now.
976 *
977 * Note: r3 is an input parameter to rtas, so don't trash it...
978 */
979 _GLOBAL(enter_rtas)
980 mflr r0
981 std r0,16(r1)
982 stdu r1,-RTAS_FRAME_SIZE(r1) /* Save SP and create stack space. */
983
984 /* Because RTAS is running in 32b mode, it clobbers the high order half
985 * of all registers that it saves. We therefore save those registers
986 * RTAS might touch to the stack. (r0, r3-r13 are caller saved)
987 */
988 SAVE_GPR(2, r1) /* Save the TOC */
989 SAVE_GPR(13, r1) /* Save paca */
990 SAVE_8GPRS(14, r1) /* Save the non-volatiles */
991 SAVE_10GPRS(22, r1) /* ditto */
992
993 mfcr r4
994 std r4,_CCR(r1)
995 mfctr r5
996 std r5,_CTR(r1)
997 mfspr r6,SPRN_XER
998 std r6,_XER(r1)
999 mfdar r7
1000 std r7,_DAR(r1)
1001 mfdsisr r8
1002 std r8,_DSISR(r1)
1003
1004 /* Temporary workaround to clear CR until RTAS can be modified to
1005 * ignore all bits.
1006 */
1007 li r0,0
1008 mtcr r0
1009
1010 #ifdef CONFIG_BUG
1011 /* There is no way it is acceptable to get here with interrupts enabled,
1012 * check it with the asm equivalent of WARN_ON
1013 */
1014 lbz r0,PACASOFTIRQEN(r13)
1015 1: tdnei r0,0
1016 EMIT_BUG_ENTRY 1b,__FILE__,__LINE__,BUGFLAG_WARNING
1017 #endif
1018
1019 /* Hard-disable interrupts */
1020 mfmsr r6
1021 rldicl r7,r6,48,1
1022 rotldi r7,r7,16
1023 mtmsrd r7,1
1024
1025 /* Unfortunately, the stack pointer and the MSR are also clobbered,
1026 * so they are saved in the PACA which allows us to restore
1027 * our original state after RTAS returns.
1028 */
1029 std r1,PACAR1(r13)
1030 std r6,PACASAVEDMSR(r13)
1031
1032 /* Setup our real return addr */
1033 LOAD_REG_ADDR(r4,rtas_return_loc)
1034 clrldi r4,r4,2 /* convert to realmode address */
1035 mtlr r4
1036
1037 li r0,0
1038 ori r0,r0,MSR_EE|MSR_SE|MSR_BE|MSR_RI
1039 andc r0,r6,r0
1040
1041 li r9,1
1042 rldicr r9,r9,MSR_SF_LG,(63-MSR_SF_LG)
1043 ori r9,r9,MSR_IR|MSR_DR|MSR_FE0|MSR_FE1|MSR_FP|MSR_RI|MSR_LE
1044 andc r6,r0,r9
1045 sync /* disable interrupts so SRR0/1 */
1046 mtmsrd r0 /* don't get trashed */
1047
1048 LOAD_REG_ADDR(r4, rtas)
1049 ld r5,RTASENTRY(r4) /* get the rtas->entry value */
1050 ld r4,RTASBASE(r4) /* get the rtas->base value */
1051
1052 mtspr SPRN_SRR0,r5
1053 mtspr SPRN_SRR1,r6
1054 rfid
1055 b . /* prevent speculative execution */
1056
1057 rtas_return_loc:
1058 FIXUP_ENDIAN
1059
1060 /* relocation is off at this point */
1061 GET_PACA(r4)
1062 clrldi r4,r4,2 /* convert to realmode address */
1063
1064 bcl 20,31,$+4
1065 0: mflr r3
1066 ld r3,(1f-0b)(r3) /* get &rtas_restore_regs */
1067
1068 mfmsr r6
1069 li r0,MSR_RI
1070 andc r6,r6,r0
1071 sync
1072 mtmsrd r6
1073
1074 ld r1,PACAR1(r4) /* Restore our SP */
1075 ld r4,PACASAVEDMSR(r4) /* Restore our MSR */
1076
1077 mtspr SPRN_SRR0,r3
1078 mtspr SPRN_SRR1,r4
1079 rfid
1080 b . /* prevent speculative execution */
1081
1082 .align 3
1083 1: .llong rtas_restore_regs
1084
1085 rtas_restore_regs:
1086 /* relocation is on at this point */
1087 REST_GPR(2, r1) /* Restore the TOC */
1088 REST_GPR(13, r1) /* Restore paca */
1089 REST_8GPRS(14, r1) /* Restore the non-volatiles */
1090 REST_10GPRS(22, r1) /* ditto */
1091
1092 GET_PACA(r13)
1093
1094 ld r4,_CCR(r1)
1095 mtcr r4
1096 ld r5,_CTR(r1)
1097 mtctr r5
1098 ld r6,_XER(r1)
1099 mtspr SPRN_XER,r6
1100 ld r7,_DAR(r1)
1101 mtdar r7
1102 ld r8,_DSISR(r1)
1103 mtdsisr r8
1104
1105 addi r1,r1,RTAS_FRAME_SIZE /* Unstack our frame */
1106 ld r0,16(r1) /* get return address */
1107
1108 mtlr r0
1109 blr /* return to caller */
1110
1111 #endif /* CONFIG_PPC_RTAS */
1112
1113 _GLOBAL(enter_prom)
1114 mflr r0
1115 std r0,16(r1)
1116 stdu r1,-PROM_FRAME_SIZE(r1) /* Save SP and create stack space */
1117
1118 /* Because PROM is running in 32b mode, it clobbers the high order half
1119 * of all registers that it saves. We therefore save those registers
1120 * PROM might touch to the stack. (r0, r3-r13 are caller saved)
1121 */
1122 SAVE_GPR(2, r1)
1123 SAVE_GPR(13, r1)
1124 SAVE_8GPRS(14, r1)
1125 SAVE_10GPRS(22, r1)
1126 mfcr r10
1127 mfmsr r11
1128 std r10,_CCR(r1)
1129 std r11,_MSR(r1)
1130
1131 /* Put PROM address in SRR0 */
1132 mtsrr0 r4
1133
1134 /* Setup our trampoline return addr in LR */
1135 bcl 20,31,$+4
1136 0: mflr r4
1137 addi r4,r4,(1f - 0b)
1138 mtlr r4
1139
1140 /* Prepare a 32-bit mode big endian MSR
1141 */
1142 #ifdef CONFIG_PPC_BOOK3E
1143 rlwinm r11,r11,0,1,31
1144 mtsrr1 r11
1145 rfi
1146 #else /* CONFIG_PPC_BOOK3E */
1147 LOAD_REG_IMMEDIATE(r12, MSR_SF | MSR_ISF | MSR_LE)
1148 andc r11,r11,r12
1149 mtsrr1 r11
1150 rfid
1151 #endif /* CONFIG_PPC_BOOK3E */
1152
1153 1: /* Return from OF */
1154 FIXUP_ENDIAN
1155
1156 /* Just make sure that r1 top 32 bits didn't get
1157 * corrupt by OF
1158 */
1159 rldicl r1,r1,0,32
1160
1161 /* Restore the MSR (back to 64 bits) */
1162 ld r0,_MSR(r1)
1163 MTMSRD(r0)
1164 isync
1165
1166 /* Restore other registers */
1167 REST_GPR(2, r1)
1168 REST_GPR(13, r1)
1169 REST_8GPRS(14, r1)
1170 REST_10GPRS(22, r1)
1171 ld r4,_CCR(r1)
1172 mtcr r4
1173
1174 addi r1,r1,PROM_FRAME_SIZE
1175 ld r0,16(r1)
1176 mtlr r0
1177 blr
1178
1179 #ifdef CONFIG_FUNCTION_TRACER
1180 #ifdef CONFIG_DYNAMIC_FTRACE
1181 _GLOBAL(mcount)
1182 _GLOBAL(_mcount)
1183 mflr r12
1184 mtctr r12
1185 mtlr r0
1186 bctr
1187
1188 #ifndef CC_USING_MPROFILE_KERNEL
1189 _GLOBAL_TOC(ftrace_caller)
1190 /* Taken from output of objdump from lib64/glibc */
1191 mflr r3
1192 ld r11, 0(r1)
1193 stdu r1, -112(r1)
1194 std r3, 128(r1)
1195 ld r4, 16(r11)
1196 subi r3, r3, MCOUNT_INSN_SIZE
1197 .globl ftrace_call
1198 ftrace_call:
1199 bl ftrace_stub
1200 nop
1201 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1202 .globl ftrace_graph_call
1203 ftrace_graph_call:
1204 b ftrace_graph_stub
1205 _GLOBAL(ftrace_graph_stub)
1206 #endif
1207 ld r0, 128(r1)
1208 mtlr r0
1209 addi r1, r1, 112
1210
1211 #else /* CC_USING_MPROFILE_KERNEL */
1212 /*
1213 *
1214 * ftrace_caller() is the function that replaces _mcount() when ftrace is
1215 * active.
1216 *
1217 * We arrive here after a function A calls function B, and we are the trace
1218 * function for B. When we enter r1 points to A's stack frame, B has not yet
1219 * had a chance to allocate one yet.
1220 *
1221 * Additionally r2 may point either to the TOC for A, or B, depending on
1222 * whether B did a TOC setup sequence before calling us.
1223 *
1224 * On entry the LR points back to the _mcount() call site, and r0 holds the
1225 * saved LR as it was on entry to B, ie. the original return address at the
1226 * call site in A.
1227 *
1228 * Our job is to save the register state into a struct pt_regs (on the stack)
1229 * and then arrange for the ftrace function to be called.
1230 */
1231 _GLOBAL(ftrace_caller)
1232 /* Save the original return address in A's stack frame */
1233 std r0,LRSAVE(r1)
1234
1235 /* Create our stack frame + pt_regs */
1236 stdu r1,-SWITCH_FRAME_SIZE(r1)
1237
1238 /* Save all gprs to pt_regs */
1239 SAVE_8GPRS(0,r1)
1240 SAVE_8GPRS(8,r1)
1241 SAVE_8GPRS(16,r1)
1242 SAVE_8GPRS(24,r1)
1243
1244 /* Load special regs for save below */
1245 mfmsr r8
1246 mfctr r9
1247 mfxer r10
1248 mfcr r11
1249
1250 /* Get the _mcount() call site out of LR */
1251 mflr r7
1252 /* Save it as pt_regs->nip & pt_regs->link */
1253 std r7, _NIP(r1)
1254 std r7, _LINK(r1)
1255
1256 /* Save callee's TOC in the ABI compliant location */
1257 std r2, 24(r1)
1258 ld r2,PACATOC(r13) /* get kernel TOC in r2 */
1259
1260 addis r3,r2,function_trace_op@toc@ha
1261 addi r3,r3,function_trace_op@toc@l
1262 ld r5,0(r3)
1263
1264 #ifdef CONFIG_LIVEPATCH
1265 mr r14,r7 /* remember old NIP */
1266 #endif
1267 /* Calculate ip from nip-4 into r3 for call below */
1268 subi r3, r7, MCOUNT_INSN_SIZE
1269
1270 /* Put the original return address in r4 as parent_ip */
1271 mr r4, r0
1272
1273 /* Save special regs */
1274 std r8, _MSR(r1)
1275 std r9, _CTR(r1)
1276 std r10, _XER(r1)
1277 std r11, _CCR(r1)
1278
1279 /* Load &pt_regs in r6 for call below */
1280 addi r6, r1 ,STACK_FRAME_OVERHEAD
1281
1282 /* ftrace_call(r3, r4, r5, r6) */
1283 .globl ftrace_call
1284 ftrace_call:
1285 bl ftrace_stub
1286 nop
1287
1288 /* Load ctr with the possibly modified NIP */
1289 ld r3, _NIP(r1)
1290 mtctr r3
1291 #ifdef CONFIG_LIVEPATCH
1292 cmpd r14,r3 /* has NIP been altered? */
1293 #endif
1294
1295 /* Restore gprs */
1296 REST_8GPRS(0,r1)
1297 REST_8GPRS(8,r1)
1298 REST_8GPRS(16,r1)
1299 REST_8GPRS(24,r1)
1300
1301 /* Restore callee's TOC */
1302 ld r2, 24(r1)
1303
1304 /* Pop our stack frame */
1305 addi r1, r1, SWITCH_FRAME_SIZE
1306
1307 /* Restore original LR for return to B */
1308 ld r0, LRSAVE(r1)
1309 mtlr r0
1310
1311 #ifdef CONFIG_LIVEPATCH
1312 /* Based on the cmpd above, if the NIP was altered handle livepatch */
1313 bne- livepatch_handler
1314 #endif
1315
1316 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1317 stdu r1, -112(r1)
1318 .globl ftrace_graph_call
1319 ftrace_graph_call:
1320 b ftrace_graph_stub
1321 _GLOBAL(ftrace_graph_stub)
1322 addi r1, r1, 112
1323 #endif
1324
1325 ld r0,LRSAVE(r1) /* restore callee's lr at _mcount site */
1326 mtlr r0
1327 bctr /* jump after _mcount site */
1328 #endif /* CC_USING_MPROFILE_KERNEL */
1329
1330 _GLOBAL(ftrace_stub)
1331 blr
1332
1333 #ifdef CONFIG_LIVEPATCH
1334 /*
1335 * This function runs in the mcount context, between two functions. As
1336 * such it can only clobber registers which are volatile and used in
1337 * function linkage.
1338 *
1339 * We get here when a function A, calls another function B, but B has
1340 * been live patched with a new function C.
1341 *
1342 * On entry:
1343 * - we have no stack frame and can not allocate one
1344 * - LR points back to the original caller (in A)
1345 * - CTR holds the new NIP in C
1346 * - r0 & r12 are free
1347 *
1348 * r0 can't be used as the base register for a DS-form load or store, so
1349 * we temporarily shuffle r1 (stack pointer) into r0 and then put it back.
1350 */
1351 livepatch_handler:
1352 CURRENT_THREAD_INFO(r12, r1)
1353
1354 /* Save stack pointer into r0 */
1355 mr r0, r1
1356
1357 /* Allocate 3 x 8 bytes */
1358 ld r1, TI_livepatch_sp(r12)
1359 addi r1, r1, 24
1360 std r1, TI_livepatch_sp(r12)
1361
1362 /* Save toc & real LR on livepatch stack */
1363 std r2, -24(r1)
1364 mflr r12
1365 std r12, -16(r1)
1366
1367 /* Store stack end marker */
1368 lis r12, STACK_END_MAGIC@h
1369 ori r12, r12, STACK_END_MAGIC@l
1370 std r12, -8(r1)
1371
1372 /* Restore real stack pointer */
1373 mr r1, r0
1374
1375 /* Put ctr in r12 for global entry and branch there */
1376 mfctr r12
1377 bctrl
1378
1379 /*
1380 * Now we are returning from the patched function to the original
1381 * caller A. We are free to use r0 and r12, and we can use r2 until we
1382 * restore it.
1383 */
1384
1385 CURRENT_THREAD_INFO(r12, r1)
1386
1387 /* Save stack pointer into r0 */
1388 mr r0, r1
1389
1390 ld r1, TI_livepatch_sp(r12)
1391
1392 /* Check stack marker hasn't been trashed */
1393 lis r2, STACK_END_MAGIC@h
1394 ori r2, r2, STACK_END_MAGIC@l
1395 ld r12, -8(r1)
1396 1: tdne r12, r2
1397 EMIT_BUG_ENTRY 1b, __FILE__, __LINE__ - 1, 0
1398
1399 /* Restore LR & toc from livepatch stack */
1400 ld r12, -16(r1)
1401 mtlr r12
1402 ld r2, -24(r1)
1403
1404 /* Pop livepatch stack frame */
1405 CURRENT_THREAD_INFO(r12, r0)
1406 subi r1, r1, 24
1407 std r1, TI_livepatch_sp(r12)
1408
1409 /* Restore real stack pointer */
1410 mr r1, r0
1411
1412 /* Return to original caller of live patched function */
1413 blr
1414 #endif
1415
1416
1417 #else
1418 _GLOBAL_TOC(_mcount)
1419 /* Taken from output of objdump from lib64/glibc */
1420 mflr r3
1421 ld r11, 0(r1)
1422 stdu r1, -112(r1)
1423 std r3, 128(r1)
1424 ld r4, 16(r11)
1425
1426 subi r3, r3, MCOUNT_INSN_SIZE
1427 LOAD_REG_ADDR(r5,ftrace_trace_function)
1428 ld r5,0(r5)
1429 ld r5,0(r5)
1430 mtctr r5
1431 bctrl
1432 nop
1433
1434
1435 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1436 b ftrace_graph_caller
1437 #endif
1438 ld r0, 128(r1)
1439 mtlr r0
1440 addi r1, r1, 112
1441 _GLOBAL(ftrace_stub)
1442 blr
1443
1444 #endif /* CONFIG_DYNAMIC_FTRACE */
1445
1446 #ifdef CONFIG_FUNCTION_GRAPH_TRACER
1447 #ifndef CC_USING_MPROFILE_KERNEL
1448 _GLOBAL(ftrace_graph_caller)
1449 /* load r4 with local address */
1450 ld r4, 128(r1)
1451 subi r4, r4, MCOUNT_INSN_SIZE
1452
1453 /* Grab the LR out of the caller stack frame */
1454 ld r11, 112(r1)
1455 ld r3, 16(r11)
1456
1457 bl prepare_ftrace_return
1458 nop
1459
1460 /*
1461 * prepare_ftrace_return gives us the address we divert to.
1462 * Change the LR in the callers stack frame to this.
1463 */
1464 ld r11, 112(r1)
1465 std r3, 16(r11)
1466
1467 ld r0, 128(r1)
1468 mtlr r0
1469 addi r1, r1, 112
1470 blr
1471
1472 #else /* CC_USING_MPROFILE_KERNEL */
1473 _GLOBAL(ftrace_graph_caller)
1474 /* with -mprofile-kernel, parameter regs are still alive at _mcount */
1475 std r10, 104(r1)
1476 std r9, 96(r1)
1477 std r8, 88(r1)
1478 std r7, 80(r1)
1479 std r6, 72(r1)
1480 std r5, 64(r1)
1481 std r4, 56(r1)
1482 std r3, 48(r1)
1483
1484 /* Save callee's TOC in the ABI compliant location */
1485 std r2, 24(r1)
1486 ld r2, PACATOC(r13) /* get kernel TOC in r2 */
1487
1488 mfctr r4 /* ftrace_caller has moved local addr here */
1489 std r4, 40(r1)
1490 mflr r3 /* ftrace_caller has restored LR from stack */
1491 subi r4, r4, MCOUNT_INSN_SIZE
1492
1493 bl prepare_ftrace_return
1494 nop
1495
1496 /*
1497 * prepare_ftrace_return gives us the address we divert to.
1498 * Change the LR to this.
1499 */
1500 mtlr r3
1501
1502 ld r0, 40(r1)
1503 mtctr r0
1504 ld r10, 104(r1)
1505 ld r9, 96(r1)
1506 ld r8, 88(r1)
1507 ld r7, 80(r1)
1508 ld r6, 72(r1)
1509 ld r5, 64(r1)
1510 ld r4, 56(r1)
1511 ld r3, 48(r1)
1512
1513 /* Restore callee's TOC */
1514 ld r2, 24(r1)
1515
1516 addi r1, r1, 112
1517 mflr r0
1518 std r0, LRSAVE(r1)
1519 bctr
1520 #endif /* CC_USING_MPROFILE_KERNEL */
1521
1522 _GLOBAL(return_to_handler)
1523 /* need to save return values */
1524 std r4, -32(r1)
1525 std r3, -24(r1)
1526 /* save TOC */
1527 std r2, -16(r1)
1528 std r31, -8(r1)
1529 mr r31, r1
1530 stdu r1, -112(r1)
1531
1532 /*
1533 * We might be called from a module.
1534 * Switch to our TOC to run inside the core kernel.
1535 */
1536 ld r2, PACATOC(r13)
1537
1538 bl ftrace_return_to_handler
1539 nop
1540
1541 /* return value has real return address */
1542 mtlr r3
1543
1544 ld r1, 0(r1)
1545 ld r4, -32(r1)
1546 ld r3, -24(r1)
1547 ld r2, -16(r1)
1548 ld r31, -8(r1)
1549
1550 /* Jump back to real return address */
1551 blr
1552 #endif /* CONFIG_FUNCTION_GRAPH_TRACER */
1553 #endif /* CONFIG_FUNCTION_TRACER */
Linux kernel - Deliverables
This page took 0.161057 seconds and 5 git commands to generate.