Commit | Line | Data |
---|---|---|
1da177e4 | 1 | /* |
1da177e4 LT |
2 | * S390 low-level entry points. |
3 | * | |
a53c8fab | 4 | * Copyright IBM Corp. 1999, 2012 |
1da177e4 | 5 | * Author(s): Martin Schwidefsky (schwidefsky@de.ibm.com), |
25d83cbf HC |
6 | * Hartmut Penner (hp@de.ibm.com), |
7 | * Denis Joseph Barrow (djbarrow@de.ibm.com,barrow_dj@yahoo.com), | |
77fa2245 | 8 | * Heiko Carstens <heiko.carstens@de.ibm.com> |
1da177e4 LT |
9 | */ |
10 | ||
2bc89b5e | 11 | #include <linux/init.h> |
144d634a | 12 | #include <linux/linkage.h> |
d3a73acb | 13 | #include <asm/processor.h> |
1da177e4 | 14 | #include <asm/cache.h> |
1da177e4 LT |
15 | #include <asm/errno.h> |
16 | #include <asm/ptrace.h> | |
17 | #include <asm/thread_info.h> | |
0013a854 | 18 | #include <asm/asm-offsets.h> |
1da177e4 LT |
19 | #include <asm/unistd.h> |
20 | #include <asm/page.h> | |
eb546195 | 21 | #include <asm/sigp.h> |
1f44a225 | 22 | #include <asm/irq.h> |
1da177e4 | 23 | |
c5328901 MS |
24 | __PT_R0 = __PT_GPRS |
25 | __PT_R1 = __PT_GPRS + 4 | |
26 | __PT_R2 = __PT_GPRS + 8 | |
27 | __PT_R3 = __PT_GPRS + 12 | |
28 | __PT_R4 = __PT_GPRS + 16 | |
29 | __PT_R5 = __PT_GPRS + 20 | |
30 | __PT_R6 = __PT_GPRS + 24 | |
31 | __PT_R7 = __PT_GPRS + 28 | |
32 | __PT_R8 = __PT_GPRS + 32 | |
33 | __PT_R9 = __PT_GPRS + 36 | |
34 | __PT_R10 = __PT_GPRS + 40 | |
35 | __PT_R11 = __PT_GPRS + 44 | |
36 | __PT_R12 = __PT_GPRS + 48 | |
37 | __PT_R13 = __PT_GPRS + 524 | |
38 | __PT_R14 = __PT_GPRS + 56 | |
39 | __PT_R15 = __PT_GPRS + 60 | |
1da177e4 | 40 | |
1da177e4 LT |
41 | STACK_SHIFT = PAGE_SHIFT + THREAD_ORDER |
42 | STACK_SIZE = 1 << STACK_SHIFT | |
dc7ee00d | 43 | STACK_INIT = STACK_SIZE - STACK_FRAME_OVERHEAD - __PT_SIZE |
1da177e4 | 44 | |
d3a73acb MS |
45 | _TIF_WORK = (_TIF_SIGPENDING | _TIF_NOTIFY_RESUME | _TIF_NEED_RESCHED) |
46 | _TIF_TRACE = (_TIF_SYSCALL_TRACE | _TIF_SYSCALL_AUDIT | _TIF_SECCOMP | \ | |
47 | _TIF_SYSCALL_TRACEPOINT) | |
48 | _CIF_WORK = (_CIF_MCCK_PENDING | _CIF_ASCE) | |
49 | _PIF_WORK = (_PIF_PER_TRAP) | |
50 | ||
1da177e4 LT |
51 | #define BASED(name) name-system_call(%r13) |
52 | ||
1f194a4c | 53 | .macro TRACE_IRQS_ON |
c5328901 | 54 | #ifdef CONFIG_TRACE_IRQFLAGS |
50bec4ce | 55 | basr %r2,%r0 |
86ed42f4 | 56 | l %r1,BASED(.Lc_hardirqs_on) |
c5328901 MS |
57 | basr %r14,%r1 # call trace_hardirqs_on_caller |
58 | #endif | |
1f194a4c HC |
59 | .endm |
60 | ||
61 | .macro TRACE_IRQS_OFF | |
c5328901 | 62 | #ifdef CONFIG_TRACE_IRQFLAGS |
50bec4ce | 63 | basr %r2,%r0 |
86ed42f4 | 64 | l %r1,BASED(.Lc_hardirqs_off) |
c5328901 | 65 | basr %r14,%r1 # call trace_hardirqs_off_caller |
411788ea | 66 | #endif |
c5328901 | 67 | .endm |
411788ea | 68 | |
411788ea | 69 | .macro LOCKDEP_SYS_EXIT |
c5328901 MS |
70 | #ifdef CONFIG_LOCKDEP |
71 | tm __PT_PSW+1(%r11),0x01 # returning to user ? | |
72 | jz .+10 | |
86ed42f4 | 73 | l %r1,BASED(.Lc_lockdep_sys_exit) |
c5328901 | 74 | basr %r14,%r1 # call lockdep_sys_exit |
1f194a4c | 75 | #endif |
987ad70a MS |
76 | .endm |
77 | ||
c5328901 | 78 | .macro CHECK_STACK stacksize,savearea |
63b12246 | 79 | #ifdef CONFIG_CHECK_STACK |
c5328901 MS |
80 | tml %r15,\stacksize - CONFIG_STACK_GUARD |
81 | la %r14,\savearea | |
82 | jz stack_overflow | |
63b12246 | 83 | #endif |
63b12246 MS |
84 | .endm |
85 | ||
c5328901 MS |
86 | .macro SWITCH_ASYNC savearea,stack,shift |
87 | tmh %r8,0x0001 # interrupting from user ? | |
88 | jnz 1f | |
89 | lr %r14,%r9 | |
86ed42f4 MS |
90 | sl %r14,BASED(.Lc_critical_start) |
91 | cl %r14,BASED(.Lc_critical_length) | |
c5328901 MS |
92 | jhe 0f |
93 | la %r11,\savearea # inside critical section, do cleanup | |
94 | bras %r14,cleanup_critical | |
95 | tmh %r8,0x0001 # retest problem state after cleanup | |
96 | jnz 1f | |
97 | 0: l %r14,\stack # are we already on the target stack? | |
1da177e4 | 98 | slr %r14,%r15 |
c5328901 MS |
99 | sra %r14,\shift |
100 | jnz 1f | |
101 | CHECK_STACK 1<<\shift,\savearea | |
dc7ee00d | 102 | ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) |
c5328901 MS |
103 | j 2f |
104 | 1: l %r15,\stack # load target stack | |
dc7ee00d | 105 | 2: la %r11,STACK_FRAME_OVERHEAD(%r15) |
77fa2245 HC |
106 | .endm |
107 | ||
c5328901 MS |
108 | .macro ADD64 high,low,timer |
109 | al \high,\timer | |
4c1051e3 | 110 | al \low,4+\timer |
c5328901 MS |
111 | brc 12,.+8 |
112 | ahi \high,1 | |
1da177e4 LT |
113 | .endm |
114 | ||
c5328901 MS |
115 | .macro SUB64 high,low,timer |
116 | sl \high,\timer | |
4c1051e3 | 117 | sl \low,4+\timer |
c5328901 MS |
118 | brc 3,.+8 |
119 | ahi \high,-1 | |
120 | .endm | |
121 | ||
122 | .macro UPDATE_VTIME high,low,enter_timer | |
123 | lm \high,\low,__LC_EXIT_TIMER | |
124 | SUB64 \high,\low,\enter_timer | |
125 | ADD64 \high,\low,__LC_USER_TIMER | |
126 | stm \high,\low,__LC_USER_TIMER | |
127 | lm \high,\low,__LC_LAST_UPDATE_TIMER | |
128 | SUB64 \high,\low,__LC_EXIT_TIMER | |
129 | ADD64 \high,\low,__LC_SYSTEM_TIMER | |
130 | stm \high,\low,__LC_SYSTEM_TIMER | |
131 | mvc __LC_LAST_UPDATE_TIMER(8),\enter_timer | |
1da177e4 LT |
132 | .endm |
133 | ||
1e54622e | 134 | .macro REENABLE_IRQS |
c5328901 MS |
135 | st %r8,__LC_RETURN_PSW |
136 | ni __LC_RETURN_PSW,0xbf | |
137 | ssm __LC_RETURN_PSW | |
1e54622e MS |
138 | .endm |
139 | ||
860dba45 MS |
140 | .section .kprobes.text, "ax" |
141 | ||
1da177e4 LT |
142 | /* |
143 | * Scheduler resume function, called by switch_to | |
144 | * gpr2 = (task_struct *) prev | |
145 | * gpr3 = (task_struct *) next | |
146 | * Returns: | |
147 | * gpr2 = prev | |
148 | */ | |
144d634a | 149 | ENTRY(__switch_to) |
eda0c6d6 MS |
150 | stm %r6,%r15,__SF_GPRS(%r15) # store gprs of prev task |
151 | st %r15,__THREAD_ksp(%r2) # store kernel stack of prev | |
c5328901 | 152 | l %r4,__THREAD_info(%r2) # get thread_info of prev |
5e9a2692 | 153 | l %r5,__THREAD_info(%r3) # get thread_info of next |
eda0c6d6 | 154 | lr %r15,%r5 |
dc7ee00d | 155 | ahi %r15,STACK_INIT # end of kernel stack of next |
eda0c6d6 MS |
156 | st %r3,__LC_CURRENT # store task struct of next |
157 | st %r5,__LC_THREAD_INFO # store thread info of next | |
158 | st %r15,__LC_KERNEL_STACK # store end of kernel stack | |
159 | lctl %c4,%c4,__TASK_pid(%r3) # load pid to control reg. 4 | |
160 | mvc __LC_CURRENT_PID(4,%r0),__TASK_pid(%r3) # store pid of next | |
161 | l %r15,__THREAD_ksp(%r3) # load kernel stack of next | |
d3a73acb | 162 | lm %r6,%r15,__SF_GPRS(%r15) # load gprs of next task |
1da177e4 LT |
163 | br %r14 |
164 | ||
86ed42f4 | 165 | .L__critical_start: |
1da177e4 LT |
166 | /* |
167 | * SVC interrupt handler routine. System calls are synchronous events and | |
168 | * are executed with interrupts enabled. | |
169 | */ | |
170 | ||
144d634a | 171 | ENTRY(system_call) |
c185b783 | 172 | stpt __LC_SYNC_ENTER_TIMER |
86ed42f4 | 173 | .Lsysc_stm: |
c5328901 MS |
174 | stm %r8,%r15,__LC_SAVE_AREA_SYNC |
175 | l %r12,__LC_THREAD_INFO | |
176 | l %r13,__LC_SVC_NEW_PSW+4 | |
d3a73acb | 177 | lhi %r14,_PIF_SYSCALL |
86ed42f4 | 178 | .Lsysc_per: |
c5328901 | 179 | l %r15,__LC_KERNEL_STACK |
c5328901 | 180 | la %r11,STACK_FRAME_OVERHEAD(%r15) # pointer to pt_regs |
86ed42f4 | 181 | .Lsysc_vtime: |
c5328901 MS |
182 | UPDATE_VTIME %r8,%r9,__LC_SYNC_ENTER_TIMER |
183 | stm %r0,%r7,__PT_R0(%r11) | |
184 | mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC | |
185 | mvc __PT_PSW(8,%r11),__LC_SVC_OLD_PSW | |
aa33c8cb | 186 | mvc __PT_INT_CODE(4,%r11),__LC_SVC_ILC |
d3a73acb | 187 | st %r14,__PT_FLAGS(%r11) |
86ed42f4 | 188 | .Lsysc_do_svc: |
61649881 | 189 | l %r10,__TI_sysc_table(%r12) # 31 bit system call table |
aa33c8cb | 190 | lh %r8,__PT_INT_CODE+2(%r11) |
c5328901 | 191 | sla %r8,2 # shift and test for svc0 |
86ed42f4 | 192 | jnz .Lsysc_nr_ok |
1da177e4 LT |
193 | # svc 0: system call number in %r1 |
194 | cl %r1,BASED(.Lnr_syscalls) | |
86ed42f4 | 195 | jnl .Lsysc_nr_ok |
aa33c8cb | 196 | sth %r1,__PT_INT_CODE+2(%r11) |
c5328901 MS |
197 | lr %r8,%r1 |
198 | sla %r8,2 | |
86ed42f4 | 199 | .Lsysc_nr_ok: |
c5328901 MS |
200 | xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) |
201 | st %r2,__PT_ORIG_GPR2(%r11) | |
202 | st %r7,STACK_FRAME_OVERHEAD(%r15) | |
203 | l %r9,0(%r8,%r10) # get system call addr. | |
d3a73acb | 204 | tm __TI_flags+3(%r12),_TIF_TRACE |
86ed42f4 | 205 | jnz .Lsysc_tracesys |
c5328901 MS |
206 | basr %r14,%r9 # call sys_xxxx |
207 | st %r2,__PT_R2(%r11) # store return value | |
1da177e4 | 208 | |
86ed42f4 | 209 | .Lsysc_return: |
6a2df3a8 | 210 | LOCKDEP_SYS_EXIT |
86ed42f4 | 211 | .Lsysc_tif: |
c5328901 | 212 | tm __PT_PSW+1(%r11),0x01 # returning to user ? |
86ed42f4 | 213 | jno .Lsysc_restore |
d3a73acb | 214 | tm __PT_FLAGS+3(%r11),_PIF_WORK |
86ed42f4 | 215 | jnz .Lsysc_work |
d3a73acb | 216 | tm __TI_flags+3(%r12),_TIF_WORK |
86ed42f4 | 217 | jnz .Lsysc_work # check for thread work |
d3a73acb | 218 | tm __LC_CPU_FLAGS+3,_CIF_WORK |
86ed42f4 MS |
219 | jnz .Lsysc_work |
220 | .Lsysc_restore: | |
c5328901 MS |
221 | mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) |
222 | stpt __LC_EXIT_TIMER | |
223 | lm %r0,%r15,__PT_R0(%r11) | |
224 | lpsw __LC_RETURN_PSW | |
86ed42f4 | 225 | .Lsysc_done: |
411788ea | 226 | |
43d399d2 MS |
227 | # |
228 | # One of the work bits is on. Find out which one. | |
229 | # | |
86ed42f4 | 230 | .Lsysc_work: |
d3a73acb | 231 | tm __LC_CPU_FLAGS+3,_CIF_MCCK_PENDING |
86ed42f4 | 232 | jo .Lsysc_mcck_pending |
1de3447a | 233 | tm __TI_flags+3(%r12),_TIF_NEED_RESCHED |
86ed42f4 | 234 | jo .Lsysc_reschedule |
d3a73acb | 235 | tm __PT_FLAGS+3(%r11),_PIF_PER_TRAP |
86ed42f4 | 236 | jo .Lsysc_singlestep |
1de3447a | 237 | tm __TI_flags+3(%r12),_TIF_SIGPENDING |
86ed42f4 | 238 | jo .Lsysc_sigpending |
1de3447a | 239 | tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME |
86ed42f4 | 240 | jo .Lsysc_notify_resume |
d3a73acb | 241 | tm __LC_CPU_FLAGS+3,_CIF_ASCE |
86ed42f4 MS |
242 | jo .Lsysc_uaccess |
243 | j .Lsysc_return # beware of critical section cleanup | |
1da177e4 LT |
244 | |
245 | # | |
246 | # _TIF_NEED_RESCHED is set, call schedule | |
25d83cbf | 247 | # |
86ed42f4 MS |
248 | .Lsysc_reschedule: |
249 | l %r1,BASED(.Lc_schedule) | |
250 | la %r14,BASED(.Lsysc_return) | |
c5328901 | 251 | br %r1 # call schedule |
1da177e4 | 252 | |
77fa2245 | 253 | # |
d3a73acb | 254 | # _CIF_MCCK_PENDING is set, call handler |
77fa2245 | 255 | # |
86ed42f4 MS |
256 | .Lsysc_mcck_pending: |
257 | l %r1,BASED(.Lc_handle_mcck) | |
258 | la %r14,BASED(.Lsysc_return) | |
77fa2245 HC |
259 | br %r1 # TIF bit will be cleared by handler |
260 | ||
457f2180 | 261 | # |
d3a73acb | 262 | # _CIF_ASCE is set, load user space asce |
457f2180 | 263 | # |
86ed42f4 | 264 | .Lsysc_uaccess: |
d3a73acb | 265 | ni __LC_CPU_FLAGS+3,255-_CIF_ASCE |
457f2180 | 266 | lctl %c1,%c1,__LC_USER_ASCE # load primary asce |
86ed42f4 | 267 | j .Lsysc_return |
457f2180 | 268 | |
1da177e4 | 269 | # |
02a029b3 | 270 | # _TIF_SIGPENDING is set, call do_signal |
1da177e4 | 271 | # |
86ed42f4 | 272 | .Lsysc_sigpending: |
c5328901 | 273 | lr %r2,%r11 # pass pointer to pt_regs |
86ed42f4 | 274 | l %r1,BASED(.Lc_do_signal) |
25d83cbf | 275 | basr %r14,%r1 # call do_signal |
d3a73acb | 276 | tm __PT_FLAGS+3(%r11),_PIF_SYSCALL |
86ed42f4 | 277 | jno .Lsysc_return |
c5328901 | 278 | lm %r2,%r7,__PT_R2(%r11) # load svc arguments |
dbbfe487 | 279 | l %r10,__TI_sysc_table(%r12) # 31 bit system call table |
c5328901 | 280 | xr %r8,%r8 # svc 0 returns -ENOSYS |
aa33c8cb | 281 | clc __PT_INT_CODE+2(2,%r11),BASED(.Lnr_syscalls+2) |
86ed42f4 | 282 | jnl .Lsysc_nr_ok # invalid svc number -> do svc 0 |
aa33c8cb | 283 | lh %r8,__PT_INT_CODE+2(%r11) # load new svc number |
c5328901 | 284 | sla %r8,2 |
86ed42f4 | 285 | j .Lsysc_nr_ok # restart svc |
1da177e4 | 286 | |
753c4dd6 MS |
287 | # |
288 | # _TIF_NOTIFY_RESUME is set, call do_notify_resume | |
289 | # | |
86ed42f4 | 290 | .Lsysc_notify_resume: |
c5328901 | 291 | lr %r2,%r11 # pass pointer to pt_regs |
86ed42f4 MS |
292 | l %r1,BASED(.Lc_do_notify_resume) |
293 | la %r14,BASED(.Lsysc_return) | |
753c4dd6 MS |
294 | br %r1 # call do_notify_resume |
295 | ||
1da177e4 | 296 | # |
d3a73acb | 297 | # _PIF_PER_TRAP is set, call do_per_trap |
1da177e4 | 298 | # |
86ed42f4 | 299 | .Lsysc_singlestep: |
d3a73acb | 300 | ni __PT_FLAGS+3(%r11),255-_PIF_PER_TRAP |
c5328901 | 301 | lr %r2,%r11 # pass pointer to pt_regs |
86ed42f4 MS |
302 | l %r1,BASED(.Lc_do_per_trap) |
303 | la %r14,BASED(.Lsysc_return) | |
c5328901 | 304 | br %r1 # call do_per_trap |
1da177e4 | 305 | |
1da177e4 | 306 | # |
753c4dd6 MS |
307 | # call tracehook_report_syscall_entry/tracehook_report_syscall_exit before |
308 | # and after the system call | |
1da177e4 | 309 | # |
86ed42f4 MS |
310 | .Lsysc_tracesys: |
311 | l %r1,BASED(.Lc_trace_enter) | |
c5328901 | 312 | lr %r2,%r11 # pass pointer to pt_regs |
1da177e4 | 313 | la %r3,0 |
1de3447a | 314 | xr %r0,%r0 |
aa33c8cb | 315 | icm %r0,3,__PT_INT_CODE+2(%r11) |
c5328901 MS |
316 | st %r0,__PT_R2(%r11) |
317 | basr %r14,%r1 # call do_syscall_trace_enter | |
753c4dd6 | 318 | cl %r2,BASED(.Lnr_syscalls) |
86ed42f4 | 319 | jnl .Lsysc_tracenogo |
c5328901 MS |
320 | lr %r8,%r2 |
321 | sll %r8,2 | |
322 | l %r9,0(%r8,%r10) | |
86ed42f4 | 323 | .Lsysc_tracego: |
c5328901 MS |
324 | lm %r3,%r7,__PT_R3(%r11) |
325 | st %r7,STACK_FRAME_OVERHEAD(%r15) | |
326 | l %r2,__PT_ORIG_GPR2(%r11) | |
327 | basr %r14,%r9 # call sys_xxx | |
328 | st %r2,__PT_R2(%r11) # store return value | |
86ed42f4 | 329 | .Lsysc_tracenogo: |
d3a73acb | 330 | tm __TI_flags+3(%r12),_TIF_TRACE |
86ed42f4 MS |
331 | jz .Lsysc_return |
332 | l %r1,BASED(.Lc_trace_exit) | |
c5328901 | 333 | lr %r2,%r11 # pass pointer to pt_regs |
86ed42f4 | 334 | la %r14,BASED(.Lsysc_return) |
c5328901 | 335 | br %r1 # call do_syscall_trace_exit |
1da177e4 LT |
336 | |
337 | # | |
338 | # a new process exits the kernel with ret_from_fork | |
339 | # | |
144d634a | 340 | ENTRY(ret_from_fork) |
c5328901 MS |
341 | la %r11,STACK_FRAME_OVERHEAD(%r15) |
342 | l %r12,__LC_THREAD_INFO | |
1da177e4 | 343 | l %r13,__LC_SVC_NEW_PSW+4 |
86ed42f4 | 344 | l %r1,BASED(.Lc_schedule_tail) |
37fe5d41 AV |
345 | basr %r14,%r1 # call schedule_tail |
346 | TRACE_IRQS_ON | |
347 | ssm __LC_SVC_NEW_PSW # reenable interrupts | |
30dcb099 | 348 | tm __PT_PSW+1(%r11),0x01 # forking a kernel thread ? |
86ed42f4 | 349 | jne .Lsysc_tracenogo |
30dcb099 AV |
350 | # it's a kernel thread |
351 | lm %r9,%r10,__PT_R9(%r11) # load gprs | |
37fe5d41 AV |
352 | ENTRY(kernel_thread_starter) |
353 | la %r2,0(%r10) | |
354 | basr %r14,%r9 | |
86ed42f4 | 355 | j .Lsysc_tracenogo |
1da177e4 LT |
356 | |
357 | /* | |
358 | * Program check handler routine | |
359 | */ | |
360 | ||
144d634a | 361 | ENTRY(pgm_check_handler) |
c185b783 | 362 | stpt __LC_SYNC_ENTER_TIMER |
c5328901 MS |
363 | stm %r8,%r15,__LC_SAVE_AREA_SYNC |
364 | l %r12,__LC_THREAD_INFO | |
365 | l %r13,__LC_SVC_NEW_PSW+4 | |
366 | lm %r8,%r9,__LC_PGM_OLD_PSW | |
367 | tmh %r8,0x0001 # test problem state bit | |
368 | jnz 1f # -> fault in user space | |
369 | tmh %r8,0x4000 # PER bit set in old PSW ? | |
370 | jnz 0f # -> enabled, can't be a double fault | |
371 | tm __LC_PGM_ILC+3,0x80 # check for per exception | |
86ed42f4 | 372 | jnz .Lpgm_svcper # -> single stepped svc |
c5328901 | 373 | 0: CHECK_STACK STACK_SIZE,__LC_SAVE_AREA_SYNC |
dc7ee00d | 374 | ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) |
c5328901 MS |
375 | j 2f |
376 | 1: UPDATE_VTIME %r14,%r15,__LC_SYNC_ENTER_TIMER | |
377 | l %r15,__LC_KERNEL_STACK | |
dc7ee00d | 378 | 2: la %r11,STACK_FRAME_OVERHEAD(%r15) |
c5328901 MS |
379 | stm %r0,%r7,__PT_R0(%r11) |
380 | mvc __PT_R8(32,%r11),__LC_SAVE_AREA_SYNC | |
381 | stm %r8,%r9,__PT_PSW(%r11) | |
aa33c8cb MS |
382 | mvc __PT_INT_CODE(4,%r11),__LC_PGM_ILC |
383 | mvc __PT_INT_PARM_LONG(4,%r11),__LC_TRANS_EXC_CODE | |
d3a73acb | 384 | xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11) |
c5328901 MS |
385 | tm __LC_PGM_ILC+3,0x80 # check for per exception |
386 | jz 0f | |
1de3447a | 387 | l %r1,__TI_task(%r12) |
c5328901 | 388 | tmh %r8,0x0001 # kernel per event ? |
86ed42f4 | 389 | jz .Lpgm_kprobe |
d3a73acb | 390 | oi __PT_FLAGS+3(%r11),_PIF_PER_TRAP |
5e9a2692 | 391 | mvc __THREAD_per_address(4,%r1),__LC_PER_ADDRESS |
21ee7ffd JF |
392 | mvc __THREAD_per_cause(2,%r1),__LC_PER_CODE |
393 | mvc __THREAD_per_paid(1,%r1),__LC_PER_ACCESS_ID | |
aa33c8cb | 394 | 0: REENABLE_IRQS |
c5328901 | 395 | xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) |
86ed42f4 | 396 | l %r1,BASED(.Lc_jump_table) |
c5328901 | 397 | la %r10,0x7f |
aa33c8cb | 398 | n %r10,__PT_INT_CODE(%r11) |
86ed42f4 | 399 | je .Lsysc_return |
c5328901 MS |
400 | sll %r10,2 |
401 | l %r1,0(%r10,%r1) # load address of handler routine | |
402 | lr %r2,%r11 # pass pointer to pt_regs | |
1de3447a | 403 | basr %r14,%r1 # branch to interrupt-handler |
86ed42f4 | 404 | j .Lsysc_return |
1da177e4 LT |
405 | |
406 | # | |
c5328901 | 407 | # PER event in supervisor state, must be kprobes |
1da177e4 | 408 | # |
86ed42f4 | 409 | .Lpgm_kprobe: |
c5328901 MS |
410 | REENABLE_IRQS |
411 | xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) | |
86ed42f4 | 412 | l %r1,BASED(.Lc_do_per_trap) |
c5328901 MS |
413 | lr %r2,%r11 # pass pointer to pt_regs |
414 | basr %r14,%r1 # call do_per_trap | |
86ed42f4 | 415 | j .Lsysc_return |
1da177e4 | 416 | |
4ba069b8 | 417 | # |
c5328901 | 418 | # single stepped system call |
4ba069b8 | 419 | # |
86ed42f4 | 420 | .Lpgm_svcper: |
c5328901 | 421 | mvc __LC_RETURN_PSW(4),__LC_SVC_NEW_PSW |
86ed42f4 | 422 | mvc __LC_RETURN_PSW+4(4),BASED(.Lc_sysc_per) |
d3a73acb | 423 | lhi %r14,_PIF_SYSCALL | _PIF_PER_TRAP |
86ed42f4 | 424 | lpsw __LC_RETURN_PSW # branch to .Lsysc_per and enable irqs |
4ba069b8 | 425 | |
1da177e4 LT |
426 | /* |
427 | * IO interrupt handler routine | |
428 | */ | |
429 | ||
144d634a | 430 | ENTRY(io_int_handler) |
1da177e4 | 431 | stck __LC_INT_CLOCK |
9cfb9b3c | 432 | stpt __LC_ASYNC_ENTER_TIMER |
c5328901 MS |
433 | stm %r8,%r15,__LC_SAVE_AREA_ASYNC |
434 | l %r12,__LC_THREAD_INFO | |
435 | l %r13,__LC_SVC_NEW_PSW+4 | |
436 | lm %r8,%r9,__LC_IO_OLD_PSW | |
437 | tmh %r8,0x0001 # interrupting from user ? | |
86ed42f4 | 438 | jz .Lio_skip |
c5328901 | 439 | UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER |
86ed42f4 | 440 | .Lio_skip: |
c5328901 MS |
441 | SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT |
442 | stm %r0,%r7,__PT_R0(%r11) | |
443 | mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC | |
444 | stm %r8,%r9,__PT_PSW(%r11) | |
48f6b00c | 445 | mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID |
d3a73acb | 446 | xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11) |
1f194a4c | 447 | TRACE_IRQS_OFF |
c5328901 | 448 | xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) |
86ed42f4 MS |
449 | .Lio_loop: |
450 | l %r1,BASED(.Lc_do_IRQ) | |
c5328901 | 451 | lr %r2,%r11 # pass pointer to pt_regs |
1f44a225 MS |
452 | lhi %r3,IO_INTERRUPT |
453 | tm __PT_INT_CODE+8(%r11),0x80 # adapter interrupt ? | |
86ed42f4 | 454 | jz .Lio_call |
1f44a225 | 455 | lhi %r3,THIN_INTERRUPT |
86ed42f4 | 456 | .Lio_call: |
c5328901 | 457 | basr %r14,%r1 # call do_IRQ |
48f6b00c | 458 | tm __LC_MACHINE_FLAGS+2,0x10 # MACHINE_FLAG_LPAR |
86ed42f4 | 459 | jz .Lio_return |
48f6b00c | 460 | tpi 0 |
86ed42f4 | 461 | jz .Lio_return |
48f6b00c | 462 | mvc __PT_INT_CODE(12,%r11),__LC_SUBCHANNEL_ID |
86ed42f4 MS |
463 | j .Lio_loop |
464 | .Lio_return: | |
6a2df3a8 MS |
465 | LOCKDEP_SYS_EXIT |
466 | TRACE_IRQS_ON | |
86ed42f4 | 467 | .Lio_tif: |
d3a73acb | 468 | tm __TI_flags+3(%r12),_TIF_WORK |
86ed42f4 | 469 | jnz .Lio_work # there is work to do (signals etc.) |
d3a73acb | 470 | tm __LC_CPU_FLAGS+3,_CIF_WORK |
86ed42f4 MS |
471 | jnz .Lio_work |
472 | .Lio_restore: | |
c5328901 | 473 | mvc __LC_RETURN_PSW(8),__PT_PSW(%r11) |
c5328901 MS |
474 | stpt __LC_EXIT_TIMER |
475 | lm %r0,%r15,__PT_R0(%r11) | |
476 | lpsw __LC_RETURN_PSW | |
86ed42f4 | 477 | .Lio_done: |
1da177e4 | 478 | |
2688905e | 479 | # |
43d399d2 | 480 | # There is work todo, find out in which context we have been interrupted: |
d3a73acb | 481 | # 1) if we return to user space we can do all _TIF_WORK work |
43d399d2 MS |
482 | # 2) if we return to kernel code and preemptive scheduling is enabled check |
483 | # the preemption counter and if it is zero call preempt_schedule_irq | |
484 | # Before any work can be done, a switch to the kernel stack is required. | |
2688905e | 485 | # |
86ed42f4 | 486 | .Lio_work: |
c5328901 | 487 | tm __PT_PSW+1(%r11),0x01 # returning to user ? |
86ed42f4 | 488 | jo .Lio_work_user # yes -> do resched & signal |
43d399d2 | 489 | #ifdef CONFIG_PREEMPT |
2688905e | 490 | # check for preemptive scheduling |
1de3447a | 491 | icm %r0,15,__TI_precount(%r12) |
86ed42f4 | 492 | jnz .Lio_restore # preemption disabled |
1de3447a | 493 | tm __TI_flags+3(%r12),_TIF_NEED_RESCHED |
86ed42f4 | 494 | jno .Lio_restore |
43d399d2 | 495 | # switch to kernel stack |
c5328901 MS |
496 | l %r1,__PT_R15(%r11) |
497 | ahi %r1,-(STACK_FRAME_OVERHEAD + __PT_SIZE) | |
498 | mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) | |
499 | xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) | |
500 | la %r11,STACK_FRAME_OVERHEAD(%r1) | |
1da177e4 | 501 | lr %r15,%r1 |
86ed42f4 | 502 | # TRACE_IRQS_ON already done at .Lio_return, call |
6a2df3a8 MS |
503 | # TRACE_IRQS_OFF to keep things symmetrical |
504 | TRACE_IRQS_OFF | |
86ed42f4 | 505 | l %r1,BASED(.Lc_preempt_irq) |
6a2df3a8 | 506 | basr %r14,%r1 # call preempt_schedule_irq |
86ed42f4 | 507 | j .Lio_return |
6a2df3a8 | 508 | #else |
86ed42f4 | 509 | j .Lio_restore |
6a2df3a8 | 510 | #endif |
1da177e4 | 511 | |
43d399d2 MS |
512 | # |
513 | # Need to do work before returning to userspace, switch to kernel stack | |
514 | # | |
86ed42f4 | 515 | .Lio_work_user: |
1da177e4 | 516 | l %r1,__LC_KERNEL_STACK |
c5328901 MS |
517 | mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) |
518 | xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) | |
519 | la %r11,STACK_FRAME_OVERHEAD(%r1) | |
1da177e4 | 520 | lr %r15,%r1 |
6a2df3a8 | 521 | |
1da177e4 LT |
522 | # |
523 | # One of the work bits is on. Find out which one. | |
1da177e4 | 524 | # |
86ed42f4 | 525 | .Lio_work_tif: |
d3a73acb | 526 | tm __LC_CPU_FLAGS+3(%r12),_CIF_MCCK_PENDING |
86ed42f4 | 527 | jo .Lio_mcck_pending |
1de3447a | 528 | tm __TI_flags+3(%r12),_TIF_NEED_RESCHED |
86ed42f4 | 529 | jo .Lio_reschedule |
1de3447a | 530 | tm __TI_flags+3(%r12),_TIF_SIGPENDING |
86ed42f4 | 531 | jo .Lio_sigpending |
1de3447a | 532 | tm __TI_flags+3(%r12),_TIF_NOTIFY_RESUME |
86ed42f4 | 533 | jo .Lio_notify_resume |
d3a73acb | 534 | tm __LC_CPU_FLAGS+3,_CIF_ASCE |
86ed42f4 MS |
535 | jo .Lio_uaccess |
536 | j .Lio_return # beware of critical section cleanup | |
1da177e4 | 537 | |
77fa2245 | 538 | # |
d3a73acb | 539 | # _CIF_MCCK_PENDING is set, call handler |
77fa2245 | 540 | # |
86ed42f4 MS |
541 | .Lio_mcck_pending: |
542 | # TRACE_IRQS_ON already done at .Lio_return | |
543 | l %r1,BASED(.Lc_handle_mcck) | |
b771aeac | 544 | basr %r14,%r1 # TIF bit will be cleared by handler |
6a2df3a8 | 545 | TRACE_IRQS_OFF |
86ed42f4 | 546 | j .Lio_return |
77fa2245 | 547 | |
457f2180 | 548 | # |
d3a73acb | 549 | # _CIF_ASCE is set, load user space asce |
457f2180 | 550 | # |
86ed42f4 | 551 | .Lio_uaccess: |
d3a73acb | 552 | ni __LC_CPU_FLAGS+3,255-_CIF_ASCE |
457f2180 | 553 | lctl %c1,%c1,__LC_USER_ASCE # load primary asce |
86ed42f4 | 554 | j .Lio_return |
457f2180 | 555 | |
1da177e4 LT |
556 | # |
557 | # _TIF_NEED_RESCHED is set, call schedule | |
25d83cbf | 558 | # |
86ed42f4 MS |
559 | .Lio_reschedule: |
560 | # TRACE_IRQS_ON already done at .Lio_return | |
561 | l %r1,BASED(.Lc_schedule) | |
c5328901 | 562 | ssm __LC_SVC_NEW_PSW # reenable interrupts |
25d83cbf | 563 | basr %r14,%r1 # call scheduler |
c5328901 | 564 | ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts |
411788ea | 565 | TRACE_IRQS_OFF |
86ed42f4 | 566 | j .Lio_return |
1da177e4 LT |
567 | |
568 | # | |
02a029b3 | 569 | # _TIF_SIGPENDING is set, call do_signal |
1da177e4 | 570 | # |
86ed42f4 MS |
571 | .Lio_sigpending: |
572 | # TRACE_IRQS_ON already done at .Lio_return | |
573 | l %r1,BASED(.Lc_do_signal) | |
c5328901 MS |
574 | ssm __LC_SVC_NEW_PSW # reenable interrupts |
575 | lr %r2,%r11 # pass pointer to pt_regs | |
25d83cbf | 576 | basr %r14,%r1 # call do_signal |
c5328901 | 577 | ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts |
411788ea | 578 | TRACE_IRQS_OFF |
86ed42f4 | 579 | j .Lio_return |
1da177e4 | 580 | |
753c4dd6 MS |
581 | # |
582 | # _TIF_SIGPENDING is set, call do_signal | |
583 | # | |
86ed42f4 MS |
584 | .Lio_notify_resume: |
585 | # TRACE_IRQS_ON already done at .Lio_return | |
586 | l %r1,BASED(.Lc_do_notify_resume) | |
c5328901 MS |
587 | ssm __LC_SVC_NEW_PSW # reenable interrupts |
588 | lr %r2,%r11 # pass pointer to pt_regs | |
589 | basr %r14,%r1 # call do_notify_resume | |
590 | ssm __LC_PGM_NEW_PSW # disable I/O and ext. interrupts | |
753c4dd6 | 591 | TRACE_IRQS_OFF |
86ed42f4 | 592 | j .Lio_return |
753c4dd6 | 593 | |
1da177e4 LT |
594 | /* |
595 | * External interrupt handler routine | |
596 | */ | |
597 | ||
144d634a | 598 | ENTRY(ext_int_handler) |
1da177e4 | 599 | stck __LC_INT_CLOCK |
9cfb9b3c | 600 | stpt __LC_ASYNC_ENTER_TIMER |
c5328901 MS |
601 | stm %r8,%r15,__LC_SAVE_AREA_ASYNC |
602 | l %r12,__LC_THREAD_INFO | |
603 | l %r13,__LC_SVC_NEW_PSW+4 | |
604 | lm %r8,%r9,__LC_EXT_OLD_PSW | |
605 | tmh %r8,0x0001 # interrupting from user ? | |
86ed42f4 | 606 | jz .Lext_skip |
c5328901 | 607 | UPDATE_VTIME %r14,%r15,__LC_ASYNC_ENTER_TIMER |
86ed42f4 | 608 | .Lext_skip: |
c5328901 MS |
609 | SWITCH_ASYNC __LC_SAVE_AREA_ASYNC,__LC_ASYNC_STACK,STACK_SHIFT |
610 | stm %r0,%r7,__PT_R0(%r11) | |
611 | mvc __PT_R8(32,%r11),__LC_SAVE_AREA_ASYNC | |
612 | stm %r8,%r9,__PT_PSW(%r11) | |
48f6b00c MS |
613 | mvc __PT_INT_CODE(4,%r11),__LC_EXT_CPU_ADDR |
614 | mvc __PT_INT_PARM(4,%r11),__LC_EXT_PARAMS | |
d3a73acb | 615 | xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11) |
1f194a4c | 616 | TRACE_IRQS_OFF |
86ed42f4 | 617 | l %r1,BASED(.Lc_do_IRQ) |
c5328901 | 618 | lr %r2,%r11 # pass pointer to pt_regs |
1f44a225 MS |
619 | lhi %r3,EXT_INTERRUPT |
620 | basr %r14,%r1 # call do_IRQ | |
86ed42f4 | 621 | j .Lio_return |
1da177e4 | 622 | |
4c1051e3 | 623 | /* |
86ed42f4 | 624 | * Load idle PSW. The second "half" of this function is in .Lcleanup_idle. |
4c1051e3 MS |
625 | */ |
626 | ENTRY(psw_idle) | |
27f6b416 | 627 | st %r3,__SF_EMPTY(%r15) |
4c1051e3 | 628 | basr %r1,0 |
86ed42f4 | 629 | la %r1,.Lpsw_idle_lpsw+4-.(%r1) |
4c1051e3 MS |
630 | st %r1,__SF_EMPTY+4(%r15) |
631 | oi __SF_EMPTY+4(%r15),0x80 | |
27f6b416 MS |
632 | stck __CLOCK_IDLE_ENTER(%r2) |
633 | stpt __TIMER_IDLE_ENTER(%r2) | |
86ed42f4 | 634 | .Lpsw_idle_lpsw: |
4c1051e3 MS |
635 | lpsw __SF_EMPTY(%r15) |
636 | br %r14 | |
86ed42f4 | 637 | .Lpsw_idle_end: |
4c1051e3 | 638 | |
86ed42f4 | 639 | .L__critical_end: |
ae6aa2ea | 640 | |
1da177e4 LT |
641 | /* |
642 | * Machine check handler routines | |
643 | */ | |
644 | ||
144d634a | 645 | ENTRY(mcck_int_handler) |
6377981f | 646 | stck __LC_MCCK_CLOCK |
77fa2245 HC |
647 | spt __LC_CPU_TIMER_SAVE_AREA # revalidate cpu timer |
648 | lm %r0,%r15,__LC_GPREGS_SAVE_AREA # revalidate gprs | |
c5328901 MS |
649 | l %r12,__LC_THREAD_INFO |
650 | l %r13,__LC_SVC_NEW_PSW+4 | |
651 | lm %r8,%r9,__LC_MCK_OLD_PSW | |
25d83cbf | 652 | tm __LC_MCCK_CODE,0x80 # system damage? |
86ed42f4 | 653 | jo .Lmcck_panic # yes -> rest of mcck code invalid |
c5328901 MS |
654 | la %r14,__LC_CPU_TIMER_SAVE_AREA |
655 | mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) | |
63b12246 | 656 | tm __LC_MCCK_CODE+5,0x02 # stored cpu timer value valid? |
c5328901 | 657 | jo 3f |
63b12246 MS |
658 | la %r14,__LC_SYNC_ENTER_TIMER |
659 | clc 0(8,%r14),__LC_ASYNC_ENTER_TIMER | |
c5328901 | 660 | jl 0f |
63b12246 MS |
661 | la %r14,__LC_ASYNC_ENTER_TIMER |
662 | 0: clc 0(8,%r14),__LC_EXIT_TIMER | |
c5328901 | 663 | jl 1f |
63b12246 | 664 | la %r14,__LC_EXIT_TIMER |
c5328901 MS |
665 | 1: clc 0(8,%r14),__LC_LAST_UPDATE_TIMER |
666 | jl 2f | |
63b12246 | 667 | la %r14,__LC_LAST_UPDATE_TIMER |
c5328901 | 668 | 2: spt 0(%r14) |
6377981f | 669 | mvc __LC_MCCK_ENTER_TIMER(8),0(%r14) |
c5328901 | 670 | 3: tm __LC_MCCK_CODE+2,0x09 # mwp + ia of old psw valid? |
86ed42f4 | 671 | jno .Lmcck_panic # no -> skip cleanup critical |
c5328901 | 672 | tm %r8,0x0001 # interrupting from user ? |
86ed42f4 | 673 | jz .Lmcck_skip |
c5328901 | 674 | UPDATE_VTIME %r14,%r15,__LC_MCCK_ENTER_TIMER |
86ed42f4 | 675 | .Lmcck_skip: |
c5328901 | 676 | SWITCH_ASYNC __LC_GPREGS_SAVE_AREA+32,__LC_PANIC_STACK,PAGE_SHIFT |
6551fbdf MS |
677 | stm %r0,%r7,__PT_R0(%r11) |
678 | mvc __PT_R8(32,%r11),__LC_GPREGS_SAVE_AREA+32 | |
c5328901 | 679 | stm %r8,%r9,__PT_PSW(%r11) |
d3a73acb | 680 | xc __PT_FLAGS(4,%r11),__PT_FLAGS(%r11) |
c5328901 | 681 | xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) |
86ed42f4 | 682 | l %r1,BASED(.Lc_do_machine_check) |
c5328901 MS |
683 | lr %r2,%r11 # pass pointer to pt_regs |
684 | basr %r14,%r1 # call s390_do_machine_check | |
685 | tm __PT_PSW+1(%r11),0x01 # returning to user ? | |
86ed42f4 | 686 | jno .Lmcck_return |
25d83cbf | 687 | l %r1,__LC_KERNEL_STACK # switch to kernel stack |
c5328901 MS |
688 | mvc STACK_FRAME_OVERHEAD(__PT_SIZE,%r1),0(%r11) |
689 | xc __SF_BACKCHAIN(4,%r1),__SF_BACKCHAIN(%r1) | |
690 | la %r11,STACK_FRAME_OVERHEAD(%r15) | |
77fa2245 | 691 | lr %r15,%r1 |
c5328901 | 692 | ssm __LC_PGM_NEW_PSW # turn dat on, keep irqs off |
d3a73acb | 693 | tm __LC_CPU_FLAGS+3,_CIF_MCCK_PENDING |
86ed42f4 | 694 | jno .Lmcck_return |
1f194a4c | 695 | TRACE_IRQS_OFF |
86ed42f4 | 696 | l %r1,BASED(.Lc_handle_mcck) |
c5328901 | 697 | basr %r14,%r1 # call s390_handle_mcck |
1f194a4c | 698 | TRACE_IRQS_ON |
86ed42f4 | 699 | .Lmcck_return: |
c5328901 | 700 | mvc __LC_RETURN_MCCK_PSW(8),__PT_PSW(%r11) # move return PSW |
63b12246 | 701 | tm __LC_RETURN_MCCK_PSW+1,0x01 # returning to user ? |
c5328901 MS |
702 | jno 0f |
703 | lm %r0,%r15,__PT_R0(%r11) | |
63b12246 | 704 | stpt __LC_EXIT_TIMER |
c5328901 MS |
705 | lpsw __LC_RETURN_MCCK_PSW |
706 | 0: lm %r0,%r15,__PT_R0(%r11) | |
707 | lpsw __LC_RETURN_MCCK_PSW | |
63b12246 | 708 | |
86ed42f4 | 709 | .Lmcck_panic: |
c5328901 MS |
710 | l %r14,__LC_PANIC_STACK |
711 | slr %r14,%r15 | |
712 | sra %r14,PAGE_SHIFT | |
713 | jz 0f | |
714 | l %r15,__LC_PANIC_STACK | |
86ed42f4 | 715 | j .Lmcck_skip |
c5328901 | 716 | 0: ahi %r15,-(STACK_FRAME_OVERHEAD + __PT_SIZE) |
86ed42f4 | 717 | j .Lmcck_skip |
1da177e4 | 718 | |
7dd6b334 MH |
719 | # |
720 | # PSW restart interrupt handler | |
721 | # | |
8b646bd7 | 722 | ENTRY(restart_int_handler) |
c5328901 | 723 | st %r15,__LC_SAVE_AREA_RESTART |
8b646bd7 | 724 | l %r15,__LC_RESTART_STACK |
c5328901 | 725 | ahi %r15,-__PT_SIZE # create pt_regs on stack |
8b646bd7 | 726 | xc 0(__PT_SIZE,%r15),0(%r15) |
c5328901 MS |
727 | stm %r0,%r14,__PT_R0(%r15) |
728 | mvc __PT_R15(4,%r15),__LC_SAVE_AREA_RESTART | |
729 | mvc __PT_PSW(8,%r15),__LC_RST_OLD_PSW # store restart old psw | |
8b646bd7 MS |
730 | ahi %r15,-STACK_FRAME_OVERHEAD # create stack frame on stack |
731 | xc 0(STACK_FRAME_OVERHEAD,%r15),0(%r15) | |
fbe76568 HC |
732 | l %r1,__LC_RESTART_FN # load fn, parm & source cpu |
733 | l %r2,__LC_RESTART_DATA | |
734 | l %r3,__LC_RESTART_SOURCE | |
8b646bd7 MS |
735 | ltr %r3,%r3 # test source cpu address |
736 | jm 1f # negative -> skip source stop | |
eb546195 | 737 | 0: sigp %r4,%r3,SIGP_SENSE # sigp sense to source cpu |
8b646bd7 MS |
738 | brc 10,0b # wait for status stored |
739 | 1: basr %r14,%r1 # call function | |
740 | stap __SF_EMPTY(%r15) # store cpu address | |
741 | lh %r3,__SF_EMPTY(%r15) | |
eb546195 | 742 | 2: sigp %r4,%r3,SIGP_STOP # sigp stop to current cpu |
8b646bd7 MS |
743 | brc 2,2b |
744 | 3: j 3b | |
7dd6b334 | 745 | |
860dba45 MS |
746 | .section .kprobes.text, "ax" |
747 | ||
1da177e4 LT |
748 | #ifdef CONFIG_CHECK_STACK |
749 | /* | |
750 | * The synchronous or the asynchronous stack overflowed. We are dead. | |
751 | * No need to properly save the registers, we are going to panic anyway. | |
752 | * Setup a pt_regs so that show_trace can provide a good call trace. | |
753 | */ | |
754 | stack_overflow: | |
755 | l %r15,__LC_PANIC_STACK # change to panic stack | |
dc7ee00d MS |
756 | la %r11,STACK_FRAME_OVERHEAD(%r15) |
757 | stm %r0,%r7,__PT_R0(%r11) | |
758 | stm %r8,%r9,__PT_PSW(%r11) | |
c5328901 | 759 | mvc __PT_R8(32,%r11),0(%r14) |
c5328901 MS |
760 | l %r1,BASED(1f) |
761 | xc __SF_BACKCHAIN(4,%r15),__SF_BACKCHAIN(%r15) | |
762 | lr %r2,%r11 # pass pointer to pt_regs | |
763 | br %r1 # branch to kernel_stack_overflow | |
25d83cbf | 764 | 1: .long kernel_stack_overflow |
1da177e4 LT |
765 | #endif |
766 | ||
86ed42f4 | 767 | .Lcleanup_table: |
c5328901 | 768 | .long system_call + 0x80000000 |
86ed42f4 MS |
769 | .long .Lsysc_do_svc + 0x80000000 |
770 | .long .Lsysc_tif + 0x80000000 | |
771 | .long .Lsysc_restore + 0x80000000 | |
772 | .long .Lsysc_done + 0x80000000 | |
773 | .long .Lio_tif + 0x80000000 | |
774 | .long .Lio_restore + 0x80000000 | |
775 | .long .Lio_done + 0x80000000 | |
4c1051e3 | 776 | .long psw_idle + 0x80000000 |
86ed42f4 | 777 | .long .Lpsw_idle_end + 0x80000000 |
1da177e4 LT |
778 | |
779 | cleanup_critical: | |
86ed42f4 | 780 | cl %r9,BASED(.Lcleanup_table) # system_call |
c5328901 | 781 | jl 0f |
86ed42f4 MS |
782 | cl %r9,BASED(.Lcleanup_table+4) # .Lsysc_do_svc |
783 | jl .Lcleanup_system_call | |
784 | cl %r9,BASED(.Lcleanup_table+8) # .Lsysc_tif | |
c5328901 | 785 | jl 0f |
86ed42f4 MS |
786 | cl %r9,BASED(.Lcleanup_table+12) # .Lsysc_restore |
787 | jl .Lcleanup_sysc_tif | |
788 | cl %r9,BASED(.Lcleanup_table+16) # .Lsysc_done | |
789 | jl .Lcleanup_sysc_restore | |
790 | cl %r9,BASED(.Lcleanup_table+20) # .Lio_tif | |
c5328901 | 791 | jl 0f |
86ed42f4 MS |
792 | cl %r9,BASED(.Lcleanup_table+24) # .Lio_restore |
793 | jl .Lcleanup_io_tif | |
794 | cl %r9,BASED(.Lcleanup_table+28) # .Lio_done | |
795 | jl .Lcleanup_io_restore | |
796 | cl %r9,BASED(.Lcleanup_table+32) # psw_idle | |
4c1051e3 | 797 | jl 0f |
86ed42f4 MS |
798 | cl %r9,BASED(.Lcleanup_table+36) # .Lpsw_idle_end |
799 | jl .Lcleanup_idle | |
c5328901 | 800 | 0: br %r14 |
1da177e4 | 801 | |
86ed42f4 | 802 | .Lcleanup_system_call: |
c5328901 | 803 | # check if stpt has been executed |
86ed42f4 | 804 | cl %r9,BASED(.Lcleanup_system_call_insn) |
c5328901 | 805 | jh 0f |
1da177e4 | 806 | mvc __LC_SYNC_ENTER_TIMER(8),__LC_ASYNC_ENTER_TIMER |
c5328901 MS |
807 | chi %r11,__LC_SAVE_AREA_ASYNC |
808 | je 0f | |
809 | mvc __LC_SYNC_ENTER_TIMER(8),__LC_MCCK_ENTER_TIMER | |
810 | 0: # check if stm has been executed | |
86ed42f4 | 811 | cl %r9,BASED(.Lcleanup_system_call_insn+4) |
c5328901 MS |
812 | jh 0f |
813 | mvc __LC_SAVE_AREA_SYNC(32),0(%r11) | |
814 | 0: # set up saved registers r12, and r13 | |
815 | st %r12,16(%r11) # r12 thread-info pointer | |
816 | st %r13,20(%r11) # r13 literal-pool pointer | |
817 | # check if the user time calculation has been done | |
86ed42f4 | 818 | cl %r9,BASED(.Lcleanup_system_call_insn+8) |
c5328901 MS |
819 | jh 0f |
820 | l %r10,__LC_EXIT_TIMER | |
821 | l %r15,__LC_EXIT_TIMER+4 | |
822 | SUB64 %r10,%r15,__LC_SYNC_ENTER_TIMER | |
823 | ADD64 %r10,%r15,__LC_USER_TIMER | |
824 | st %r10,__LC_USER_TIMER | |
825 | st %r15,__LC_USER_TIMER+4 | |
826 | 0: # check if the system time calculation has been done | |
86ed42f4 | 827 | cl %r9,BASED(.Lcleanup_system_call_insn+12) |
c5328901 MS |
828 | jh 0f |
829 | l %r10,__LC_LAST_UPDATE_TIMER | |
830 | l %r15,__LC_LAST_UPDATE_TIMER+4 | |
831 | SUB64 %r10,%r15,__LC_EXIT_TIMER | |
832 | ADD64 %r10,%r15,__LC_SYSTEM_TIMER | |
833 | st %r10,__LC_SYSTEM_TIMER | |
834 | st %r15,__LC_SYSTEM_TIMER+4 | |
835 | 0: # update accounting time stamp | |
1da177e4 | 836 | mvc __LC_LAST_UPDATE_TIMER(8),__LC_SYNC_ENTER_TIMER |
c5328901 MS |
837 | # set up saved register 11 |
838 | l %r15,__LC_KERNEL_STACK | |
dc7ee00d MS |
839 | la %r9,STACK_FRAME_OVERHEAD(%r15) |
840 | st %r9,12(%r11) # r11 pt_regs pointer | |
c5328901 | 841 | # fill pt_regs |
dc7ee00d MS |
842 | mvc __PT_R8(32,%r9),__LC_SAVE_AREA_SYNC |
843 | stm %r0,%r7,__PT_R0(%r9) | |
844 | mvc __PT_PSW(8,%r9),__LC_SVC_OLD_PSW | |
845 | mvc __PT_INT_CODE(4,%r9),__LC_SVC_ILC | |
d3a73acb MS |
846 | xc __PT_FLAGS(4,%r9),__PT_FLAGS(%r9) |
847 | mvi __PT_FLAGS+3(%r9),_PIF_SYSCALL | |
c5328901 | 848 | # setup saved register 15 |
c5328901 MS |
849 | st %r15,28(%r11) # r15 stack pointer |
850 | # set new psw address and exit | |
86ed42f4 | 851 | l %r9,BASED(.Lcleanup_table+4) # .Lsysc_do_svc + 0x80000000 |
1da177e4 | 852 | br %r14 |
86ed42f4 | 853 | .Lcleanup_system_call_insn: |
25d83cbf | 854 | .long system_call + 0x80000000 |
86ed42f4 MS |
855 | .long .Lsysc_stm + 0x80000000 |
856 | .long .Lsysc_vtime + 0x80000000 + 36 | |
857 | .long .Lsysc_vtime + 0x80000000 + 76 | |
1da177e4 | 858 | |
86ed42f4 MS |
859 | .Lcleanup_sysc_tif: |
860 | l %r9,BASED(.Lcleanup_table+8) # .Lsysc_tif + 0x80000000 | |
1da177e4 LT |
861 | br %r14 |
862 | ||
86ed42f4 MS |
863 | .Lcleanup_sysc_restore: |
864 | cl %r9,BASED(.Lcleanup_sysc_restore_insn) | |
c5328901 MS |
865 | jhe 0f |
866 | l %r9,12(%r11) # get saved pointer to pt_regs | |
867 | mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) | |
868 | mvc 0(32,%r11),__PT_R8(%r9) | |
869 | lm %r0,%r7,__PT_R0(%r9) | |
870 | 0: lm %r8,%r9,__LC_RETURN_PSW | |
1da177e4 | 871 | br %r14 |
86ed42f4 MS |
872 | .Lcleanup_sysc_restore_insn: |
873 | .long .Lsysc_done - 4 + 0x80000000 | |
1da177e4 | 874 | |
86ed42f4 MS |
875 | .Lcleanup_io_tif: |
876 | l %r9,BASED(.Lcleanup_table+20) # .Lio_tif + 0x80000000 | |
176b1803 MS |
877 | br %r14 |
878 | ||
86ed42f4 MS |
879 | .Lcleanup_io_restore: |
880 | cl %r9,BASED(.Lcleanup_io_restore_insn) | |
c5328901 MS |
881 | jhe 0f |
882 | l %r9,12(%r11) # get saved r11 pointer to pt_regs | |
883 | mvc __LC_RETURN_PSW(8),__PT_PSW(%r9) | |
c5328901 MS |
884 | mvc 0(32,%r11),__PT_R8(%r9) |
885 | lm %r0,%r7,__PT_R0(%r9) | |
886 | 0: lm %r8,%r9,__LC_RETURN_PSW | |
ae6aa2ea | 887 | br %r14 |
86ed42f4 MS |
888 | .Lcleanup_io_restore_insn: |
889 | .long .Lio_done - 4 + 0x80000000 | |
ae6aa2ea | 890 | |
86ed42f4 | 891 | .Lcleanup_idle: |
4c1051e3 | 892 | # copy interrupt clock & cpu timer |
27f6b416 MS |
893 | mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_INT_CLOCK |
894 | mvc __TIMER_IDLE_EXIT(8,%r2),__LC_ASYNC_ENTER_TIMER | |
4c1051e3 MS |
895 | chi %r11,__LC_SAVE_AREA_ASYNC |
896 | je 0f | |
27f6b416 MS |
897 | mvc __CLOCK_IDLE_EXIT(8,%r2),__LC_MCCK_CLOCK |
898 | mvc __TIMER_IDLE_EXIT(8,%r2),__LC_MCCK_ENTER_TIMER | |
4c1051e3 | 899 | 0: # check if stck has been executed |
86ed42f4 | 900 | cl %r9,BASED(.Lcleanup_idle_insn) |
4c1051e3 | 901 | jhe 1f |
27f6b416 MS |
902 | mvc __CLOCK_IDLE_ENTER(8,%r2),__CLOCK_IDLE_EXIT(%r2) |
903 | mvc __TIMER_IDLE_ENTER(8,%r2),__TIMER_IDLE_EXIT(%r3) | |
904 | 1: # account system time going idle | |
4c1051e3 | 905 | lm %r9,%r10,__LC_STEAL_TIMER |
27f6b416 | 906 | ADD64 %r9,%r10,__CLOCK_IDLE_ENTER(%r2) |
4c1051e3 MS |
907 | SUB64 %r9,%r10,__LC_LAST_UPDATE_CLOCK |
908 | stm %r9,%r10,__LC_STEAL_TIMER | |
27f6b416 | 909 | mvc __LC_LAST_UPDATE_CLOCK(8),__CLOCK_IDLE_EXIT(%r2) |
4c1051e3 MS |
910 | lm %r9,%r10,__LC_SYSTEM_TIMER |
911 | ADD64 %r9,%r10,__LC_LAST_UPDATE_TIMER | |
27f6b416 | 912 | SUB64 %r9,%r10,__TIMER_IDLE_ENTER(%r2) |
4c1051e3 | 913 | stm %r9,%r10,__LC_SYSTEM_TIMER |
27f6b416 | 914 | mvc __LC_LAST_UPDATE_TIMER(8),__TIMER_IDLE_EXIT(%r2) |
4c1051e3 | 915 | # prepare return psw |
86ed42f4 | 916 | n %r8,BASED(.Lcleanup_idle_wait) # clear irq & wait state bits |
4c1051e3 MS |
917 | l %r9,24(%r11) # return from psw_idle |
918 | br %r14 | |
86ed42f4 MS |
919 | .Lcleanup_idle_insn: |
920 | .long .Lpsw_idle_lpsw + 0x80000000 | |
921 | .Lcleanup_idle_wait: | |
0587d409 | 922 | .long 0xfcfdffff |
4c1051e3 | 923 | |
1da177e4 LT |
924 | /* |
925 | * Integer constants | |
926 | */ | |
c5328901 | 927 | .align 4 |
4c1051e3 MS |
928 | .Lnr_syscalls: |
929 | .long NR_syscalls | |
930 | .Lvtimer_max: | |
931 | .quad 0x7fffffffffffffff | |
1da177e4 LT |
932 | |
933 | /* | |
934 | * Symbol constants | |
935 | */ | |
86ed42f4 MS |
936 | .Lc_do_machine_check: .long s390_do_machine_check |
937 | .Lc_handle_mcck: .long s390_handle_mcck | |
938 | .Lc_do_IRQ: .long do_IRQ | |
939 | .Lc_do_signal: .long do_signal | |
940 | .Lc_do_notify_resume: .long do_notify_resume | |
941 | .Lc_do_per_trap: .long do_per_trap | |
942 | .Lc_jump_table: .long pgm_check_table | |
943 | .Lc_schedule: .long schedule | |
ab1809b4 | 944 | #ifdef CONFIG_PREEMPT |
86ed42f4 | 945 | .Lc_preempt_irq: .long preempt_schedule_irq |
ab1809b4 | 946 | #endif |
86ed42f4 MS |
947 | .Lc_trace_enter: .long do_syscall_trace_enter |
948 | .Lc_trace_exit: .long do_syscall_trace_exit | |
949 | .Lc_schedule_tail: .long schedule_tail | |
950 | .Lc_sysc_per: .long .Lsysc_per + 0x80000000 | |
1f194a4c | 951 | #ifdef CONFIG_TRACE_IRQFLAGS |
86ed42f4 MS |
952 | .Lc_hardirqs_on: .long trace_hardirqs_on_caller |
953 | .Lc_hardirqs_off: .long trace_hardirqs_off_caller | |
af4c6874 HC |
954 | #endif |
955 | #ifdef CONFIG_LOCKDEP | |
86ed42f4 | 956 | .Lc_lockdep_sys_exit: .long lockdep_sys_exit |
1f194a4c | 957 | #endif |
86ed42f4 MS |
958 | .Lc_critical_start: .long .L__critical_start + 0x80000000 |
959 | .Lc_critical_length: .long .L__critical_end - .L__critical_start | |
1da177e4 | 960 | |
25d83cbf | 961 | .section .rodata, "a" |
1da177e4 | 962 | #define SYSCALL(esa,esame,emu) .long esa |
9bf1226b | 963 | .globl sys_call_table |
1da177e4 LT |
964 | sys_call_table: |
965 | #include "syscalls.S" | |
966 | #undef SYSCALL |