projects / deliverable / linux.git / blob
? search:


d274612e05cdbde1e2190d43646878551042d737
[deliverable/linux.git] / arch / i386 / kernel / entry.S
1 /*
2 * linux/arch/i386/entry.S
3 *
4 * Copyright (C) 1991, 1992 Linus Torvalds
5 */
6
7 /*
8 * entry.S contains the system-call and fault low-level handling routines.
9 * This also contains the timer-interrupt handler, as well as all interrupts
10 * and faults that can result in a task-switch.
11 *
12 * NOTE: This code handles signal-recognition, which happens every time
13 * after a timer-interrupt and after each system call.
14 *
15 * I changed all the .align's to 4 (16 byte alignment), as that's faster
16 * on a 486.
17 *
18 * Stack layout in 'ret_from_system_call':
19 * ptrace needs to have all regs on the stack.
20 * if the order here is changed, it needs to be
21 * updated in fork.c:copy_process, signal.c:do_signal,
22 * ptrace.c and ptrace.h
23 *
24 * 0(%esp) - %ebx
25 * 4(%esp) - %ecx
26 * 8(%esp) - %edx
27 * C(%esp) - %esi
28 * 10(%esp) - %edi
29 * 14(%esp) - %ebp
30 * 18(%esp) - %eax
31 * 1C(%esp) - %ds
32 * 20(%esp) - %es
33 * 24(%esp) - %gs
34 * 28(%esp) - orig_eax
35 * 2C(%esp) - %eip
36 * 30(%esp) - %cs
37 * 34(%esp) - %eflags
38 * 38(%esp) - %oldesp
39 * 3C(%esp) - %oldss
40 *
41 * "current" is in register %ebx during any slow entries.
42 */
43
44 #include <linux/linkage.h>
45 #include <asm/thread_info.h>
46 #include <asm/irqflags.h>
47 #include <asm/errno.h>
48 #include <asm/segment.h>
49 #include <asm/smp.h>
50 #include <asm/page.h>
51 #include <asm/desc.h>
52 #include <asm/percpu.h>
53 #include <asm/dwarf2.h>
54 #include "irq_vectors.h"
55
56 #define nr_syscalls ((syscall_table_size)/4)
57
58 CF_MASK = 0x00000001
59 TF_MASK = 0x00000100
60 IF_MASK = 0x00000200
61 DF_MASK = 0x00000400
62 NT_MASK = 0x00004000
63 VM_MASK = 0x00020000
64
65 #ifdef CONFIG_PREEMPT
66 #define preempt_stop DISABLE_INTERRUPTS; TRACE_IRQS_OFF
67 #else
68 #define preempt_stop
69 #define resume_kernel restore_nocheck
70 #endif
71
72 .macro TRACE_IRQS_IRET
73 #ifdef CONFIG_TRACE_IRQFLAGS
74 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off?
75 jz 1f
76 TRACE_IRQS_ON
77 1:
78 #endif
79 .endm
80
81 #ifdef CONFIG_VM86
82 #define resume_userspace_sig check_userspace
83 #else
84 #define resume_userspace_sig resume_userspace
85 #endif
86
87 #define SAVE_ALL \
88 cld; \
89 pushl %gs; \
90 CFI_ADJUST_CFA_OFFSET 4;\
91 /*CFI_REL_OFFSET gs, 0;*/\
92 pushl %es; \
93 CFI_ADJUST_CFA_OFFSET 4;\
94 /*CFI_REL_OFFSET es, 0;*/\
95 pushl %ds; \
96 CFI_ADJUST_CFA_OFFSET 4;\
97 /*CFI_REL_OFFSET ds, 0;*/\
98 pushl %eax; \
99 CFI_ADJUST_CFA_OFFSET 4;\
100 CFI_REL_OFFSET eax, 0;\
101 pushl %ebp; \
102 CFI_ADJUST_CFA_OFFSET 4;\
103 CFI_REL_OFFSET ebp, 0;\
104 pushl %edi; \
105 CFI_ADJUST_CFA_OFFSET 4;\
106 CFI_REL_OFFSET edi, 0;\
107 pushl %esi; \
108 CFI_ADJUST_CFA_OFFSET 4;\
109 CFI_REL_OFFSET esi, 0;\
110 pushl %edx; \
111 CFI_ADJUST_CFA_OFFSET 4;\
112 CFI_REL_OFFSET edx, 0;\
113 pushl %ecx; \
114 CFI_ADJUST_CFA_OFFSET 4;\
115 CFI_REL_OFFSET ecx, 0;\
116 pushl %ebx; \
117 CFI_ADJUST_CFA_OFFSET 4;\
118 CFI_REL_OFFSET ebx, 0;\
119 movl $(__USER_DS), %edx; \
120 movl %edx, %ds; \
121 movl %edx, %es; \
122 movl $(__KERNEL_PDA), %edx; \
123 movl %edx, %gs
124
125 #define RESTORE_INT_REGS \
126 popl %ebx; \
127 CFI_ADJUST_CFA_OFFSET -4;\
128 CFI_RESTORE ebx;\
129 popl %ecx; \
130 CFI_ADJUST_CFA_OFFSET -4;\
131 CFI_RESTORE ecx;\
132 popl %edx; \
133 CFI_ADJUST_CFA_OFFSET -4;\
134 CFI_RESTORE edx;\
135 popl %esi; \
136 CFI_ADJUST_CFA_OFFSET -4;\
137 CFI_RESTORE esi;\
138 popl %edi; \
139 CFI_ADJUST_CFA_OFFSET -4;\
140 CFI_RESTORE edi;\
141 popl %ebp; \
142 CFI_ADJUST_CFA_OFFSET -4;\
143 CFI_RESTORE ebp;\
144 popl %eax; \
145 CFI_ADJUST_CFA_OFFSET -4;\
146 CFI_RESTORE eax
147
148 #define RESTORE_REGS \
149 RESTORE_INT_REGS; \
150 1: popl %ds; \
151 CFI_ADJUST_CFA_OFFSET -4;\
152 /*CFI_RESTORE ds;*/\
153 2: popl %es; \
154 CFI_ADJUST_CFA_OFFSET -4;\
155 /*CFI_RESTORE es;*/\
156 3: popl %gs; \
157 CFI_ADJUST_CFA_OFFSET -4;\
158 /*CFI_RESTORE gs;*/\
159 .pushsection .fixup,"ax"; \
160 4: movl $0,(%esp); \
161 jmp 1b; \
162 5: movl $0,(%esp); \
163 jmp 2b; \
164 6: movl $0,(%esp); \
165 jmp 3b; \
166 .section __ex_table,"a";\
167 .align 4; \
168 .long 1b,4b; \
169 .long 2b,5b; \
170 .long 3b,6b; \
171 .popsection
172
173 #define RING0_INT_FRAME \
174 CFI_STARTPROC simple;\
175 CFI_SIGNAL_FRAME;\
176 CFI_DEF_CFA esp, 3*4;\
177 /*CFI_OFFSET cs, -2*4;*/\
178 CFI_OFFSET eip, -3*4
179
180 #define RING0_EC_FRAME \
181 CFI_STARTPROC simple;\
182 CFI_SIGNAL_FRAME;\
183 CFI_DEF_CFA esp, 4*4;\
184 /*CFI_OFFSET cs, -2*4;*/\
185 CFI_OFFSET eip, -3*4
186
187 #define RING0_PTREGS_FRAME \
188 CFI_STARTPROC simple;\
189 CFI_SIGNAL_FRAME;\
190 CFI_DEF_CFA esp, PT_OLDESP-PT_EBX;\
191 /*CFI_OFFSET cs, PT_CS-PT_OLDESP;*/\
192 CFI_OFFSET eip, PT_EIP-PT_OLDESP;\
193 /*CFI_OFFSET es, PT_ES-PT_OLDESP;*/\
194 /*CFI_OFFSET ds, PT_DS-PT_OLDESP;*/\
195 CFI_OFFSET eax, PT_EAX-PT_OLDESP;\
196 CFI_OFFSET ebp, PT_EBP-PT_OLDESP;\
197 CFI_OFFSET edi, PT_EDI-PT_OLDESP;\
198 CFI_OFFSET esi, PT_ESI-PT_OLDESP;\
199 CFI_OFFSET edx, PT_EDX-PT_OLDESP;\
200 CFI_OFFSET ecx, PT_ECX-PT_OLDESP;\
201 CFI_OFFSET ebx, PT_EBX-PT_OLDESP
202
203 ENTRY(ret_from_fork)
204 CFI_STARTPROC
205 pushl %eax
206 CFI_ADJUST_CFA_OFFSET 4
207 call schedule_tail
208 GET_THREAD_INFO(%ebp)
209 popl %eax
210 CFI_ADJUST_CFA_OFFSET -4
211 pushl $0x0202 # Reset kernel eflags
212 CFI_ADJUST_CFA_OFFSET 4
213 popfl
214 CFI_ADJUST_CFA_OFFSET -4
215 jmp syscall_exit
216 CFI_ENDPROC
217
218 /*
219 * Return to user mode is not as complex as all this looks,
220 * but we want the default path for a system call return to
221 * go as quickly as possible which is why some of this is
222 * less clear than it otherwise should be.
223 */
224
225 # userspace resumption stub bypassing syscall exit tracing
226 ALIGN
227 RING0_PTREGS_FRAME
228 ret_from_exception:
229 preempt_stop
230 ret_from_intr:
231 GET_THREAD_INFO(%ebp)
232 check_userspace:
233 movl PT_EFLAGS(%esp), %eax # mix EFLAGS and CS
234 movb PT_CS(%esp), %al
235 andl $(VM_MASK | SEGMENT_RPL_MASK), %eax
236 cmpl $USER_RPL, %eax
237 jb resume_kernel # not returning to v8086 or userspace
238
239 ENTRY(resume_userspace)
240 DISABLE_INTERRUPTS # make sure we don't miss an interrupt
241 # setting need_resched or sigpending
242 # between sampling and the iret
243 movl TI_flags(%ebp), %ecx
244 andl $_TIF_WORK_MASK, %ecx # is there any work to be done on
245 # int/exception return?
246 jne work_pending
247 jmp restore_all
248
249 #ifdef CONFIG_PREEMPT
250 ENTRY(resume_kernel)
251 DISABLE_INTERRUPTS
252 cmpl $0,TI_preempt_count(%ebp) # non-zero preempt_count ?
253 jnz restore_nocheck
254 need_resched:
255 movl TI_flags(%ebp), %ecx # need_resched set ?
256 testb $_TIF_NEED_RESCHED, %cl
257 jz restore_all
258 testl $IF_MASK,PT_EFLAGS(%esp) # interrupts off (exception path) ?
259 jz restore_all
260 call preempt_schedule_irq
261 jmp need_resched
262 #endif
263 CFI_ENDPROC
264
265 /* SYSENTER_RETURN points to after the "sysenter" instruction in
266 the vsyscall page. See vsyscall-sysentry.S, which defines the symbol. */
267
268 # sysenter call handler stub
269 ENTRY(sysenter_entry)
270 CFI_STARTPROC simple
271 CFI_SIGNAL_FRAME
272 CFI_DEF_CFA esp, 0
273 CFI_REGISTER esp, ebp
274 movl TSS_sysenter_esp0(%esp),%esp
275 sysenter_past_esp:
276 /*
277 * No need to follow this irqs on/off section: the syscall
278 * disabled irqs and here we enable it straight after entry:
279 */
280 ENABLE_INTERRUPTS
281 pushl $(__USER_DS)
282 CFI_ADJUST_CFA_OFFSET 4
283 /*CFI_REL_OFFSET ss, 0*/
284 pushl %ebp
285 CFI_ADJUST_CFA_OFFSET 4
286 CFI_REL_OFFSET esp, 0
287 pushfl
288 CFI_ADJUST_CFA_OFFSET 4
289 pushl $(__USER_CS)
290 CFI_ADJUST_CFA_OFFSET 4
291 /*CFI_REL_OFFSET cs, 0*/
292 /*
293 * Push current_thread_info()->sysenter_return to the stack.
294 * A tiny bit of offset fixup is necessary - 4*4 means the 4 words
295 * pushed above; +8 corresponds to copy_thread's esp0 setting.
296 */
297 pushl (TI_sysenter_return-THREAD_SIZE+8+4*4)(%esp)
298 CFI_ADJUST_CFA_OFFSET 4
299 CFI_REL_OFFSET eip, 0
300
301 /*
302 * Load the potential sixth argument from user stack.
303 * Careful about security.
304 */
305 cmpl $__PAGE_OFFSET-3,%ebp
306 jae syscall_fault
307 1: movl (%ebp),%ebp
308 .section __ex_table,"a"
309 .align 4
310 .long 1b,syscall_fault
311 .previous
312
313 pushl %eax
314 CFI_ADJUST_CFA_OFFSET 4
315 SAVE_ALL
316 GET_THREAD_INFO(%ebp)
317
318 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
319 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
320 jnz syscall_trace_entry
321 cmpl $(nr_syscalls), %eax
322 jae syscall_badsys
323 call *sys_call_table(,%eax,4)
324 movl %eax,PT_EAX(%esp)
325 DISABLE_INTERRUPTS
326 TRACE_IRQS_OFF
327 movl TI_flags(%ebp), %ecx
328 testw $_TIF_ALLWORK_MASK, %cx
329 jne syscall_exit_work
330 /* if something modifies registers it must also disable sysexit */
331 movl PT_EIP(%esp), %edx
332 movl PT_OLDESP(%esp), %ecx
333 xorl %ebp,%ebp
334 TRACE_IRQS_ON
335 1: mov PT_GS(%esp), %gs
336 ENABLE_INTERRUPTS_SYSEXIT
337 CFI_ENDPROC
338 .pushsection .fixup,"ax"
339 2: movl $0,PT_GS(%esp)
340 jmp 1b
341 .section __ex_table,"a"
342 .align 4
343 .long 1b,2b
344 .popsection
345
346 # system call handler stub
347 ENTRY(system_call)
348 RING0_INT_FRAME # can't unwind into user space anyway
349 pushl %eax # save orig_eax
350 CFI_ADJUST_CFA_OFFSET 4
351 SAVE_ALL
352 GET_THREAD_INFO(%ebp)
353 testl $TF_MASK,PT_EFLAGS(%esp)
354 jz no_singlestep
355 orl $_TIF_SINGLESTEP,TI_flags(%ebp)
356 no_singlestep:
357 # system call tracing in operation / emulation
358 /* Note, _TIF_SECCOMP is bit number 8, and so it needs testw and not testb */
359 testw $(_TIF_SYSCALL_EMU|_TIF_SYSCALL_TRACE|_TIF_SECCOMP|_TIF_SYSCALL_AUDIT),TI_flags(%ebp)
360 jnz syscall_trace_entry
361 cmpl $(nr_syscalls), %eax
362 jae syscall_badsys
363 syscall_call:
364 call *sys_call_table(,%eax,4)
365 movl %eax,PT_EAX(%esp) # store the return value
366 syscall_exit:
367 DISABLE_INTERRUPTS # make sure we don't miss an interrupt
368 # setting need_resched or sigpending
369 # between sampling and the iret
370 TRACE_IRQS_OFF
371 movl TI_flags(%ebp), %ecx
372 testw $_TIF_ALLWORK_MASK, %cx # current->work
373 jne syscall_exit_work
374
375 restore_all:
376 movl PT_EFLAGS(%esp), %eax # mix EFLAGS, SS and CS
377 # Warning: PT_OLDSS(%esp) contains the wrong/random values if we
378 # are returning to the kernel.
379 # See comments in process.c:copy_thread() for details.
380 movb PT_OLDSS(%esp), %ah
381 movb PT_CS(%esp), %al
382 andl $(VM_MASK | (SEGMENT_TI_MASK << 8) | SEGMENT_RPL_MASK), %eax
383 cmpl $((SEGMENT_LDT << 8) | USER_RPL), %eax
384 CFI_REMEMBER_STATE
385 je ldt_ss # returning to user-space with LDT SS
386 restore_nocheck:
387 TRACE_IRQS_IRET
388 restore_nocheck_notrace:
389 RESTORE_REGS
390 addl $4, %esp # skip orig_eax/error_code
391 CFI_ADJUST_CFA_OFFSET -4
392 1: INTERRUPT_RETURN
393 .section .fixup,"ax"
394 iret_exc:
395 TRACE_IRQS_ON
396 ENABLE_INTERRUPTS
397 pushl $0 # no error code
398 pushl $do_iret_error
399 jmp error_code
400 .previous
401 .section __ex_table,"a"
402 .align 4
403 .long 1b,iret_exc
404 .previous
405
406 CFI_RESTORE_STATE
407 ldt_ss:
408 larl PT_OLDSS(%esp), %eax
409 jnz restore_nocheck
410 testl $0x00400000, %eax # returning to 32bit stack?
411 jnz restore_nocheck # allright, normal return
412
413 #ifdef CONFIG_PARAVIRT
414 /*
415 * The kernel can't run on a non-flat stack if paravirt mode
416 * is active. Rather than try to fixup the high bits of
417 * ESP, bypass this code entirely. This may break DOSemu
418 * and/or Wine support in a paravirt VM, although the option
419 * is still available to implement the setting of the high
420 * 16-bits in the INTERRUPT_RETURN paravirt-op.
421 */
422 cmpl $0, paravirt_ops+PARAVIRT_enabled
423 jne restore_nocheck
424 #endif
425
426 /* If returning to userspace with 16bit stack,
427 * try to fix the higher word of ESP, as the CPU
428 * won't restore it.
429 * This is an "official" bug of all the x86-compatible
430 * CPUs, which we can try to work around to make
431 * dosemu and wine happy. */
432 movl PT_OLDESP(%esp), %eax
433 movl %esp, %edx
434 call patch_espfix_desc
435 pushl $__ESPFIX_SS
436 CFI_ADJUST_CFA_OFFSET 4
437 pushl %eax
438 CFI_ADJUST_CFA_OFFSET 4
439 DISABLE_INTERRUPTS
440 TRACE_IRQS_OFF
441 lss (%esp), %esp
442 CFI_ADJUST_CFA_OFFSET -8
443 jmp restore_nocheck
444 CFI_ENDPROC
445
446 # perform work that needs to be done immediately before resumption
447 ALIGN
448 RING0_PTREGS_FRAME # can't unwind into user space anyway
449 work_pending:
450 testb $_TIF_NEED_RESCHED, %cl
451 jz work_notifysig
452 work_resched:
453 call schedule
454 DISABLE_INTERRUPTS # make sure we don't miss an interrupt
455 # setting need_resched or sigpending
456 # between sampling and the iret
457 TRACE_IRQS_OFF
458 movl TI_flags(%ebp), %ecx
459 andl $_TIF_WORK_MASK, %ecx # is there any work to be done other
460 # than syscall tracing?
461 jz restore_all
462 testb $_TIF_NEED_RESCHED, %cl
463 jnz work_resched
464
465 work_notifysig: # deal with pending signals and
466 # notify-resume requests
467 #ifdef CONFIG_VM86
468 testl $VM_MASK, PT_EFLAGS(%esp)
469 movl %esp, %eax
470 jne work_notifysig_v86 # returning to kernel-space or
471 # vm86-space
472 xorl %edx, %edx
473 call do_notify_resume
474 jmp resume_userspace_sig
475
476 ALIGN
477 work_notifysig_v86:
478 pushl %ecx # save ti_flags for do_notify_resume
479 CFI_ADJUST_CFA_OFFSET 4
480 call save_v86_state # %eax contains pt_regs pointer
481 popl %ecx
482 CFI_ADJUST_CFA_OFFSET -4
483 movl %eax, %esp
484 #else
485 movl %esp, %eax
486 #endif
487 xorl %edx, %edx
488 call do_notify_resume
489 jmp resume_userspace_sig
490
491 # perform syscall exit tracing
492 ALIGN
493 syscall_trace_entry:
494 movl $-ENOSYS,PT_EAX(%esp)
495 movl %esp, %eax
496 xorl %edx,%edx
497 call do_syscall_trace
498 cmpl $0, %eax
499 jne resume_userspace # ret != 0 -> running under PTRACE_SYSEMU,
500 # so must skip actual syscall
501 movl PT_ORIG_EAX(%esp), %eax
502 cmpl $(nr_syscalls), %eax
503 jnae syscall_call
504 jmp syscall_exit
505
506 # perform syscall exit tracing
507 ALIGN
508 syscall_exit_work:
509 testb $(_TIF_SYSCALL_TRACE|_TIF_SYSCALL_AUDIT|_TIF_SINGLESTEP), %cl
510 jz work_pending
511 TRACE_IRQS_ON
512 ENABLE_INTERRUPTS # could let do_syscall_trace() call
513 # schedule() instead
514 movl %esp, %eax
515 movl $1, %edx
516 call do_syscall_trace
517 jmp resume_userspace
518 CFI_ENDPROC
519
520 RING0_INT_FRAME # can't unwind into user space anyway
521 syscall_fault:
522 pushl %eax # save orig_eax
523 CFI_ADJUST_CFA_OFFSET 4
524 SAVE_ALL
525 GET_THREAD_INFO(%ebp)
526 movl $-EFAULT,PT_EAX(%esp)
527 jmp resume_userspace
528
529 syscall_badsys:
530 movl $-ENOSYS,PT_EAX(%esp)
531 jmp resume_userspace
532 CFI_ENDPROC
533
534 #define FIXUP_ESPFIX_STACK \
535 /* since we are on a wrong stack, we cant make it a C code :( */ \
536 movl %gs:PDA_cpu, %ebx; \
537 PER_CPU(cpu_gdt_descr, %ebx); \
538 movl GDS_address(%ebx), %ebx; \
539 GET_DESC_BASE(GDT_ENTRY_ESPFIX_SS, %ebx, %eax, %ax, %al, %ah); \
540 addl %esp, %eax; \
541 pushl $__KERNEL_DS; \
542 CFI_ADJUST_CFA_OFFSET 4; \
543 pushl %eax; \
544 CFI_ADJUST_CFA_OFFSET 4; \
545 lss (%esp), %esp; \
546 CFI_ADJUST_CFA_OFFSET -8;
547 #define UNWIND_ESPFIX_STACK \
548 movl %ss, %eax; \
549 /* see if on espfix stack */ \
550 cmpw $__ESPFIX_SS, %ax; \
551 jne 27f; \
552 movl $__KERNEL_DS, %eax; \
553 movl %eax, %ds; \
554 movl %eax, %es; \
555 /* switch to normal stack */ \
556 FIXUP_ESPFIX_STACK; \
557 27:;
558
559 /*
560 * Build the entry stubs and pointer table with
561 * some assembler magic.
562 */
563 .data
564 ENTRY(interrupt)
565 .text
566
567 vector=0
568 ENTRY(irq_entries_start)
569 RING0_INT_FRAME
570 .rept NR_IRQS
571 ALIGN
572 .if vector
573 CFI_ADJUST_CFA_OFFSET -4
574 .endif
575 1: pushl $~(vector)
576 CFI_ADJUST_CFA_OFFSET 4
577 jmp common_interrupt
578 .data
579 .long 1b
580 .text
581 vector=vector+1
582 .endr
583
584 /*
585 * the CPU automatically disables interrupts when executing an IRQ vector,
586 * so IRQ-flags tracing has to follow that:
587 */
588 ALIGN
589 common_interrupt:
590 SAVE_ALL
591 TRACE_IRQS_OFF
592 movl %esp,%eax
593 call do_IRQ
594 jmp ret_from_intr
595 CFI_ENDPROC
596
597 #define BUILD_INTERRUPT(name, nr) \
598 ENTRY(name) \
599 RING0_INT_FRAME; \
600 pushl $~(nr); \
601 CFI_ADJUST_CFA_OFFSET 4; \
602 SAVE_ALL; \
603 TRACE_IRQS_OFF \
604 movl %esp,%eax; \
605 call smp_/**/name; \
606 jmp ret_from_intr; \
607 CFI_ENDPROC
608
609 /* The include is where all of the SMP etc. interrupts come from */
610 #include "entry_arch.h"
611
612 KPROBE_ENTRY(page_fault)
613 RING0_EC_FRAME
614 pushl $do_page_fault
615 CFI_ADJUST_CFA_OFFSET 4
616 ALIGN
617 error_code:
618 /* the function address is in %gs's slot on the stack */
619 pushl %es
620 CFI_ADJUST_CFA_OFFSET 4
621 /*CFI_REL_OFFSET es, 0*/
622 pushl %ds
623 CFI_ADJUST_CFA_OFFSET 4
624 /*CFI_REL_OFFSET ds, 0*/
625 pushl %eax
626 CFI_ADJUST_CFA_OFFSET 4
627 CFI_REL_OFFSET eax, 0
628 pushl %ebp
629 CFI_ADJUST_CFA_OFFSET 4
630 CFI_REL_OFFSET ebp, 0
631 pushl %edi
632 CFI_ADJUST_CFA_OFFSET 4
633 CFI_REL_OFFSET edi, 0
634 pushl %esi
635 CFI_ADJUST_CFA_OFFSET 4
636 CFI_REL_OFFSET esi, 0
637 pushl %edx
638 CFI_ADJUST_CFA_OFFSET 4
639 CFI_REL_OFFSET edx, 0
640 pushl %ecx
641 CFI_ADJUST_CFA_OFFSET 4
642 CFI_REL_OFFSET ecx, 0
643 pushl %ebx
644 CFI_ADJUST_CFA_OFFSET 4
645 CFI_REL_OFFSET ebx, 0
646 cld
647 pushl %gs
648 CFI_ADJUST_CFA_OFFSET 4
649 /*CFI_REL_OFFSET gs, 0*/
650 movl $(__KERNEL_PDA), %ecx
651 movl %ecx, %gs
652 UNWIND_ESPFIX_STACK
653 popl %ecx
654 CFI_ADJUST_CFA_OFFSET -4
655 /*CFI_REGISTER es, ecx*/
656 movl PT_GS(%esp), %edi # get the function address
657 movl PT_ORIG_EAX(%esp), %edx # get the error code
658 movl $-1, PT_ORIG_EAX(%esp) # no syscall to restart
659 mov %ecx, PT_GS(%esp)
660 /*CFI_REL_OFFSET gs, ES*/
661 movl $(__USER_DS), %ecx
662 movl %ecx, %ds
663 movl %ecx, %es
664 movl %esp,%eax # pt_regs pointer
665 call *%edi
666 jmp ret_from_exception
667 CFI_ENDPROC
668 KPROBE_END(page_fault)
669
670 ENTRY(coprocessor_error)
671 RING0_INT_FRAME
672 pushl $0
673 CFI_ADJUST_CFA_OFFSET 4
674 pushl $do_coprocessor_error
675 CFI_ADJUST_CFA_OFFSET 4
676 jmp error_code
677 CFI_ENDPROC
678
679 ENTRY(simd_coprocessor_error)
680 RING0_INT_FRAME
681 pushl $0
682 CFI_ADJUST_CFA_OFFSET 4
683 pushl $do_simd_coprocessor_error
684 CFI_ADJUST_CFA_OFFSET 4
685 jmp error_code
686 CFI_ENDPROC
687
688 ENTRY(device_not_available)
689 RING0_INT_FRAME
690 pushl $-1 # mark this as an int
691 CFI_ADJUST_CFA_OFFSET 4
692 SAVE_ALL
693 GET_CR0_INTO_EAX
694 testl $0x4, %eax # EM (math emulation bit)
695 jne device_not_available_emulate
696 preempt_stop
697 call math_state_restore
698 jmp ret_from_exception
699 device_not_available_emulate:
700 pushl $0 # temporary storage for ORIG_EIP
701 CFI_ADJUST_CFA_OFFSET 4
702 call math_emulate
703 addl $4, %esp
704 CFI_ADJUST_CFA_OFFSET -4
705 jmp ret_from_exception
706 CFI_ENDPROC
707
708 /*
709 * Debug traps and NMI can happen at the one SYSENTER instruction
710 * that sets up the real kernel stack. Check here, since we can't
711 * allow the wrong stack to be used.
712 *
713 * "TSS_sysenter_esp0+12" is because the NMI/debug handler will have
714 * already pushed 3 words if it hits on the sysenter instruction:
715 * eflags, cs and eip.
716 *
717 * We just load the right stack, and push the three (known) values
718 * by hand onto the new stack - while updating the return eip past
719 * the instruction that would have done it for sysenter.
720 */
721 #define FIX_STACK(offset, ok, label) \
722 cmpw $__KERNEL_CS,4(%esp); \
723 jne ok; \
724 label: \
725 movl TSS_sysenter_esp0+offset(%esp),%esp; \
726 CFI_DEF_CFA esp, 0; \
727 CFI_UNDEFINED eip; \
728 pushfl; \
729 CFI_ADJUST_CFA_OFFSET 4; \
730 pushl $__KERNEL_CS; \
731 CFI_ADJUST_CFA_OFFSET 4; \
732 pushl $sysenter_past_esp; \
733 CFI_ADJUST_CFA_OFFSET 4; \
734 CFI_REL_OFFSET eip, 0
735
736 KPROBE_ENTRY(debug)
737 RING0_INT_FRAME
738 cmpl $sysenter_entry,(%esp)
739 jne debug_stack_correct
740 FIX_STACK(12, debug_stack_correct, debug_esp_fix_insn)
741 debug_stack_correct:
742 pushl $-1 # mark this as an int
743 CFI_ADJUST_CFA_OFFSET 4
744 SAVE_ALL
745 xorl %edx,%edx # error code 0
746 movl %esp,%eax # pt_regs pointer
747 call do_debug
748 jmp ret_from_exception
749 CFI_ENDPROC
750 KPROBE_END(debug)
751
752 /*
753 * NMI is doubly nasty. It can happen _while_ we're handling
754 * a debug fault, and the debug fault hasn't yet been able to
755 * clear up the stack. So we first check whether we got an
756 * NMI on the sysenter entry path, but after that we need to
757 * check whether we got an NMI on the debug path where the debug
758 * fault happened on the sysenter path.
759 */
760 KPROBE_ENTRY(nmi)
761 RING0_INT_FRAME
762 pushl %eax
763 CFI_ADJUST_CFA_OFFSET 4
764 movl %ss, %eax
765 cmpw $__ESPFIX_SS, %ax
766 popl %eax
767 CFI_ADJUST_CFA_OFFSET -4
768 je nmi_espfix_stack
769 cmpl $sysenter_entry,(%esp)
770 je nmi_stack_fixup
771 pushl %eax
772 CFI_ADJUST_CFA_OFFSET 4
773 movl %esp,%eax
774 /* Do not access memory above the end of our stack page,
775 * it might not exist.
776 */
777 andl $(THREAD_SIZE-1),%eax
778 cmpl $(THREAD_SIZE-20),%eax
779 popl %eax
780 CFI_ADJUST_CFA_OFFSET -4
781 jae nmi_stack_correct
782 cmpl $sysenter_entry,12(%esp)
783 je nmi_debug_stack_check
784 nmi_stack_correct:
785 /* We have a RING0_INT_FRAME here */
786 pushl %eax
787 CFI_ADJUST_CFA_OFFSET 4
788 SAVE_ALL
789 xorl %edx,%edx # zero error code
790 movl %esp,%eax # pt_regs pointer
791 call do_nmi
792 jmp restore_nocheck_notrace
793 CFI_ENDPROC
794
795 nmi_stack_fixup:
796 RING0_INT_FRAME
797 FIX_STACK(12,nmi_stack_correct, 1)
798 jmp nmi_stack_correct
799
800 nmi_debug_stack_check:
801 /* We have a RING0_INT_FRAME here */
802 cmpw $__KERNEL_CS,16(%esp)
803 jne nmi_stack_correct
804 cmpl $debug,(%esp)
805 jb nmi_stack_correct
806 cmpl $debug_esp_fix_insn,(%esp)
807 ja nmi_stack_correct
808 FIX_STACK(24,nmi_stack_correct, 1)
809 jmp nmi_stack_correct
810
811 nmi_espfix_stack:
812 /* We have a RING0_INT_FRAME here.
813 *
814 * create the pointer to lss back
815 */
816 pushl %ss
817 CFI_ADJUST_CFA_OFFSET 4
818 pushl %esp
819 CFI_ADJUST_CFA_OFFSET 4
820 addw $4, (%esp)
821 /* copy the iret frame of 12 bytes */
822 .rept 3
823 pushl 16(%esp)
824 CFI_ADJUST_CFA_OFFSET 4
825 .endr
826 pushl %eax
827 CFI_ADJUST_CFA_OFFSET 4
828 SAVE_ALL
829 FIXUP_ESPFIX_STACK # %eax == %esp
830 xorl %edx,%edx # zero error code
831 call do_nmi
832 RESTORE_REGS
833 lss 12+4(%esp), %esp # back to espfix stack
834 CFI_ADJUST_CFA_OFFSET -24
835 1: INTERRUPT_RETURN
836 CFI_ENDPROC
837 .section __ex_table,"a"
838 .align 4
839 .long 1b,iret_exc
840 .previous
841 KPROBE_END(nmi)
842
843 #ifdef CONFIG_PARAVIRT
844 ENTRY(native_iret)
845 1: iret
846 .section __ex_table,"a"
847 .align 4
848 .long 1b,iret_exc
849 .previous
850
851 ENTRY(native_irq_enable_sysexit)
852 sti
853 sysexit
854 #endif
855
856 KPROBE_ENTRY(int3)
857 RING0_INT_FRAME
858 pushl $-1 # mark this as an int
859 CFI_ADJUST_CFA_OFFSET 4
860 SAVE_ALL
861 xorl %edx,%edx # zero error code
862 movl %esp,%eax # pt_regs pointer
863 call do_int3
864 jmp ret_from_exception
865 CFI_ENDPROC
866 KPROBE_END(int3)
867
868 ENTRY(overflow)
869 RING0_INT_FRAME
870 pushl $0
871 CFI_ADJUST_CFA_OFFSET 4
872 pushl $do_overflow
873 CFI_ADJUST_CFA_OFFSET 4
874 jmp error_code
875 CFI_ENDPROC
876
877 ENTRY(bounds)
878 RING0_INT_FRAME
879 pushl $0
880 CFI_ADJUST_CFA_OFFSET 4
881 pushl $do_bounds
882 CFI_ADJUST_CFA_OFFSET 4
883 jmp error_code
884 CFI_ENDPROC
885
886 ENTRY(invalid_op)
887 RING0_INT_FRAME
888 pushl $0
889 CFI_ADJUST_CFA_OFFSET 4
890 pushl $do_invalid_op
891 CFI_ADJUST_CFA_OFFSET 4
892 jmp error_code
893 CFI_ENDPROC
894
895 ENTRY(coprocessor_segment_overrun)
896 RING0_INT_FRAME
897 pushl $0
898 CFI_ADJUST_CFA_OFFSET 4
899 pushl $do_coprocessor_segment_overrun
900 CFI_ADJUST_CFA_OFFSET 4
901 jmp error_code
902 CFI_ENDPROC
903
904 ENTRY(invalid_TSS)
905 RING0_EC_FRAME
906 pushl $do_invalid_TSS
907 CFI_ADJUST_CFA_OFFSET 4
908 jmp error_code
909 CFI_ENDPROC
910
911 ENTRY(segment_not_present)
912 RING0_EC_FRAME
913 pushl $do_segment_not_present
914 CFI_ADJUST_CFA_OFFSET 4
915 jmp error_code
916 CFI_ENDPROC
917
918 ENTRY(stack_segment)
919 RING0_EC_FRAME
920 pushl $do_stack_segment
921 CFI_ADJUST_CFA_OFFSET 4
922 jmp error_code
923 CFI_ENDPROC
924
925 KPROBE_ENTRY(general_protection)
926 RING0_EC_FRAME
927 pushl $do_general_protection
928 CFI_ADJUST_CFA_OFFSET 4
929 jmp error_code
930 CFI_ENDPROC
931 KPROBE_END(general_protection)
932
933 ENTRY(alignment_check)
934 RING0_EC_FRAME
935 pushl $do_alignment_check
936 CFI_ADJUST_CFA_OFFSET 4
937 jmp error_code
938 CFI_ENDPROC
939
940 ENTRY(divide_error)
941 RING0_INT_FRAME
942 pushl $0 # no error code
943 CFI_ADJUST_CFA_OFFSET 4
944 pushl $do_divide_error
945 CFI_ADJUST_CFA_OFFSET 4
946 jmp error_code
947 CFI_ENDPROC
948
949 #ifdef CONFIG_X86_MCE
950 ENTRY(machine_check)
951 RING0_INT_FRAME
952 pushl $0
953 CFI_ADJUST_CFA_OFFSET 4
954 pushl machine_check_vector
955 CFI_ADJUST_CFA_OFFSET 4
956 jmp error_code
957 CFI_ENDPROC
958 #endif
959
960 ENTRY(spurious_interrupt_bug)
961 RING0_INT_FRAME
962 pushl $0
963 CFI_ADJUST_CFA_OFFSET 4
964 pushl $do_spurious_interrupt_bug
965 CFI_ADJUST_CFA_OFFSET 4
966 jmp error_code
967 CFI_ENDPROC
968
969 #ifdef CONFIG_STACK_UNWIND
970 ENTRY(arch_unwind_init_running)
971 CFI_STARTPROC
972 movl 4(%esp), %edx
973 movl (%esp), %ecx
974 leal 4(%esp), %eax
975 movl %ebx, PT_EBX(%edx)
976 xorl %ebx, %ebx
977 movl %ebx, PT_ECX(%edx)
978 movl %ebx, PT_EDX(%edx)
979 movl %esi, PT_ESI(%edx)
980 movl %edi, PT_EDI(%edx)
981 movl %ebp, PT_EBP(%edx)
982 movl %ebx, PT_EAX(%edx)
983 movl $__USER_DS, PT_DS(%edx)
984 movl $__USER_DS, PT_ES(%edx)
985 movl $0, PT_GS(%edx)
986 movl %ebx, PT_ORIG_EAX(%edx)
987 movl %ecx, PT_EIP(%edx)
988 movl 12(%esp), %ecx
989 movl $__KERNEL_CS, PT_CS(%edx)
990 movl %ebx, PT_EFLAGS(%edx)
991 movl %eax, PT_OLDESP(%edx)
992 movl 8(%esp), %eax
993 movl %ecx, 8(%esp)
994 movl PT_EBX(%edx), %ebx
995 movl $__KERNEL_DS, PT_OLDSS(%edx)
996 jmpl *%eax
997 CFI_ENDPROC
998 ENDPROC(arch_unwind_init_running)
999 #endif
1000
1001 ENTRY(kernel_thread_helper)
1002 pushl $0 # fake return address for unwinder
1003 CFI_STARTPROC
1004 movl %edx,%eax
1005 push %edx
1006 CFI_ADJUST_CFA_OFFSET 4
1007 call *%ebx
1008 push %eax
1009 CFI_ADJUST_CFA_OFFSET 4
1010 call do_exit
1011 CFI_ENDPROC
1012 ENDPROC(kernel_thread_helper)
1013
1014 .section .rodata,"a"
1015 #include "syscall_table.S"
1016
1017 syscall_table_size=(.-sys_call_table)
Linux kernel - Deliverables
This page took 0.04936 seconds and 4 git commands to generate.