2 * Copyright (C) 1992, 1998 Linus Torvalds, Ingo Molnar
4 * This file contains the lowest level x86-specific interrupt
5 * entry, irq-stacks and irq statistics code. All the remaining
6 * irq logic is done by the generic kernel/irq/ code and
7 * by the x86-specific irq controller code. (e.g. i8259.c and
11 #include <linux/module.h>
12 #include <linux/seq_file.h>
13 #include <linux/interrupt.h>
14 #include <linux/kernel_stat.h>
15 #include <linux/notifier.h>
16 #include <linux/cpu.h>
17 #include <linux/delay.h>
18 #include <linux/uaccess.h>
19 #include <linux/percpu.h>
24 DEFINE_PER_CPU_SHARED_ALIGNED(irq_cpustat_t
, irq_stat
);
25 EXPORT_PER_CPU_SYMBOL(irq_stat
);
27 DEFINE_PER_CPU(struct pt_regs
*, irq_regs
);
28 EXPORT_PER_CPU_SYMBOL(irq_regs
);
30 #ifdef CONFIG_DEBUG_STACKOVERFLOW
32 int sysctl_panic_on_stackoverflow __read_mostly
;
34 /* Debugging check for stack overflow: is there less than 1KB free? */
35 static int check_stack_overflow(void)
39 __asm__
__volatile__("andl %%esp,%0" :
40 "=r" (sp
) : "0" (THREAD_SIZE
- 1));
42 return sp
< (sizeof(struct thread_info
) + STACK_WARN
);
45 static void print_stack_overflow(void)
47 printk(KERN_WARNING
"low stack detected by irq handler\n");
49 if (sysctl_panic_on_stackoverflow
)
50 panic("low stack detected by irq handler - check messages\n");
54 static inline int check_stack_overflow(void) { return 0; }
55 static inline void print_stack_overflow(void) { }
58 DEFINE_PER_CPU(struct irq_stack
*, hardirq_stack
);
59 DEFINE_PER_CPU(struct irq_stack
*, softirq_stack
);
61 static void call_on_stack(void *func
, void *stack
)
63 asm volatile("xchgl %%ebx,%%esp \n"
69 : "memory", "cc", "edx", "ecx", "eax");
72 /* how to get the current stack pointer from C */
73 #define current_stack_pointer ({ \
75 asm("mov %%esp,%0" : "=g" (sp)); \
79 static inline void *current_stack(void)
81 return (void *)(current_stack_pointer
& ~(THREAD_SIZE
- 1));
85 execute_on_irq_stack(int overflow
, struct irq_desc
*desc
, int irq
)
87 struct irq_stack
*curstk
, *irqstk
;
88 u32
*isp
, *prev_esp
, arg1
, arg2
;
90 curstk
= (struct irq_stack
*) current_stack();
91 irqstk
= __this_cpu_read(hardirq_stack
);
94 * this is where we switch to the IRQ stack. However, if we are
95 * already using the IRQ stack (because we interrupted a hardirq
96 * handler) we can't do that and just have to keep using the
97 * current stack (which is the irq stack already after all)
99 if (unlikely(curstk
== irqstk
))
102 isp
= (u32
*) ((char *)irqstk
+ sizeof(*irqstk
));
104 /* Save the next esp at the bottom of the stack */
105 prev_esp
= (u32
*)irqstk
;
106 *prev_esp
= current_stack_pointer
;
108 if (unlikely(overflow
))
109 call_on_stack(print_stack_overflow
, isp
);
111 asm volatile("xchgl %%ebx,%%esp \n"
113 "movl %%ebx,%%esp \n"
114 : "=a" (arg1
), "=d" (arg2
), "=b" (isp
)
115 : "0" (irq
), "1" (desc
), "2" (isp
),
116 "D" (desc
->handle_irq
)
117 : "memory", "cc", "ecx");
122 * allocate per-cpu stacks for hardirq and for softirq processing
124 void irq_ctx_init(int cpu
)
126 struct irq_stack
*irqstk
;
128 if (per_cpu(hardirq_stack
, cpu
))
131 irqstk
= page_address(alloc_pages_node(cpu_to_node(cpu
),
134 per_cpu(hardirq_stack
, cpu
) = irqstk
;
136 irqstk
= page_address(alloc_pages_node(cpu_to_node(cpu
),
139 per_cpu(softirq_stack
, cpu
) = irqstk
;
141 printk(KERN_DEBUG
"CPU %u irqstacks, hard=%p soft=%p\n",
142 cpu
, per_cpu(hardirq_stack
, cpu
), per_cpu(softirq_stack
, cpu
));
145 void do_softirq_own_stack(void)
147 struct thread_info
*curstk
;
148 struct irq_stack
*irqstk
;
151 curstk
= current_stack();
152 irqstk
= __this_cpu_read(softirq_stack
);
154 /* build the stack frame on the softirq stack */
155 isp
= (u32
*) ((char *)irqstk
+ sizeof(*irqstk
));
157 /* Push the previous esp onto the stack */
158 prev_esp
= (u32
*)irqstk
;
159 *prev_esp
= current_stack_pointer
;
161 call_on_stack(__do_softirq
, isp
);
164 bool handle_irq(unsigned irq
, struct pt_regs
*regs
)
166 struct irq_desc
*desc
;
169 overflow
= check_stack_overflow();
171 desc
= irq_to_desc(irq
);
175 if (user_mode_vm(regs
) || !execute_on_irq_stack(overflow
, desc
, irq
)) {
176 if (unlikely(overflow
))
177 print_stack_overflow();
178 desc
->handle_irq(irq
, desc
);