x86: Keep thread_info on thread stack in x86_32

x86_64 uses a per_cpu variable kernel_stack to always point to
the thread stack of current. This is where the thread_info is stored
and is accessed from this location even when the irq or exception stack
is in use. This removes the complexity of having to maintain the
thread info on the stack when interrupts are running and having to
copy the preempt_count and other fields to the interrupt stack.

x86_32 uses the old method of copying the thread_info from the thread
stack to the exception stack just before executing the exception.

Having the two different requires #ifdefs and also the x86_32 way
is a bit of a pain to maintain. By converting x86_32 to the same
method of x86_64, we can remove #ifdefs, clean up the x86_32 code
a little, and remove the overhead of the copy.

Cc: Andrew Morton <akpm@linux-foundation.org>
Cc: Peter Zijlstra <peterz@infradead.org>
Cc: Brian Gerst <brgerst@gmail.com>
Signed-off-by: Steven Rostedt <rostedt@goodmis.org>
Link: http://lkml.kernel.org/r/20110806012354.263834829@goodmis.org
Link: http://lkml.kernel.org/r/20140206144321.852942014@goodmis.org
Signed-off-by: H. Peter Anvin <hpa@linux.intel.com>

diff --git a/arch/x86/include/asm/processor.h b/arch/x86/include/asm/processor.h
index fdedd38fd0fc7feeabc26f8b366b525b65685904..a4ea02351f4d02bee5a43e78a9013b77ec94ea48 100644 (file)
--- a/arch/x86/include/asm/processor.h
+++ b/arch/x86/include/asm/processor.h
@@ -449,6 +449,15 @@ struct stack_canary {
 };
 DECLARE_PER_CPU_ALIGNED(struct stack_canary, stack_canary);
 #endif
+/*
+ * per-CPU IRQ handling stacks
+ */
+struct irq_stack {
+       u32                     stack[THREAD_SIZE/sizeof(u32)];
+} __aligned(THREAD_SIZE);
+
+DECLARE_PER_CPU(struct irq_stack *, hardirq_stack);
+DECLARE_PER_CPU(struct irq_stack *, softirq_stack);
 #endif /* X86_64 */
 
 extern unsigned int xstate_size;

diff --git a/arch/x86/include/asm/thread_info.h b/arch/x86/include/asm/thread_info.h
index ca2de1b48fac3692f8800f75ba8d21664d035892..47e5de25ba799f787d7f9344c28daa797fde47b7 100644 (file)
--- a/arch/x86/include/asm/thread_info.h
+++ b/arch/x86/include/asm/thread_info.h
@@ -9,6 +9,7 @@
 
 #include <linux/compiler.h>
 #include <asm/page.h>
+#include <asm/percpu.h>
 #include <asm/types.h>
 
 /*
@@ -34,12 +35,6 @@ struct thread_info {
        void __user             *sysenter_return;
        unsigned int            sig_on_uaccess_error:1;
        unsigned int            uaccess_err:1;  /* uaccess failed */
-#ifdef CONFIG_X86_32
-       unsigned long           previous_esp;   /* ESP of the previous stack in
-                                                  case of nested (IRQ) stacks
-                                                  (Moved to end, to be removed soon)
-                                               */
-#endif
 };
 
 #define INIT_THREAD_INFO(tsk)                  \
@@ -153,9 +148,9 @@ struct thread_info {
 #define _TIF_WORK_CTXSW_PREV (_TIF_WORK_CTXSW|_TIF_USER_RETURN_NOTIFY)
 #define _TIF_WORK_CTXSW_NEXT (_TIF_WORK_CTXSW)
 
-#ifdef CONFIG_X86_32
+#define STACK_WARN             (THREAD_SIZE/8)
+#define KERNEL_STACK_OFFSET    (5*(BITS_PER_LONG/8))
 
-#define STACK_WARN     (THREAD_SIZE/8)
 /*
  * macros/functions for gaining access to the thread information structure
  *
@@ -163,38 +158,6 @@ struct thread_info {
  */
 #ifndef __ASSEMBLY__
 
-#define current_stack_pointer ({               \
-       unsigned long sp;                       \
-       asm("mov %%esp,%0" : "=g" (sp));        \
-       sp;                                     \
-})
-
-/* how to get the thread information struct from C */
-static inline struct thread_info *current_thread_info(void)
-{
-       return (struct thread_info *)
-               (current_stack_pointer & ~(THREAD_SIZE - 1));
-}
-
-#else /* !__ASSEMBLY__ */
-
-/* how to get the thread information struct from ASM */
-#define GET_THREAD_INFO(reg)    \
-       movl $-THREAD_SIZE, reg; \
-       andl %esp, reg
-
-#endif
-
-#else /* X86_32 */
-
-#include <asm/percpu.h>
-#define KERNEL_STACK_OFFSET (5*8)
-
-/*
- * macros/functions for gaining access to the thread information structure
- * preempt_count needs to be 1 initially, until the scheduler is functional.
- */
-#ifndef __ASSEMBLY__
 DECLARE_PER_CPU(unsigned long, kernel_stack);
 
 static inline struct thread_info *current_thread_info(void)
@@ -209,8 +172,8 @@ static inline struct thread_info *current_thread_info(void)
 
 /* how to get the thread information struct from ASM */
 #define GET_THREAD_INFO(reg) \
-       movq PER_CPU_VAR(kernel_stack),reg ; \
-       subq $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg
+       _ASM_MOV PER_CPU_VAR(kernel_stack),reg ; \
+       _ASM_SUB $(THREAD_SIZE-KERNEL_STACK_OFFSET),reg ;
 
 /*
  * Same if PER_CPU_VAR(kernel_stack) is, perhaps with some offset, already in
@@ -220,8 +183,6 @@ static inline struct thread_info *current_thread_info(void)
 
 #endif
 
-#endif /* !X86_32 */
-
 /*
  * Thread-synchronous status.
  *

diff --git a/arch/x86/kernel/cpu/common.c b/arch/x86/kernel/cpu/common.c
index 8e28bf2fc3ef4a15bf672dee29bc6729552556f2..29c1944a98ac0127c230da1d911f45e07e57b41b 100644 (file)
--- a/arch/x86/kernel/cpu/common.c
+++ b/arch/x86/kernel/cpu/common.c
@@ -1078,6 +1078,10 @@ static __init int setup_disablecpuid(char *arg)
 }
 __setup("clearcpuid=", setup_disablecpuid);
 
+DEFINE_PER_CPU(unsigned long, kernel_stack) =
+       (unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
+EXPORT_PER_CPU_SYMBOL(kernel_stack);
+
 #ifdef CONFIG_X86_64
 struct desc_ptr idt_descr = { NR_VECTORS * 16 - 1, (unsigned long) idt_table };
 struct desc_ptr debug_idt_descr = { NR_VECTORS * 16 - 1,
@@ -1094,10 +1098,6 @@ DEFINE_PER_CPU(struct task_struct *, current_task) ____cacheline_aligned =
        &init_task;
 EXPORT_PER_CPU_SYMBOL(current_task);
 
-DEFINE_PER_CPU(unsigned long, kernel_stack) =
-       (unsigned long)&init_thread_union - KERNEL_STACK_OFFSET + THREAD_SIZE;
-EXPORT_PER_CPU_SYMBOL(kernel_stack);
-
 DEFINE_PER_CPU(char *, irq_stack_ptr) =
        init_per_cpu_var(irq_stack_union.irq_stack) + IRQ_STACK_SIZE - 64;
 

diff --git a/arch/x86/kernel/dumpstack_32.c b/arch/x86/kernel/dumpstack_32.c
index 187d6a749c191135339552c57d8828989c2eb3b1..dca820b627d673797f89e891bc42a72352893884 100644 (file)
--- a/arch/x86/kernel/dumpstack_32.c
+++ b/arch/x86/kernel/dumpstack_32.c
@@ -16,11 +16,33 @@
 
 #include <asm/stacktrace.h>
 
+static void *is_irq_stack(void *p, void *irq)
+{
+       if (p < irq || p >= (irq + THREAD_SIZE))
+               return NULL;
+       return irq + THREAD_SIZE;
+}
+
+
+static void *is_hardirq_stack(unsigned long *stack, int cpu)
+{
+       void *irq = per_cpu(hardirq_stack, cpu);
+
+       return is_irq_stack(stack, irq);
+}
+
+static void *is_softirq_stack(unsigned long *stack, int cpu)
+{
+       void *irq = per_cpu(softirq_stack, cpu);
+
+       return is_irq_stack(stack, irq);
+}
 
 void dump_trace(struct task_struct *task, struct pt_regs *regs,
                unsigned long *stack, unsigned long bp,
                const struct stacktrace_ops *ops, void *data)
 {
+       const unsigned cpu = get_cpu();
        int graph = 0;
        u32 *prev_esp;
 
@@ -40,18 +62,22 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
 
        for (;;) {
                struct thread_info *context;
+               void *end_stack;
+
+               end_stack = is_hardirq_stack(stack, cpu);
+               if (!end_stack)
+                       end_stack = is_softirq_stack(stack, cpu);
 
-               context = (struct thread_info *)
-                       ((unsigned long)stack & (~(THREAD_SIZE - 1)));
-               bp = ops->walk_stack(context, stack, bp, ops, data, NULL, &graph);
+               context = task_thread_info(task);
+               bp = ops->walk_stack(context, stack, bp, ops, data,
+                                    end_stack, &graph);
 
                /* Stop if not on irq stack */
-               if (task_stack_page(task) == context)
+               if (!end_stack)
                        break;
 
-               /* The previous esp is just above the context */
-               prev_esp = (u32 *) ((char *)context + sizeof(struct thread_info) -
-                                   sizeof(long));
+               /* The previous esp is saved on the bottom of the stack */
+               prev_esp = (u32 *)(end_stack - THREAD_SIZE);
                stack = (unsigned long *)*prev_esp;
                if (!stack)
                        break;
@@ -60,6 +86,7 @@ void dump_trace(struct task_struct *task, struct pt_regs *regs,
                        break;
                touch_nmi_watchdog();
        }
+       put_cpu();
 }
 EXPORT_SYMBOL(dump_trace);
 

diff --git a/arch/x86/kernel/irq_32.c b/arch/x86/kernel/irq_32.c
index f135cc2ff3014ea2b4c34a5bfab8c78e26c1fa05..988dc8bcaebf92d73649752edc972b22b902dbdc 100644 (file)
--- a/arch/x86/kernel/irq_32.c
+++ b/arch/x86/kernel/irq_32.c
@@ -55,16 +55,8 @@ static inline int check_stack_overflow(void) { return 0; }
 static inline void print_stack_overflow(void) { }
 #endif
 
-/*
- * per-CPU IRQ handling contexts (thread information and stack)
- */
-union irq_ctx {
-       struct thread_info      tinfo;
-       u32                     stack[THREAD_SIZE/sizeof(u32)];
-} __attribute__((aligned(THREAD_SIZE)));
-
-static DEFINE_PER_CPU(union irq_ctx *, hardirq_ctx);
-static DEFINE_PER_CPU(union irq_ctx *, softirq_ctx);
+DEFINE_PER_CPU(struct irq_stack *, hardirq_stack);
+DEFINE_PER_CPU(struct irq_stack *, softirq_stack);
 
 static void call_on_stack(void *func, void *stack)
 {
@@ -77,14 +69,22 @@ static void call_on_stack(void *func, void *stack)
                     : "memory", "cc", "edx", "ecx", "eax");
 }
 
+/* how to get the current stack pointer from C */
+register unsigned long current_stack_pointer asm("esp") __used;
+
+static inline void *current_stack(void)
+{
+       return (void *)(current_stack_pointer & ~(THREAD_SIZE - 1));
+}
+
 static inline int
 execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
 {
-       union irq_ctx *curctx, *irqctx;
+       struct irq_stack *curstk, *irqstk;
        u32 *isp, *prev_esp, arg1, arg2;
 
-       curctx = (union irq_ctx *) current_thread_info();
-       irqctx = __this_cpu_read(hardirq_ctx);
+       curstk = (struct irq_stack *) current_stack();
+       irqstk = __this_cpu_read(hardirq_stack);
 
        /*
         * this is where we switch to the IRQ stack. However, if we are
@@ -92,15 +92,13 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
         * handler) we can't do that and just have to keep using the
         * current stack (which is the irq stack already after all)
         */
-       if (unlikely(curctx == irqctx))
+       if (unlikely(curstk == irqstk))
                return 0;
 
-       /* build the stack frame on the IRQ stack */
-       isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
-       irqctx->tinfo.task = curctx->tinfo.task;
-       /* Save the next esp after thread_info */
-       prev_esp = (u32 *) ((char *)irqctx + sizeof(struct thread_info) -
-                           sizeof(long));
+       isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));
+
+       /* Save the next esp at the bottom of the stack */
+       prev_esp = (u32 *)irqstk;
        *prev_esp = current_stack_pointer;
 
        if (unlikely(overflow))
@@ -121,49 +119,39 @@ execute_on_irq_stack(int overflow, struct irq_desc *desc, int irq)
  */
 void irq_ctx_init(int cpu)
 {
-       union irq_ctx *irqctx;
+       struct irq_stack *irqstk;
 
-       if (per_cpu(hardirq_ctx, cpu))
+       if (per_cpu(hardirq_stack, cpu))
                return;
 
-       irqctx = page_address(alloc_pages_node(cpu_to_node(cpu),
+       irqstk = page_address(alloc_pages_node(cpu_to_node(cpu),
                                               THREADINFO_GFP,
                                               THREAD_SIZE_ORDER));
-       memset(&irqctx->tinfo, 0, sizeof(struct thread_info));
-       irqctx->tinfo.cpu               = cpu;
-       irqctx->tinfo.addr_limit        = MAKE_MM_SEG(0);
+       per_cpu(hardirq_stack, cpu) = irqstk;
 
-       per_cpu(hardirq_ctx, cpu) = irqctx;
-
-       irqctx = page_address(alloc_pages_node(cpu_to_node(cpu),
+       irqstk = page_address(alloc_pages_node(cpu_to_node(cpu),
                                               THREADINFO_GFP,
                                               THREAD_SIZE_ORDER));
-       memset(&irqctx->tinfo, 0, sizeof(struct thread_info));
-       irqctx->tinfo.cpu               = cpu;
-       irqctx->tinfo.addr_limit        = MAKE_MM_SEG(0);
-
-       per_cpu(softirq_ctx, cpu) = irqctx;
+       per_cpu(softirq_stack, cpu) = irqstk;
 
        printk(KERN_DEBUG "CPU %u irqstacks, hard=%p soft=%p\n",
-              cpu, per_cpu(hardirq_ctx, cpu),  per_cpu(softirq_ctx, cpu));
+              cpu, per_cpu(hardirq_stack, cpu),  per_cpu(softirq_stack, cpu));
 }
 
 void do_softirq_own_stack(void)
 {
-       struct thread_info *curctx;
-       union irq_ctx *irqctx;
+       struct thread_info *curstk;
+       struct irq_stack *irqstk;
        u32 *isp, *prev_esp;
 
-       curctx = current_thread_info();
-       irqctx = __this_cpu_read(softirq_ctx);
-       irqctx->tinfo.task = curctx->task;
+       curstk = current_stack();
+       irqstk = __this_cpu_read(softirq_stack);
 
        /* build the stack frame on the softirq stack */
-       isp = (u32 *) ((char *)irqctx + sizeof(*irqctx));
+       isp = (u32 *) ((char *)irqstk + sizeof(*irqstk));
 
        /* Push the previous esp onto the stack */
-       prev_esp = (u32 *) ((char *)irqctx + sizeof(struct thread_info) -
-                           sizeof(long));
+       prev_esp = (u32 *)irqstk;
        *prev_esp = current_stack_pointer;
 
        call_on_stack(__do_softirq, isp);

diff --git a/arch/x86/kernel/process_32.c b/arch/x86/kernel/process_32.c
index 0de43e98ce08604afa886ceba8e17b7c66583e97..7bc86bbe748599b92c2b9b2b221f98b10224c5b2 100644 (file)
--- a/arch/x86/kernel/process_32.c
+++ b/arch/x86/kernel/process_32.c
@@ -314,6 +314,10 @@ __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
         */
        arch_end_context_switch(next_p);
 
+       this_cpu_write(kernel_stack,
+                 (unsigned long)task_stack_page(next_p) +
+                 THREAD_SIZE - KERNEL_STACK_OFFSET);
+
        /*
         * Restore %gs if needed (which is common)
         */

diff --git a/arch/x86/kernel/ptrace.c b/arch/x86/kernel/ptrace.c
index f352a7cc43a101e24e2913641d4cf551fbc2b90a..678c0ada3b3ce5f94135c7076cf59800850ca474 100644 (file)
--- a/arch/x86/kernel/ptrace.c
+++ b/arch/x86/kernel/ptrace.c
@@ -189,7 +189,7 @@ unsigned long kernel_stack_pointer(struct pt_regs *regs)
        if (context == (sp & ~(THREAD_SIZE - 1)))
                return sp;
 
-       prev_esp = (u32 *)(context + sizeof(struct thread_info) - sizeof(long));
+       prev_esp = (u32 *)(context);
        if (prev_esp)
                return (unsigned long)prev_esp;
 

diff --git a/arch/x86/kernel/smpboot.c b/arch/x86/kernel/smpboot.c
index a32da804252e374b5d266e6788653f24fc98705d..867d53ea88a3ddc8e0d6e2e51d680c4aefac457c 100644 (file)
--- a/arch/x86/kernel/smpboot.c
+++ b/arch/x86/kernel/smpboot.c
@@ -758,10 +758,10 @@ static int do_boot_cpu(int apicid, int cpu, struct task_struct *idle)
 #else
        clear_tsk_thread_flag(idle, TIF_FORK);
        initial_gs = per_cpu_offset(cpu);
+#endif
        per_cpu(kernel_stack, cpu) =
                (unsigned long)task_stack_page(idle) -
                KERNEL_STACK_OFFSET + THREAD_SIZE;
-#endif
        early_gdt_descr.address = (unsigned long)get_cpu_gdt_table(cpu);
        initial_code = (unsigned long)start_secondary;
        stack_start  = idle->thread.sp;