arch/x86/kernel/process_64.c

   1 /*
   2  *  Copyright (C) 1995  Linus Torvalds
   3  *
   4  *  Pentium III FXSR, SSE support
   5  *      Gareth Hughes <gareth@valinux.com>, May 2000
   6  *
   7  *  X86-64 port
   8  *      Andi Kleen.
   9  *
  10  *      CPU hotplug support - ashok.raj@intel.com
  11  */
  12
  13 /*
  14  * This file handles the architecture-dependent parts of process handling..
  15  */
  16
  17 #include <stdarg.h>
  18
  19 #include <linux/cpu.h>
  20 #include <linux/errno.h>
  21 #include <linux/sched.h>
  22 #include <linux/kernel.h>
  23 #include <linux/mm.h>
  24 #include <linux/fs.h>
  25 #include <linux/elfcore.h>
  26 #include <linux/smp.h>
  27 #include <linux/slab.h>
  28 #include <linux/user.h>
  29 #include <linux/module.h>
  30 #include <linux/a.out.h>
  31 #include <linux/interrupt.h>
  32 #include <linux/delay.h>
  33 #include <linux/ptrace.h>
  34 #include <linux/utsname.h>
  35 #include <linux/random.h>
  36 #include <linux/notifier.h>
  37 #include <linux/kprobes.h>
  38 #include <linux/kdebug.h>
  39 #include <linux/tick.h>
  40
  41 #include <asm/uaccess.h>
  42 #include <asm/pgtable.h>
  43 #include <asm/system.h>
  44 #include <asm/io.h>
  45 #include <asm/processor.h>
  46 #include <asm/i387.h>
  47 #include <asm/mmu_context.h>
  48 #include <asm/pda.h>
  49 #include <asm/prctl.h>
  50 #include <asm/desc.h>
  51 #include <asm/proto.h>
  52 #include <asm/ia32.h>
  53 #include <asm/idle.h>
  54
  55 asmlinkage extern void ret_from_fork(void);
  56
  57 unsigned long kernel_thread_flags = CLONE_VM | CLONE_UNTRACED;
  58
  59 unsigned long boot_option_idle_override = 0;
  60 EXPORT_SYMBOL(boot_option_idle_override);
  61
  62 /*
  63  * Powermanagement idle function, if any..
  64  */
  65 void (*pm_idle)(void);
  66 EXPORT_SYMBOL(pm_idle);
  67 static DEFINE_PER_CPU(unsigned int, cpu_idle_state);
  68
  69 static ATOMIC_NOTIFIER_HEAD(idle_notifier);
  70
  71 void idle_notifier_register(struct notifier_block *n)
  72 {
  73         atomic_notifier_chain_register(&idle_notifier, n);
  74 }
  75 EXPORT_SYMBOL_GPL(idle_notifier_register);
  76
  77 void idle_notifier_unregister(struct notifier_block *n)
  78 {
  79         atomic_notifier_chain_unregister(&idle_notifier, n);
  80 }
  81 EXPORT_SYMBOL(idle_notifier_unregister);
  82
  83 void enter_idle(void)
  84 {
  85         write_pda(isidle, 1);
  86         atomic_notifier_call_chain(&idle_notifier, IDLE_START, NULL);
  87 }
  88
  89 static void __exit_idle(void)
  90 {
  91         if (test_and_clear_bit_pda(0, isidle) == 0)
  92                 return;
  93         atomic_notifier_call_chain(&idle_notifier, IDLE_END, NULL);
  94 }
  95
  96 /* Called from interrupts to signify idle end */
  97 void exit_idle(void)
  98 {
  99         /* idle loop has pid 0 */
 100         if (current->pid)
 101                 return;
 102         __exit_idle();
 103 }
 104
 105 /*
 106  * We use this if we don't have any better
 107  * idle routine..
 108  */
 109 static void default_idle(void)
 110 {
 111         current_thread_info()->status &= ~TS_POLLING;
 112         /*
 113          * TS_POLLING-cleared state must be visible before we
 114          * test NEED_RESCHED:
 115          */
 116         smp_mb();
 117         local_irq_disable();
 118         if (!need_resched()) {
 119                 /* Enables interrupts one instruction before HLT.
 120                    x86 special cases this so there is no race. */
 121                 safe_halt();
 122         } else
 123                 local_irq_enable();
 124         current_thread_info()->status |= TS_POLLING;
 125 }
 126
 127 /*
 128  * On SMP it's slightly faster (but much more power-consuming!)
 129  * to poll the ->need_resched flag instead of waiting for the
 130  * cross-CPU IPI to arrive. Use this option with caution.
 131  */
 132 static void poll_idle (void)
 133 {
 134         local_irq_enable();
 135         cpu_relax();
 136 }
 137
 138 static void do_nothing(void *unused)
 139 {
 140 }
 141
 142 void cpu_idle_wait(void)
 143 {
 144         unsigned int cpu, this_cpu = get_cpu();
 145         cpumask_t map, tmp = current->cpus_allowed;
 146
 147         set_cpus_allowed(current, cpumask_of_cpu(this_cpu));
 148         put_cpu();
 149
 150         cpus_clear(map);
 151         for_each_online_cpu(cpu) {
 152                 per_cpu(cpu_idle_state, cpu) = 1;
 153                 cpu_set(cpu, map);
 154         }
 155
 156         __get_cpu_var(cpu_idle_state) = 0;
 157
 158         wmb();
 159         do {
 160                 ssleep(1);
 161                 for_each_online_cpu(cpu) {
 162                         if (cpu_isset(cpu, map) &&
 163                                         !per_cpu(cpu_idle_state, cpu))
 164                                 cpu_clear(cpu, map);
 165                 }
 166                 cpus_and(map, map, cpu_online_map);
 167                 /*
 168                  * We waited 1 sec, if a CPU still did not call idle
 169                  * it may be because it is in idle and not waking up
 170                  * because it has nothing to do.
 171                  * Give all the remaining CPUS a kick.
 172                  */
 173                 smp_call_function_mask(map, do_nothing, 0, 0);
 174         } while (!cpus_empty(map));
 175
 176         set_cpus_allowed(current, tmp);
 177 }
 178 EXPORT_SYMBOL_GPL(cpu_idle_wait);
 179
 180 #ifdef CONFIG_HOTPLUG_CPU
 181 DECLARE_PER_CPU(int, cpu_state);
 182
 183 #include <asm/nmi.h>
 184 /* We halt the CPU with physical CPU hotplug */
 185 static inline void play_dead(void)
 186 {
 187         idle_task_exit();
 188         wbinvd();
 189         mb();
 190         /* Ack it */
 191         __get_cpu_var(cpu_state) = CPU_DEAD;
 192
 193         local_irq_disable();
 194         while (1)
 195                 halt();
 196 }
 197 #else
 198 static inline void play_dead(void)
 199 {
 200         BUG();
 201 }
 202 #endif /* CONFIG_HOTPLUG_CPU */
 203
 204 /*
 205  * The idle thread. There's no useful work to be
 206  * done, so just try to conserve power and have a
 207  * low exit latency (ie sit in a loop waiting for
 208  * somebody to say that they'd like to reschedule)
 209  */
 210 void cpu_idle (void)
 211 {
 212         current_thread_info()->status |= TS_POLLING;
 213         /* endless idle loop with no priority at all */
 214         while (1) {
 215                 while (!need_resched()) {
 216                         void (*idle)(void);
 217
 218                         if (__get_cpu_var(cpu_idle_state))
 219                                 __get_cpu_var(cpu_idle_state) = 0;
 220
 221                         tick_nohz_stop_sched_tick();
 222
 223                         rmb();
 224                         idle = pm_idle;
 225                         if (!idle)
 226                                 idle = default_idle;
 227                         if (cpu_is_offline(smp_processor_id()))
 228                                 play_dead();
 229                         /*
 230                          * Idle routines should keep interrupts disabled
 231                          * from here on, until they go to idle.
 232                          * Otherwise, idle callbacks can misfire.
 233                          */
 234                         local_irq_disable();
 235                         enter_idle();
 236                         idle();
 237                         /* In many cases the interrupt that ended idle
 238                            has already called exit_idle. But some idle
 239                            loops can be woken up without interrupt. */
 240                         __exit_idle();
 241                 }
 242
 243                 tick_nohz_restart_sched_tick();
 244                 preempt_enable_no_resched();
 245                 schedule();
 246                 preempt_disable();
 247         }
 248 }
 249
 250 /*
 251  * This uses new MONITOR/MWAIT instructions on P4 processors with PNI,
 252  * which can obviate IPI to trigger checking of need_resched.
 253  * We execute MONITOR against need_resched and enter optimized wait state
 254  * through MWAIT. Whenever someone changes need_resched, we would be woken
 255  * up from MWAIT (without an IPI).
 256  *
 257  * New with Core Duo processors, MWAIT can take some hints based on CPU
 258  * capability.
 259  */
 260 void mwait_idle_with_hints(unsigned long eax, unsigned long ecx)
 261 {
 262         if (!need_resched()) {
 263                 __monitor((void *)&current_thread_info()->flags, 0, 0);
 264                 smp_mb();
 265                 if (!need_resched())
 266                         __mwait(eax, ecx);
 267         }
 268 }
 269
 270 /* Default MONITOR/MWAIT with no hints, used for default C1 state */
 271 static void mwait_idle(void)
 272 {
 273         if (!need_resched()) {
 274                 __monitor((void *)&current_thread_info()->flags, 0, 0);
 275                 smp_mb();
 276                 if (!need_resched())
 277                         __sti_mwait(0, 0);
 278                 else
 279                         local_irq_enable();
 280         } else {
 281                 local_irq_enable();
 282         }
 283 }
 284
 285 void __cpuinit select_idle_routine(const struct cpuinfo_x86 *c)
 286 {
 287         static int printed;
 288         if (cpu_has(c, X86_FEATURE_MWAIT)) {
 289                 /*
 290                  * Skip, if setup has overridden idle.
 291                  * One CPU supports mwait => All CPUs supports mwait
 292                  */
 293                 if (!pm_idle) {
 294                         if (!printed) {
 295                                 printk(KERN_INFO "using mwait in idle threads.\n");
 296                                 printed = 1;
 297                         }
 298                         pm_idle = mwait_idle;
 299                 }
 300         }
 301 }
 302
 303 static int __init idle_setup (char *str)
 304 {
 305         if (!strcmp(str, "poll")) {
 306                 printk("using polling idle threads.\n");
 307                 pm_idle = poll_idle;
 308         } else if (!strcmp(str, "mwait"))
 309                 force_mwait = 1;
 310         else
 311                 return -1;
 312
 313         boot_option_idle_override = 1;
 314         return 0;
 315 }
 316 early_param("idle", idle_setup);
 317
 318 /* Prints also some state that isn't saved in the pt_regs */
 319 void __show_regs(struct pt_regs * regs)
 320 {
 321         unsigned long cr0 = 0L, cr2 = 0L, cr3 = 0L, cr4 = 0L, fs, gs, shadowgs;
 322         unsigned long d0, d1, d2, d3, d6, d7;
 323         unsigned int fsindex,gsindex;
 324         unsigned int ds,cs,es;
 325
 326         printk("\n");
 327         print_modules();
 328         printk("Pid: %d, comm: %.20s %s %s %.*s\n",
 329                 current->pid, current->comm, print_tainted(),
 330                 init_utsname()->release,
 331                 (int)strcspn(init_utsname()->version, " "),
 332                 init_utsname()->version);
 333         printk("RIP: %04lx:[<%016lx>] ", regs->cs & 0xffff, regs->rip);
 334         printk_address(regs->rip);
 335         printk("RSP: %04lx:%016lx  EFLAGS: %08lx\n", regs->ss, regs->rsp,
 336                 regs->eflags);
 337         printk("RAX: %016lx RBX: %016lx RCX: %016lx\n",
 338                regs->rax, regs->rbx, regs->rcx);
 339         printk("RDX: %016lx RSI: %016lx RDI: %016lx\n",
 340                regs->rdx, regs->rsi, regs->rdi);
 341         printk("RBP: %016lx R08: %016lx R09: %016lx\n",
 342                regs->rbp, regs->r8, regs->r9);
 343         printk("R10: %016lx R11: %016lx R12: %016lx\n",
 344                regs->r10, regs->r11, regs->r12);
 345         printk("R13: %016lx R14: %016lx R15: %016lx\n",
 346                regs->r13, regs->r14, regs->r15);
 347
 348         asm("movl %%ds,%0" : "=r" (ds));
 349         asm("movl %%cs,%0" : "=r" (cs));
 350         asm("movl %%es,%0" : "=r" (es));
 351         asm("movl %%fs,%0" : "=r" (fsindex));
 352         asm("movl %%gs,%0" : "=r" (gsindex));
 353
 354         rdmsrl(MSR_FS_BASE, fs);
 355         rdmsrl(MSR_GS_BASE, gs);
 356         rdmsrl(MSR_KERNEL_GS_BASE, shadowgs);
 357
 358         cr0 = read_cr0();
 359         cr2 = read_cr2();
 360         cr3 = read_cr3();
 361         cr4 = read_cr4();
 362
 363         printk("FS:  %016lx(%04x) GS:%016lx(%04x) knlGS:%016lx\n",
 364                fs,fsindex,gs,gsindex,shadowgs);
 365         printk("CS:  %04x DS: %04x ES: %04x CR0: %016lx\n", cs, ds, es, cr0);
 366         printk("CR2: %016lx CR3: %016lx CR4: %016lx\n", cr2, cr3, cr4);
 367
 368         get_debugreg(d0, 0);
 369         get_debugreg(d1, 1);
 370         get_debugreg(d2, 2);
 371         printk("DR0: %016lx DR1: %016lx DR2: %016lx\n", d0, d1, d2);
 372         get_debugreg(d3, 3);
 373         get_debugreg(d6, 6);
 374         get_debugreg(d7, 7);
 375         printk("DR3: %016lx DR6: %016lx DR7: %016lx\n", d3, d6, d7);
 376 }
 377
 378 void show_regs(struct pt_regs *regs)
 379 {
 380         printk("CPU %d:", smp_processor_id());
 381         __show_regs(regs);
 382         show_trace(NULL, regs, (void *)(regs + 1));
 383 }
 384
 385 /*
 386  * Free current thread data structures etc..
 387  */
 388 void exit_thread(void)
 389 {
 390         struct task_struct *me = current;
 391         struct thread_struct *t = &me->thread;
 392
 393         if (me->thread.io_bitmap_ptr) {
 394                 struct tss_struct *tss = &per_cpu(init_tss, get_cpu());
 395
 396                 kfree(t->io_bitmap_ptr);
 397                 t->io_bitmap_ptr = NULL;
 398                 clear_thread_flag(TIF_IO_BITMAP);
 399                 /*
 400                  * Careful, clear this in the TSS too:
 401                  */
 402                 memset(tss->io_bitmap, 0xff, t->io_bitmap_max);
 403                 t->io_bitmap_max = 0;
 404                 put_cpu();
 405         }
 406 }
 407
 408 void flush_thread(void)
 409 {
 410         struct task_struct *tsk = current;
 411
 412         if (test_tsk_thread_flag(tsk, TIF_ABI_PENDING)) {
 413                 clear_tsk_thread_flag(tsk, TIF_ABI_PENDING);
 414                 if (test_tsk_thread_flag(tsk, TIF_IA32)) {
 415                         clear_tsk_thread_flag(tsk, TIF_IA32);
 416                 } else {
 417                         set_tsk_thread_flag(tsk, TIF_IA32);
 418                         current_thread_info()->status |= TS_COMPAT;
 419                 }
 420         }
 421         clear_tsk_thread_flag(tsk, TIF_DEBUG);
 422
 423         tsk->thread.debugreg0 = 0;
 424         tsk->thread.debugreg1 = 0;
 425         tsk->thread.debugreg2 = 0;
 426         tsk->thread.debugreg3 = 0;
 427         tsk->thread.debugreg6 = 0;
 428         tsk->thread.debugreg7 = 0;
 429         memset(tsk->thread.tls_array, 0, sizeof(tsk->thread.tls_array));
 430         /*
 431          * Forget coprocessor state..
 432          */
 433         clear_fpu(tsk);
 434         clear_used_math();
 435 }
 436
 437 void release_thread(struct task_struct *dead_task)
 438 {
 439         if (dead_task->mm) {
 440                 if (dead_task->mm->context.size) {
 441                         printk("WARNING: dead process %8s still has LDT? <%p/%d>\n",
 442                                         dead_task->comm,
 443                                         dead_task->mm->context.ldt,
 444                                         dead_task->mm->context.size);
 445                         BUG();
 446                 }
 447         }
 448 }
 449
 450 static inline void set_32bit_tls(struct task_struct *t, int tls, u32 addr)
 451 {
 452         struct user_desc ud = {
 453                 .base_addr = addr,
 454                 .limit = 0xfffff,
 455                 .seg_32bit = 1,
 456                 .limit_in_pages = 1,
 457                 .useable = 1,
 458         };
 459         struct n_desc_struct *desc = (void *)t->thread.tls_array;
 460         desc += tls;
 461         desc->a = LDT_entry_a(&ud);
 462         desc->b = LDT_entry_b(&ud);
 463 }
 464
 465 static inline u32 read_32bit_tls(struct task_struct *t, int tls)
 466 {
 467         struct desc_struct *desc = (void *)t->thread.tls_array;
 468         desc += tls;
 469         return desc->base0 |
 470                 (((u32)desc->base1) << 16) |
 471                 (((u32)desc->base2) << 24);
 472 }
 473
 474 /*
 475  * This gets called before we allocate a new thread and copy
 476  * the current task into it.
 477  */
 478 void prepare_to_copy(struct task_struct *tsk)
 479 {
 480         unlazy_fpu(tsk);
 481 }
 482
 483 int copy_thread(int nr, unsigned long clone_flags, unsigned long rsp,
 484                 unsigned long unused,
 485         struct task_struct * p, struct pt_regs * regs)
 486 {
 487         int err;
 488         struct pt_regs * childregs;
 489         struct task_struct *me = current;
 490
 491         childregs = ((struct pt_regs *)
 492                         (THREAD_SIZE + task_stack_page(p))) - 1;
 493         *childregs = *regs;
 494
 495         childregs->rax = 0;
 496         childregs->rsp = rsp;
 497         if (rsp == ~0UL)
 498                 childregs->rsp = (unsigned long)childregs;
 499
 500         p->thread.rsp = (unsigned long) childregs;
 501         p->thread.rsp0 = (unsigned long) (childregs+1);
 502         p->thread.userrsp = me->thread.userrsp;
 503
 504         set_tsk_thread_flag(p, TIF_FORK);
 505
 506         p->thread.fs = me->thread.fs;
 507         p->thread.gs = me->thread.gs;
 508
 509         asm("mov %%gs,%0" : "=m" (p->thread.gsindex));
 510         asm("mov %%fs,%0" : "=m" (p->thread.fsindex));
 511         asm("mov %%es,%0" : "=m" (p->thread.es));
 512         asm("mov %%ds,%0" : "=m" (p->thread.ds));
 513
 514         if (unlikely(test_tsk_thread_flag(me, TIF_IO_BITMAP))) {
 515                 p->thread.io_bitmap_ptr = kmalloc(IO_BITMAP_BYTES, GFP_KERNEL);
 516                 if (!p->thread.io_bitmap_ptr) {
 517                         p->thread.io_bitmap_max = 0;
 518                         return -ENOMEM;
 519                 }
 520                 memcpy(p->thread.io_bitmap_ptr, me->thread.io_bitmap_ptr,
 521                                 IO_BITMAP_BYTES);
 522                 set_tsk_thread_flag(p, TIF_IO_BITMAP);
 523         }
 524
 525         /*
 526          * Set a new TLS for the child thread?
 527          */
 528         if (clone_flags & CLONE_SETTLS) {
 529 #ifdef CONFIG_IA32_EMULATION
 530                 if (test_thread_flag(TIF_IA32))
 531                         err = ia32_child_tls(p, childregs);
 532                 else
 533 #endif
 534                         err = do_arch_prctl(p, ARCH_SET_FS, childregs->r8);
 535                 if (err)
 536                         goto out;
 537         }
 538         err = 0;
 539 out:
 540         if (err && p->thread.io_bitmap_ptr) {
 541                 kfree(p->thread.io_bitmap_ptr);
 542                 p->thread.io_bitmap_max = 0;
 543         }
 544         return err;
 545 }
 546
 547 /*
 548  * This special macro can be used to load a debugging register
 549  */
 550 #define loaddebug(thread,r) set_debugreg(thread->debugreg ## r, r)
 551
 552 static inline void __switch_to_xtra(struct task_struct *prev_p,
 553                                     struct task_struct *next_p,
 554                                     struct tss_struct *tss)
 555 {
 556         struct thread_struct *prev, *next;
 557
 558         prev = &prev_p->thread,
 559         next = &next_p->thread;
 560
 561         if (test_tsk_thread_flag(next_p, TIF_DEBUG)) {
 562                 loaddebug(next, 0);
 563                 loaddebug(next, 1);
 564                 loaddebug(next, 2);
 565                 loaddebug(next, 3);
 566                 /* no 4 and 5 */
 567                 loaddebug(next, 6);
 568                 loaddebug(next, 7);
 569         }
 570
 571         if (test_tsk_thread_flag(next_p, TIF_IO_BITMAP)) {
 572                 /*
 573                  * Copy the relevant range of the IO bitmap.
 574                  * Normally this is 128 bytes or less:
 575                  */
 576                 memcpy(tss->io_bitmap, next->io_bitmap_ptr,
 577                        max(prev->io_bitmap_max, next->io_bitmap_max));
 578         } else if (test_tsk_thread_flag(prev_p, TIF_IO_BITMAP)) {
 579                 /*
 580                  * Clear any possible leftover bits:
 581                  */
 582                 memset(tss->io_bitmap, 0xff, prev->io_bitmap_max);
 583         }
 584 }
 585
 586 /*
 587  *      switch_to(x,y) should switch tasks from x to y.
 588  *
 589  * This could still be optimized:
 590  * - fold all the options into a flag word and test it with a single test.
 591  * - could test fs/gs bitsliced
 592  *
 593  * Kprobes not supported here. Set the probe on schedule instead.
 594  */
 595 struct task_struct *
 596 __switch_to(struct task_struct *prev_p, struct task_struct *next_p)
 597 {
 598         struct thread_struct *prev = &prev_p->thread,
 599                                  *next = &next_p->thread;
 600         int cpu = smp_processor_id();
 601         struct tss_struct *tss = &per_cpu(init_tss, cpu);
 602
 603         /* we're going to use this soon, after a few expensive things */
 604         if (next_p->fpu_counter>5)
 605                 prefetch(&next->i387.fxsave);
 606
 607         /*
 608          * Reload esp0, LDT and the page table pointer:
 609          */
 610         tss->rsp0 = next->rsp0;
 611
 612         /*
 613          * Switch DS and ES.
 614          * This won't pick up thread selector changes, but I guess that is ok.
 615          */
 616         asm volatile("mov %%es,%0" : "=m" (prev->es));
 617         if (unlikely(next->es | prev->es))
 618                 loadsegment(es, next->es);
 619
 620         asm volatile ("mov %%ds,%0" : "=m" (prev->ds));
 621         if (unlikely(next->ds | prev->ds))
 622                 loadsegment(ds, next->ds);
 623
 624         load_TLS(next, cpu);
 625
 626         /*
 627          * Switch FS and GS.
 628          */
 629         {
 630                 unsigned fsindex;
 631                 asm volatile("movl %%fs,%0" : "=r" (fsindex));
 632                 /* segment register != 0 always requires a reload.
 633                    also reload when it has changed.
 634                    when prev process used 64bit base always reload
 635                    to avoid an information leak. */
 636                 if (unlikely(fsindex | next->fsindex | prev->fs)) {
 637                         loadsegment(fs, next->fsindex);
 638                         /* check if the user used a selector != 0
 639                          * if yes clear 64bit base, since overloaded base
 640                          * is always mapped to the Null selector
 641                          */
 642                         if (fsindex)
 643                         prev->fs = 0;
 644                 }
 645                 /* when next process has a 64bit base use it */
 646                 if (next->fs)
 647                         wrmsrl(MSR_FS_BASE, next->fs);
 648                 prev->fsindex = fsindex;
 649         }
 650         {
 651                 unsigned gsindex;
 652                 asm volatile("movl %%gs,%0" : "=r" (gsindex));
 653                 if (unlikely(gsindex | next->gsindex | prev->gs)) {
 654                         load_gs_index(next->gsindex);
 655                         if (gsindex)
 656                         prev->gs = 0;
 657                 }
 658                 if (next->gs)
 659                         wrmsrl(MSR_KERNEL_GS_BASE, next->gs);
 660                 prev->gsindex = gsindex;
 661         }
 662
 663         /* Must be after DS reload */
 664         unlazy_fpu(prev_p);
 665
 666         /*
 667          * Switch the PDA and FPU contexts.
 668          */
 669         prev->userrsp = read_pda(oldrsp);
 670         write_pda(oldrsp, next->userrsp);
 671         write_pda(pcurrent, next_p);
 672
 673         write_pda(kernelstack,
 674         (unsigned long)task_stack_page(next_p) + THREAD_SIZE - PDA_STACKOFFSET);
 675 #ifdef CONFIG_CC_STACKPROTECTOR
 676         write_pda(stack_canary, next_p->stack_canary);
 677         /*
 678          * Build time only check to make sure the stack_canary is at
 679          * offset 40 in the pda; this is a gcc ABI requirement
 680          */
 681         BUILD_BUG_ON(offsetof(struct x8664_pda, stack_canary) != 40);
 682 #endif
 683
 684         /*
 685          * Now maybe reload the debug registers and handle I/O bitmaps
 686          */
 687         if (unlikely((task_thread_info(next_p)->flags & _TIF_WORK_CTXSW))
 688             || test_tsk_thread_flag(prev_p, TIF_IO_BITMAP))
 689                 __switch_to_xtra(prev_p, next_p, tss);
 690
 691         /* If the task has used fpu the last 5 timeslices, just do a full
 692          * restore of the math state immediately to avoid the trap; the
 693          * chances of needing FPU soon are obviously high now
 694          */
 695         if (next_p->fpu_counter>5)
 696                 math_state_restore();
 697         return prev_p;
 698 }
 699
 700 /*
 701  * sys_execve() executes a new program.
 702  */
 703 asmlinkage
 704 long sys_execve(char __user *name, char __user * __user *argv,
 705                 char __user * __user *envp, struct pt_regs regs)
 706 {
 707         long error;
 708         char * filename;
 709
 710         filename = getname(name);
 711         error = PTR_ERR(filename);
 712         if (IS_ERR(filename))
 713                 return error;
 714         error = do_execve(filename, argv, envp, &regs);
 715         if (error == 0) {
 716                 task_lock(current);
 717                 current->ptrace &= ~PT_DTRACE;
 718                 task_unlock(current);
 719         }
 720         putname(filename);
 721         return error;
 722 }
 723
 724 void set_personality_64bit(void)
 725 {
 726         /* inherit personality from parent */
 727
 728         /* Make sure to be in 64bit mode */
 729         clear_thread_flag(TIF_IA32);
 730
 731         /* TBD: overwrites user setup. Should have two bits.
 732            But 64bit processes have always behaved this way,
 733            so it's not too bad. The main problem is just that
 734            32bit childs are affected again. */
 735         current->personality &= ~READ_IMPLIES_EXEC;
 736 }
 737
 738 asmlinkage long sys_fork(struct pt_regs *regs)
 739 {
 740         return do_fork(SIGCHLD, regs->rsp, regs, 0, NULL, NULL);
 741 }
 742
 743 asmlinkage long
 744 sys_clone(unsigned long clone_flags, unsigned long newsp,
 745           void __user *parent_tid, void __user *child_tid, struct pt_regs *regs)
 746 {
 747         if (!newsp)
 748                 newsp = regs->rsp;
 749         return do_fork(clone_flags, newsp, regs, 0, parent_tid, child_tid);
 750 }
 751
 752 /*
 753  * This is trivial, and on the face of it looks like it
 754  * could equally well be done in user mode.
 755  *
 756  * Not so, for quite unobvious reasons - register pressure.
 757  * In user mode vfork() cannot have a stack frame, and if
 758  * done by calling the "clone()" system call directly, you
 759  * do not have enough call-clobbered registers to hold all
 760  * the information you need.
 761  */
 762 asmlinkage long sys_vfork(struct pt_regs *regs)
 763 {
 764         return do_fork(CLONE_VFORK | CLONE_VM | SIGCHLD, regs->rsp, regs, 0,
 765                     NULL, NULL);
 766 }
 767
 768 unsigned long get_wchan(struct task_struct *p)
 769 {
 770         unsigned long stack;
 771         u64 fp,rip;
 772         int count = 0;
 773
 774         if (!p || p == current || p->state==TASK_RUNNING)
 775                 return 0;
 776         stack = (unsigned long)task_stack_page(p);
 777         if (p->thread.rsp < stack || p->thread.rsp > stack+THREAD_SIZE)
 778                 return 0;
 779         fp = *(u64 *)(p->thread.rsp);
 780         do {
 781                 if (fp < (unsigned long)stack ||
 782                     fp > (unsigned long)stack+THREAD_SIZE)
 783                         return 0;
 784                 rip = *(u64 *)(fp+8);
 785                 if (!in_sched_functions(rip))
 786                         return rip;
 787                 fp = *(u64 *)fp;
 788         } while (count++ < 16);
 789         return 0;
 790 }
 791
 792 long do_arch_prctl(struct task_struct *task, int code, unsigned long addr)
 793 {
 794         int ret = 0;
 795         int doit = task == current;
 796         int cpu;
 797
 798         switch (code) {
 799         case ARCH_SET_GS:
 800                 if (addr >= TASK_SIZE_OF(task))
 801                         return -EPERM;
 802                 cpu = get_cpu();
 803                 /* handle small bases via the GDT because that's faster to
 804                    switch. */
 805                 if (addr <= 0xffffffff) {
 806                         set_32bit_tls(task, GS_TLS, addr);
 807                         if (doit) {
 808                                 load_TLS(&task->thread, cpu);
 809                                 load_gs_index(GS_TLS_SEL);
 810                         }
 811                         task->thread.gsindex = GS_TLS_SEL;
 812                         task->thread.gs = 0;
 813                 } else {
 814                         task->thread.gsindex = 0;
 815                         task->thread.gs = addr;
 816                         if (doit) {
 817                                 load_gs_index(0);
 818                                 ret = checking_wrmsrl(MSR_KERNEL_GS_BASE, addr);
 819                         }
 820                 }
 821                 put_cpu();
 822                 break;
 823         case ARCH_SET_FS:
 824                 /* Not strictly needed for fs, but do it for symmetry
 825                    with gs */
 826                 if (addr >= TASK_SIZE_OF(task))
 827                         return -EPERM;
 828                 cpu = get_cpu();
 829                 /* handle small bases via the GDT because that's faster to
 830                    switch. */
 831                 if (addr <= 0xffffffff) {
 832                         set_32bit_tls(task, FS_TLS, addr);
 833                         if (doit) {
 834                                 load_TLS(&task->thread, cpu);
 835                                 asm volatile("movl %0,%%fs" :: "r"(FS_TLS_SEL));
 836                         }
 837                         task->thread.fsindex = FS_TLS_SEL;
 838                         task->thread.fs = 0;
 839                 } else {
 840                         task->thread.fsindex = 0;
 841                         task->thread.fs = addr;
 842                         if (doit) {
 843                                 /* set the selector to 0 to not confuse
 844                                    __switch_to */
 845                                 asm volatile("movl %0,%%fs" :: "r" (0));
 846                                 ret = checking_wrmsrl(MSR_FS_BASE, addr);
 847                         }
 848                 }
 849                 put_cpu();
 850                 break;
 851         case ARCH_GET_FS: {
 852                 unsigned long base;
 853                 if (task->thread.fsindex == FS_TLS_SEL)
 854                         base = read_32bit_tls(task, FS_TLS);
 855                 else if (doit)
 856                         rdmsrl(MSR_FS_BASE, base);
 857                 else
 858                         base = task->thread.fs;
 859                 ret = put_user(base, (unsigned long __user *)addr);
 860                 break;
 861         }
 862         case ARCH_GET_GS: {
 863                 unsigned long base;
 864                 unsigned gsindex;
 865                 if (task->thread.gsindex == GS_TLS_SEL)
 866                         base = read_32bit_tls(task, GS_TLS);
 867                 else if (doit) {
 868                         asm("movl %%gs,%0" : "=r" (gsindex));
 869                         if (gsindex)
 870                                 rdmsrl(MSR_KERNEL_GS_BASE, base);
 871                         else
 872                                 base = task->thread.gs;
 873                 }
 874                 else
 875                         base = task->thread.gs;
 876                 ret = put_user(base, (unsigned long __user *)addr);
 877                 break;
 878         }
 879
 880         default:
 881                 ret = -EINVAL;
 882                 break;
 883         }
 884
 885         return ret;
 886 }
 887
 888 long sys_arch_prctl(int code, unsigned long addr)
 889 {
 890         return do_arch_prctl(current, code, addr);
 891 }
 892
 893 /*
 894  * Capture the user space registers if the task is not running (in user space)
 895  */
 896 int dump_task_regs(struct task_struct *tsk, elf_gregset_t *regs)
 897 {
 898         struct pt_regs *pp, ptregs;
 899
 900         pp = task_pt_regs(tsk);
 901
 902         ptregs = *pp;
 903         ptregs.cs &= 0xffff;
 904         ptregs.ss &= 0xffff;
 905
 906         elf_core_copy_regs(regs, &ptregs);
 907
 908         return 1;
 909 }
 910
 911 unsigned long arch_align_stack(unsigned long sp)
 912 {
 913         if (!(current->personality & ADDR_NO_RANDOMIZE) && randomize_va_space)
 914                 sp -= get_random_int() % 8192;
 915         return sp & ~0xf;
 916 }