arch/x86/mm/fault_64.c

   1 /*
   2  *  linux/arch/x86-64/mm/fault.c
   3  *
   4  *  Copyright (C) 1995  Linus Torvalds
   5  *  Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
   6  */
   7
   8 #include <linux/signal.h>
   9 #include <linux/sched.h>
  10 #include <linux/kernel.h>
  11 #include <linux/errno.h>
  12 #include <linux/string.h>
  13 #include <linux/types.h>
  14 #include <linux/ptrace.h>
  15 #include <linux/mman.h>
  16 #include <linux/mm.h>
  17 #include <linux/smp.h>
  18 #include <linux/interrupt.h>
  19 #include <linux/init.h>
  20 #include <linux/tty.h>
  21 #include <linux/vt_kern.h>              /* For unblank_screen() */
  22 #include <linux/compiler.h>
  23 #include <linux/vmalloc.h>
  24 #include <linux/module.h>
  25 #include <linux/kprobes.h>
  26 #include <linux/uaccess.h>
  27 #include <linux/kdebug.h>
  28
  29 #include <asm/system.h>
  30 #include <asm/pgalloc.h>
  31 #include <asm/smp.h>
  32 #include <asm/tlbflush.h>
  33 #include <asm/proto.h>
  34 #include <asm-generic/sections.h>
  35
  36 /* Page fault error code bits */
  37 #define PF_PROT (1<<0)          /* or no page found */
  38 #define PF_WRITE        (1<<1)
  39 #define PF_USER (1<<2)
  40 #define PF_RSVD (1<<3)
  41 #define PF_INSTR        (1<<4)
  42
  43 #ifdef CONFIG_KPROBES
  44 static inline int notify_page_fault(struct pt_regs *regs)
  45 {
  46         int ret = 0;
  47
  48         /* kprobe_running() needs smp_processor_id() */
  49         if (!user_mode(regs)) {
  50                 preempt_disable();
  51                 if (kprobe_running() && kprobe_fault_handler(regs, 14))
  52                         ret = 1;
  53                 preempt_enable();
  54         }
  55
  56         return ret;
  57 }
  58 #else
  59 static inline int notify_page_fault(struct pt_regs *regs)
  60 {
  61         return 0;
  62 }
  63 #endif
  64
  65 /* Sometimes the CPU reports invalid exceptions on prefetch.
  66    Check that here and ignore.
  67    Opcode checker based on code by Richard Brunner */
  68 static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
  69                                 unsigned long error_code)
  70 {
  71         unsigned char *instr;
  72         int scan_more = 1;
  73         int prefetch = 0;
  74         unsigned char *max_instr;
  75
  76         /* If it was a exec fault ignore */
  77         if (error_code & PF_INSTR)
  78                 return 0;
  79
  80         instr = (unsigned char __user *)convert_rip_to_linear(current, regs);
  81         max_instr = instr + 15;
  82
  83         if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
  84                 return 0;
  85
  86         while (scan_more && instr < max_instr) {
  87                 unsigned char opcode;
  88                 unsigned char instr_hi;
  89                 unsigned char instr_lo;
  90
  91                 if (probe_kernel_address(instr, opcode))
  92                         break;
  93
  94                 instr_hi = opcode & 0xf0;
  95                 instr_lo = opcode & 0x0f;
  96                 instr++;
  97
  98                 switch (instr_hi) {
  99                 case 0x20:
 100                 case 0x30:
 101                         /* Values 0x26,0x2E,0x36,0x3E are valid x86
 102                            prefixes.  In long mode, the CPU will signal
 103                            invalid opcode if some of these prefixes are
 104                            present so we will never get here anyway */
 105                         scan_more = ((instr_lo & 7) == 0x6);
 106                         break;
 107
 108                 case 0x40:
 109                         /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
 110                            Need to figure out under what instruction mode the
 111                            instruction was issued ... */
 112                         /* Could check the LDT for lm, but for now it's good
 113                            enough to assume that long mode only uses well known
 114                            segments or kernel. */
 115                         scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
 116                         break;
 117
 118                 case 0x60:
 119                         /* 0x64 thru 0x67 are valid prefixes in all modes. */
 120                         scan_more = (instr_lo & 0xC) == 0x4;
 121                         break;
 122                 case 0xF0:
 123                         /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
 124                         scan_more = !instr_lo || (instr_lo>>1) == 1;
 125                         break;
 126                 case 0x00:
 127                         /* Prefetch instruction is 0x0F0D or 0x0F18 */
 128                         scan_more = 0;
 129                         if (probe_kernel_address(instr, opcode))
 130                                 break;
 131                         prefetch = (instr_lo == 0xF) &&
 132                                 (opcode == 0x0D || opcode == 0x18);
 133                         break;
 134                 default:
 135                         scan_more = 0;
 136                         break;
 137                 }
 138         }
 139         return prefetch;
 140 }
 141
 142 static int bad_address(void *p)
 143 {
 144         unsigned long dummy;
 145         return probe_kernel_address((unsigned long *)p, dummy);
 146 }
 147
 148 void dump_pagetable(unsigned long address)
 149 {
 150         pgd_t *pgd;
 151         pud_t *pud;
 152         pmd_t *pmd;
 153         pte_t *pte;
 154
 155         pgd = (pgd_t *)read_cr3();
 156
 157         pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
 158         pgd += pgd_index(address);
 159         if (bad_address(pgd)) goto bad;
 160         printk("PGD %lx ", pgd_val(*pgd));
 161         if (!pgd_present(*pgd)) goto ret;
 162
 163         pud = pud_offset(pgd, address);
 164         if (bad_address(pud)) goto bad;
 165         printk("PUD %lx ", pud_val(*pud));
 166         if (!pud_present(*pud)) goto ret;
 167
 168         pmd = pmd_offset(pud, address);
 169         if (bad_address(pmd)) goto bad;
 170         printk("PMD %lx ", pmd_val(*pmd));
 171         if (!pmd_present(*pmd) || pmd_large(*pmd)) goto ret;
 172
 173         pte = pte_offset_kernel(pmd, address);
 174         if (bad_address(pte)) goto bad;
 175         printk("PTE %lx", pte_val(*pte));
 176 ret:
 177         printk("\n");
 178         return;
 179 bad:
 180         printk("BAD\n");
 181 }
 182
 183 static const char errata93_warning[] =
 184 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
 185 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
 186 KERN_ERR "******* Please consider a BIOS update.\n"
 187 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
 188
 189 /* Workaround for K8 erratum #93 & buggy BIOS.
 190    BIOS SMM functions are required to use a specific workaround
 191    to avoid corruption of the 64bit RIP register on C stepping K8.
 192    A lot of BIOS that didn't get tested properly miss this.
 193    The OS sees this as a page fault with the upper 32bits of RIP cleared.
 194    Try to work around it here.
 195    Note we only handle faults in kernel here. */
 196
 197 static int is_errata93(struct pt_regs *regs, unsigned long address)
 198 {
 199         static int warned;
 200         if (address != regs->ip)
 201                 return 0;
 202         if ((address >> 32) != 0)
 203                 return 0;
 204         address |= 0xffffffffUL << 32;
 205         if ((address >= (u64)_stext && address <= (u64)_etext) ||
 206             (address >= MODULES_VADDR && address <= MODULES_END)) {
 207                 if (!warned) {
 208                         printk(errata93_warning);
 209                         warned = 1;
 210                 }
 211                 regs->ip = address;
 212                 return 1;
 213         }
 214         return 0;
 215 }
 216
 217 static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
 218                                  unsigned long error_code)
 219 {
 220         unsigned long flags = oops_begin();
 221         struct task_struct *tsk;
 222
 223         printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
 224                current->comm, address);
 225         dump_pagetable(address);
 226         tsk = current;
 227         tsk->thread.cr2 = address;
 228         tsk->thread.trap_no = 14;
 229         tsk->thread.error_code = error_code;
 230         if (__die("Bad pagetable", regs, error_code))
 231                 regs = NULL;
 232         oops_end(flags, regs, SIGKILL);
 233 }
 234
 235 /*
 236  * Handle a fault on the vmalloc area
 237  *
 238  * This assumes no large pages in there.
 239  */
 240 static int vmalloc_fault(unsigned long address)
 241 {
 242         pgd_t *pgd, *pgd_ref;
 243         pud_t *pud, *pud_ref;
 244         pmd_t *pmd, *pmd_ref;
 245         pte_t *pte, *pte_ref;
 246
 247         /* Copy kernel mappings over when needed. This can also
 248            happen within a race in page table update. In the later
 249            case just flush. */
 250
 251         pgd = pgd_offset(current->mm ?: &init_mm, address);
 252         pgd_ref = pgd_offset_k(address);
 253         if (pgd_none(*pgd_ref))
 254                 return -1;
 255         if (pgd_none(*pgd))
 256                 set_pgd(pgd, *pgd_ref);
 257         else
 258                 BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
 259
 260         /* Below here mismatches are bugs because these lower tables
 261            are shared */
 262
 263         pud = pud_offset(pgd, address);
 264         pud_ref = pud_offset(pgd_ref, address);
 265         if (pud_none(*pud_ref))
 266                 return -1;
 267         if (pud_none(*pud) || pud_page_vaddr(*pud) != pud_page_vaddr(*pud_ref))
 268                 BUG();
 269         pmd = pmd_offset(pud, address);
 270         pmd_ref = pmd_offset(pud_ref, address);
 271         if (pmd_none(*pmd_ref))
 272                 return -1;
 273         if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
 274                 BUG();
 275         pte_ref = pte_offset_kernel(pmd_ref, address);
 276         if (!pte_present(*pte_ref))
 277                 return -1;
 278         pte = pte_offset_kernel(pmd, address);
 279         /* Don't use pte_page here, because the mappings can point
 280            outside mem_map, and the NUMA hash lookup cannot handle
 281            that. */
 282         if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
 283                 BUG();
 284         return 0;
 285 }
 286
 287 int show_unhandled_signals = 1;
 288
 289 /*
 290  * This routine handles page faults.  It determines the address,
 291  * and the problem, and then passes it off to one of the appropriate
 292  * routines.
 293  */
 294 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
 295                                         unsigned long error_code)
 296 {
 297         struct task_struct *tsk;
 298         struct mm_struct *mm;
 299         struct vm_area_struct * vma;
 300         unsigned long address;
 301         int write, fault;
 302         unsigned long flags;
 303         siginfo_t info;
 304
 305         /*
 306          * We can fault from pretty much anywhere, with unknown IRQ state.
 307          */
 308         trace_hardirqs_fixup();
 309
 310         tsk = current;
 311         mm = tsk->mm;
 312         prefetchw(&mm->mmap_sem);
 313
 314         /* get the address */
 315         address = read_cr2();
 316
 317         info.si_code = SEGV_MAPERR;
 318
 319
 320         /*
 321          * We fault-in kernel-space virtual memory on-demand. The
 322          * 'reference' page table is init_mm.pgd.
 323          *
 324          * NOTE! We MUST NOT take any locks for this case. We may
 325          * be in an interrupt or a critical region, and should
 326          * only copy the information from the master page table,
 327          * nothing more.
 328          *
 329          * This verifies that the fault happens in kernel space
 330          * (error_code & 4) == 0, and that the fault was not a
 331          * protection error (error_code & 9) == 0.
 332          */
 333         if (unlikely(address >= TASK_SIZE64)) {
 334                 /*
 335                  * Don't check for the module range here: its PML4
 336                  * is always initialized because it's shared with the main
 337                  * kernel text. Only vmalloc may need PML4 syncups.
 338                  */
 339                 if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
 340                       ((address >= VMALLOC_START && address < VMALLOC_END))) {
 341                         if (vmalloc_fault(address) >= 0)
 342                                 return;
 343                 }
 344                 if (notify_page_fault(regs))
 345                         return;
 346                 /*
 347                  * Don't take the mm semaphore here. If we fixup a prefetch
 348                  * fault we could otherwise deadlock.
 349                  */
 350                 goto bad_area_nosemaphore;
 351         }
 352
 353         if (notify_page_fault(regs))
 354                 return;
 355
 356         if (likely(regs->flags & X86_EFLAGS_IF))
 357                 local_irq_enable();
 358
 359         if (unlikely(error_code & PF_RSVD))
 360                 pgtable_bad(address, regs, error_code);
 361
 362         /*
 363          * If we're in an interrupt or have no user
 364          * context, we must not take the fault..
 365          */
 366         if (unlikely(in_atomic() || !mm))
 367                 goto bad_area_nosemaphore;
 368
 369         /*
 370          * User-mode registers count as a user access even for any
 371          * potential system fault or CPU buglet.
 372          */
 373         if (user_mode_vm(regs))
 374                 error_code |= PF_USER;
 375
 376  again:
 377         /* When running in the kernel we expect faults to occur only to
 378          * addresses in user space.  All other faults represent errors in the
 379          * kernel and should generate an OOPS.  Unfortunately, in the case of an
 380          * erroneous fault occurring in a code path which already holds mmap_sem
 381          * we will deadlock attempting to validate the fault against the
 382          * address space.  Luckily the kernel only validly references user
 383          * space from well defined areas of code, which are listed in the
 384          * exceptions table.
 385          *
 386          * As the vast majority of faults will be valid we will only perform
 387          * the source reference check when there is a possibility of a deadlock.
 388          * Attempt to lock the address space, if we cannot we then validate the
 389          * source.  If this is invalid we can skip the address space check,
 390          * thus avoiding the deadlock.
 391          */
 392         if (!down_read_trylock(&mm->mmap_sem)) {
 393                 if ((error_code & PF_USER) == 0 &&
 394                     !search_exception_tables(regs->ip))
 395                         goto bad_area_nosemaphore;
 396                 down_read(&mm->mmap_sem);
 397         }
 398
 399         vma = find_vma(mm, address);
 400         if (!vma)
 401                 goto bad_area;
 402         if (likely(vma->vm_start <= address))
 403                 goto good_area;
 404         if (!(vma->vm_flags & VM_GROWSDOWN))
 405                 goto bad_area;
 406         if (error_code & 4) {
 407                 /* Allow userspace just enough access below the stack pointer
 408                  * to let the 'enter' instruction work.
 409                  */
 410                 if (address + 65536 + 32 * sizeof(unsigned long) < regs->sp)
 411                         goto bad_area;
 412         }
 413         if (expand_stack(vma, address))
 414                 goto bad_area;
 415 /*
 416  * Ok, we have a good vm_area for this memory access, so
 417  * we can handle it..
 418  */
 419 good_area:
 420         info.si_code = SEGV_ACCERR;
 421         write = 0;
 422         switch (error_code & (PF_PROT|PF_WRITE)) {
 423                 default:        /* 3: write, present */
 424                         /* fall through */
 425                 case PF_WRITE:          /* write, not present */
 426                         if (!(vma->vm_flags & VM_WRITE))
 427                                 goto bad_area;
 428                         write++;
 429                         break;
 430                 case PF_PROT:           /* read, present */
 431                         goto bad_area;
 432                 case 0:                 /* read, not present */
 433                         if (!(vma->vm_flags & (VM_READ | VM_EXEC | VM_WRITE)))
 434                                 goto bad_area;
 435         }
 436
 437         /*
 438          * If for any reason at all we couldn't handle the fault,
 439          * make sure we exit gracefully rather than endlessly redo
 440          * the fault.
 441          */
 442         fault = handle_mm_fault(mm, vma, address, write);
 443         if (unlikely(fault & VM_FAULT_ERROR)) {
 444                 if (fault & VM_FAULT_OOM)
 445                         goto out_of_memory;
 446                 else if (fault & VM_FAULT_SIGBUS)
 447                         goto do_sigbus;
 448                 BUG();
 449         }
 450         if (fault & VM_FAULT_MAJOR)
 451                 tsk->maj_flt++;
 452         else
 453                 tsk->min_flt++;
 454         up_read(&mm->mmap_sem);
 455         return;
 456
 457 /*
 458  * Something tried to access memory that isn't in our memory map..
 459  * Fix it, but check if it's kernel or user first..
 460  */
 461 bad_area:
 462         up_read(&mm->mmap_sem);
 463
 464 bad_area_nosemaphore:
 465         /* User mode accesses just cause a SIGSEGV */
 466         if (error_code & PF_USER) {
 467
 468                 /*
 469                  * It's possible to have interrupts off here.
 470                  */
 471                 local_irq_enable();
 472
 473                 if (is_prefetch(regs, address, error_code))
 474                         return;
 475
 476                 /* Work around K8 erratum #100 K8 in compat mode
 477                    occasionally jumps to illegal addresses >4GB.  We
 478                    catch this here in the page fault handler because
 479                    these addresses are not reachable. Just detect this
 480                    case and return.  Any code segment in LDT is
 481                    compatibility mode. */
 482                 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
 483                     (address >> 32))
 484                         return;
 485
 486                 if (show_unhandled_signals && unhandled_signal(tsk, SIGSEGV) &&
 487                     printk_ratelimit()) {
 488                         printk(
 489                        "%s%s[%d]: segfault at %lx ip %lx sp %lx error %lx\n",
 490                                         tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
 491                                         tsk->comm, tsk->pid, address, regs->ip,
 492                                         regs->sp, error_code);
 493                 }
 494
 495                 tsk->thread.cr2 = address;
 496                 /* Kernel addresses are always protection faults */
 497                 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
 498                 tsk->thread.trap_no = 14;
 499                 info.si_signo = SIGSEGV;
 500                 info.si_errno = 0;
 501                 /* info.si_code has been set above */
 502                 info.si_addr = (void __user *)address;
 503                 force_sig_info(SIGSEGV, &info, tsk);
 504                 return;
 505         }
 506
 507 no_context:
 508
 509         /* Are we prepared to handle this kernel fault?  */
 510         if (fixup_exception(regs)) {
 511                 return;
 512         }
 513
 514         /*
 515          * Hall of shame of CPU/BIOS bugs.
 516          */
 517
 518         if (is_prefetch(regs, address, error_code))
 519                 return;
 520
 521         if (is_errata93(regs, address))
 522                 return;
 523
 524 /*
 525  * Oops. The kernel tried to access some bad page. We'll have to
 526  * terminate things with extreme prejudice.
 527  */
 528
 529         flags = oops_begin();
 530
 531         if (address < PAGE_SIZE)
 532                 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 533         else
 534                 printk(KERN_ALERT "Unable to handle kernel paging request");
 535         printk(" at %016lx RIP: \n" KERN_ALERT,address);
 536         printk_address(regs->ip);
 537         dump_pagetable(address);
 538         tsk->thread.cr2 = address;
 539         tsk->thread.trap_no = 14;
 540         tsk->thread.error_code = error_code;
 541         if (__die("Oops", regs, error_code))
 542                 regs = NULL;
 543         /* Executive summary in case the body of the oops scrolled away */
 544         printk(KERN_EMERG "CR2: %016lx\n", address);
 545         oops_end(flags, regs, SIGKILL);
 546
 547 /*
 548  * We ran out of memory, or some other thing happened to us that made
 549  * us unable to handle the page fault gracefully.
 550  */
 551 out_of_memory:
 552         up_read(&mm->mmap_sem);
 553         if (is_global_init(current)) {
 554                 yield();
 555                 goto again;
 556         }
 557         printk("VM: killing process %s\n", tsk->comm);
 558         if (error_code & 4)
 559                 do_group_exit(SIGKILL);
 560         goto no_context;
 561
 562 do_sigbus:
 563         up_read(&mm->mmap_sem);
 564
 565         /* Kernel mode? Handle exceptions or die */
 566         if (!(error_code & PF_USER))
 567                 goto no_context;
 568
 569         tsk->thread.cr2 = address;
 570         tsk->thread.error_code = error_code;
 571         tsk->thread.trap_no = 14;
 572         info.si_signo = SIGBUS;
 573         info.si_errno = 0;
 574         info.si_code = BUS_ADRERR;
 575         info.si_addr = (void __user *)address;
 576         force_sig_info(SIGBUS, &info, tsk);
 577         return;
 578 }
 579
 580 DEFINE_SPINLOCK(pgd_lock);
 581 LIST_HEAD(pgd_list);
 582
 583 void vmalloc_sync_all(void)
 584 {
 585         /* Note that races in the updates of insync and start aren't
 586            problematic:
 587            insync can only get set bits added, and updates to start are only
 588            improving performance (without affecting correctness if undone). */
 589         static DECLARE_BITMAP(insync, PTRS_PER_PGD);
 590         static unsigned long start = VMALLOC_START & PGDIR_MASK;
 591         unsigned long address;
 592
 593         for (address = start; address <= VMALLOC_END; address += PGDIR_SIZE) {
 594                 if (!test_bit(pgd_index(address), insync)) {
 595                         const pgd_t *pgd_ref = pgd_offset_k(address);
 596                         struct page *page;
 597
 598                         if (pgd_none(*pgd_ref))
 599                                 continue;
 600                         spin_lock(&pgd_lock);
 601                         list_for_each_entry(page, &pgd_list, lru) {
 602                                 pgd_t *pgd;
 603                                 pgd = (pgd_t *)page_address(page) + pgd_index(address);
 604                                 if (pgd_none(*pgd))
 605                                         set_pgd(pgd, *pgd_ref);
 606                                 else
 607                                         BUG_ON(pgd_page_vaddr(*pgd) != pgd_page_vaddr(*pgd_ref));
 608                         }
 609                         spin_unlock(&pgd_lock);
 610                         set_bit(pgd_index(address), insync);
 611                 }
 612                 if (address == start)
 613                         start = address + PGDIR_SIZE;
 614         }
 615         /* Check that there is no need to do the same for the modules area. */
 616         BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
 617         BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
 618                                 (__START_KERNEL & PGDIR_MASK)));
 619 }