arch/x86_64/mm/fault.c

   1 /*
   2  *  linux/arch/x86-64/mm/fault.c
   3  *
   4  *  Copyright (C) 1995  Linus Torvalds
   5  *  Copyright (C) 2001,2002 Andi Kleen, SuSE Labs.
   6  */
   7
   8 #include <linux/config.h>
   9 #include <linux/signal.h>
  10 #include <linux/sched.h>
  11 #include <linux/kernel.h>
  12 #include <linux/errno.h>
  13 #include <linux/string.h>
  14 #include <linux/types.h>
  15 #include <linux/ptrace.h>
  16 #include <linux/mman.h>
  17 #include <linux/mm.h>
  18 #include <linux/smp.h>
  19 #include <linux/smp_lock.h>
  20 #include <linux/interrupt.h>
  21 #include <linux/init.h>
  22 #include <linux/tty.h>
  23 #include <linux/vt_kern.h>              /* For unblank_screen() */
  24 #include <linux/compiler.h>
  25 #include <linux/module.h>
  26 #include <linux/kprobes.h>
  27
  28 #include <asm/system.h>
  29 #include <asm/uaccess.h>
  30 #include <asm/pgalloc.h>
  31 #include <asm/smp.h>
  32 #include <asm/tlbflush.h>
  33 #include <asm/proto.h>
  34 #include <asm/kdebug.h>
  35 #include <asm-generic/sections.h>
  36
  37 /* Page fault error code bits */
  38 #define PF_PROT (1<<0)          /* or no page found */
  39 #define PF_WRITE        (1<<1)
  40 #define PF_USER (1<<2)
  41 #define PF_RSVD (1<<3)
  42 #define PF_INSTR        (1<<4)
  43
  44 void bust_spinlocks(int yes)
  45 {
  46         int loglevel_save = console_loglevel;
  47         if (yes) {
  48                 oops_in_progress = 1;
  49         } else {
  50 #ifdef CONFIG_VT
  51                 unblank_screen();
  52 #endif
  53                 oops_in_progress = 0;
  54                 /*
  55                  * OK, the message is on the console.  Now we call printk()
  56                  * without oops_in_progress set so that printk will give klogd
  57                  * a poke.  Hold onto your hats...
  58                  */
  59                 console_loglevel = 15;          /* NMI oopser may have shut the console up */
  60                 printk(" ");
  61                 console_loglevel = loglevel_save;
  62         }
  63 }
  64
  65 /* Sometimes the CPU reports invalid exceptions on prefetch.
  66    Check that here and ignore.
  67    Opcode checker based on code by Richard Brunner */
  68 static noinline int is_prefetch(struct pt_regs *regs, unsigned long addr,
  69                                 unsigned long error_code)
  70 {
  71         unsigned char *instr;
  72         int scan_more = 1;
  73         int prefetch = 0;
  74         unsigned char *max_instr;
  75
  76         /* If it was a exec fault ignore */
  77         if (error_code & PF_INSTR)
  78                 return 0;
  79
  80         instr = (unsigned char *)convert_rip_to_linear(current, regs);
  81         max_instr = instr + 15;
  82
  83         if (user_mode(regs) && instr >= (unsigned char *)TASK_SIZE)
  84                 return 0;
  85
  86         while (scan_more && instr < max_instr) {
  87                 unsigned char opcode;
  88                 unsigned char instr_hi;
  89                 unsigned char instr_lo;
  90
  91                 if (__get_user(opcode, instr))
  92                         break;
  93
  94                 instr_hi = opcode & 0xf0;
  95                 instr_lo = opcode & 0x0f;
  96                 instr++;
  97
  98                 switch (instr_hi) {
  99                 case 0x20:
 100                 case 0x30:
 101                         /* Values 0x26,0x2E,0x36,0x3E are valid x86
 102                            prefixes.  In long mode, the CPU will signal
 103                            invalid opcode if some of these prefixes are
 104                            present so we will never get here anyway */
 105                         scan_more = ((instr_lo & 7) == 0x6);
 106                         break;
 107
 108                 case 0x40:
 109                         /* In AMD64 long mode, 0x40 to 0x4F are valid REX prefixes
 110                            Need to figure out under what instruction mode the
 111                            instruction was issued ... */
 112                         /* Could check the LDT for lm, but for now it's good
 113                            enough to assume that long mode only uses well known
 114                            segments or kernel. */
 115                         scan_more = (!user_mode(regs)) || (regs->cs == __USER_CS);
 116                         break;
 117
 118                 case 0x60:
 119                         /* 0x64 thru 0x67 are valid prefixes in all modes. */
 120                         scan_more = (instr_lo & 0xC) == 0x4;
 121                         break;
 122                 case 0xF0:
 123                         /* 0xF0, 0xF2, and 0xF3 are valid prefixes in all modes. */
 124                         scan_more = !instr_lo || (instr_lo>>1) == 1;
 125                         break;
 126                 case 0x00:
 127                         /* Prefetch instruction is 0x0F0D or 0x0F18 */
 128                         scan_more = 0;
 129                         if (__get_user(opcode, instr))
 130                                 break;
 131                         prefetch = (instr_lo == 0xF) &&
 132                                 (opcode == 0x0D || opcode == 0x18);
 133                         break;
 134                 default:
 135                         scan_more = 0;
 136                         break;
 137                 }
 138         }
 139         return prefetch;
 140 }
 141
 142 static int bad_address(void *p)
 143 {
 144         unsigned long dummy;
 145         return __get_user(dummy, (unsigned long *)p);
 146 }
 147
 148 void dump_pagetable(unsigned long address)
 149 {
 150         pgd_t *pgd;
 151         pud_t *pud;
 152         pmd_t *pmd;
 153         pte_t *pte;
 154
 155         asm("movq %%cr3,%0" : "=r" (pgd));
 156
 157         pgd = __va((unsigned long)pgd & PHYSICAL_PAGE_MASK);
 158         pgd += pgd_index(address);
 159         if (bad_address(pgd)) goto bad;
 160         printk("PGD %lx ", pgd_val(*pgd));
 161         if (!pgd_present(*pgd)) goto ret;
 162
 163         pud = __pud_offset_k((pud_t *)pgd_page(*pgd), address);
 164         if (bad_address(pud)) goto bad;
 165         printk("PUD %lx ", pud_val(*pud));
 166         if (!pud_present(*pud)) goto ret;
 167
 168         pmd = pmd_offset(pud, address);
 169         if (bad_address(pmd)) goto bad;
 170         printk("PMD %lx ", pmd_val(*pmd));
 171         if (!pmd_present(*pmd)) goto ret;
 172
 173         pte = pte_offset_kernel(pmd, address);
 174         if (bad_address(pte)) goto bad;
 175         printk("PTE %lx", pte_val(*pte));
 176 ret:
 177         printk("\n");
 178         return;
 179 bad:
 180         printk("BAD\n");
 181 }
 182
 183 static const char errata93_warning[] =
 184 KERN_ERR "******* Your BIOS seems to not contain a fix for K8 errata #93\n"
 185 KERN_ERR "******* Working around it, but it may cause SEGVs or burn power.\n"
 186 KERN_ERR "******* Please consider a BIOS update.\n"
 187 KERN_ERR "******* Disabling USB legacy in the BIOS may also help.\n";
 188
 189 /* Workaround for K8 erratum #93 & buggy BIOS.
 190    BIOS SMM functions are required to use a specific workaround
 191    to avoid corruption of the 64bit RIP register on C stepping K8.
 192    A lot of BIOS that didn't get tested properly miss this.
 193    The OS sees this as a page fault with the upper 32bits of RIP cleared.
 194    Try to work around it here.
 195    Note we only handle faults in kernel here. */
 196
 197 static int is_errata93(struct pt_regs *regs, unsigned long address)
 198 {
 199         static int warned;
 200         if (address != regs->rip)
 201                 return 0;
 202         if ((address >> 32) != 0)
 203                 return 0;
 204         address |= 0xffffffffUL << 32;
 205         if ((address >= (u64)_stext && address <= (u64)_etext) ||
 206             (address >= MODULES_VADDR && address <= MODULES_END)) {
 207                 if (!warned) {
 208                         printk(errata93_warning);
 209                         warned = 1;
 210                 }
 211                 regs->rip = address;
 212                 return 1;
 213         }
 214         return 0;
 215 }
 216
 217 int unhandled_signal(struct task_struct *tsk, int sig)
 218 {
 219         if (tsk->pid == 1)
 220                 return 1;
 221         if (tsk->ptrace & PT_PTRACED)
 222                 return 0;
 223         return (tsk->sighand->action[sig-1].sa.sa_handler == SIG_IGN) ||
 224                 (tsk->sighand->action[sig-1].sa.sa_handler == SIG_DFL);
 225 }
 226
 227 static noinline void pgtable_bad(unsigned long address, struct pt_regs *regs,
 228                                  unsigned long error_code)
 229 {
 230         unsigned long flags = oops_begin();
 231         struct task_struct *tsk;
 232
 233         printk(KERN_ALERT "%s: Corrupted page table at address %lx\n",
 234                current->comm, address);
 235         dump_pagetable(address);
 236         tsk = current;
 237         tsk->thread.cr2 = address;
 238         tsk->thread.trap_no = 14;
 239         tsk->thread.error_code = error_code;
 240         __die("Bad pagetable", regs, error_code);
 241         oops_end(flags);
 242         do_exit(SIGKILL);
 243 }
 244
 245 /*
 246  * Handle a fault on the vmalloc area
 247  *
 248  * This assumes no large pages in there.
 249  */
 250 static int vmalloc_fault(unsigned long address)
 251 {
 252         pgd_t *pgd, *pgd_ref;
 253         pud_t *pud, *pud_ref;
 254         pmd_t *pmd, *pmd_ref;
 255         pte_t *pte, *pte_ref;
 256
 257         /* Copy kernel mappings over when needed. This can also
 258            happen within a race in page table update. In the later
 259            case just flush. */
 260
 261         pgd = pgd_offset(current->mm ?: &init_mm, address);
 262         pgd_ref = pgd_offset_k(address);
 263         if (pgd_none(*pgd_ref))
 264                 return -1;
 265         if (pgd_none(*pgd))
 266                 set_pgd(pgd, *pgd_ref);
 267         else
 268                 BUG_ON(pgd_page(*pgd) != pgd_page(*pgd_ref));
 269
 270         /* Below here mismatches are bugs because these lower tables
 271            are shared */
 272
 273         pud = pud_offset(pgd, address);
 274         pud_ref = pud_offset(pgd_ref, address);
 275         if (pud_none(*pud_ref))
 276                 return -1;
 277         if (pud_none(*pud) || pud_page(*pud) != pud_page(*pud_ref))
 278                 BUG();
 279         pmd = pmd_offset(pud, address);
 280         pmd_ref = pmd_offset(pud_ref, address);
 281         if (pmd_none(*pmd_ref))
 282                 return -1;
 283         if (pmd_none(*pmd) || pmd_page(*pmd) != pmd_page(*pmd_ref))
 284                 BUG();
 285         pte_ref = pte_offset_kernel(pmd_ref, address);
 286         if (!pte_present(*pte_ref))
 287                 return -1;
 288         pte = pte_offset_kernel(pmd, address);
 289         /* Don't use pte_page here, because the mappings can point
 290            outside mem_map, and the NUMA hash lookup cannot handle
 291            that. */
 292         if (!pte_present(*pte) || pte_pfn(*pte) != pte_pfn(*pte_ref))
 293                 BUG();
 294         return 0;
 295 }
 296
 297 int page_fault_trace = 0;
 298 int exception_trace = 1;
 299
 300 /*
 301  * This routine handles page faults.  It determines the address,
 302  * and the problem, and then passes it off to one of the appropriate
 303  * routines.
 304  */
 305 asmlinkage void __kprobes do_page_fault(struct pt_regs *regs,
 306                                         unsigned long error_code)
 307 {
 308         struct task_struct *tsk;
 309         struct mm_struct *mm;
 310         struct vm_area_struct * vma;
 311         unsigned long address;
 312         const struct exception_table_entry *fixup;
 313         int write;
 314         unsigned long flags;
 315         siginfo_t info;
 316
 317         /* get the address */
 318         __asm__("movq %%cr2,%0":"=r" (address));
 319
 320         tsk = current;
 321         mm = tsk->mm;
 322         info.si_code = SEGV_MAPERR;
 323
 324
 325         /*
 326          * We fault-in kernel-space virtual memory on-demand. The
 327          * 'reference' page table is init_mm.pgd.
 328          *
 329          * NOTE! We MUST NOT take any locks for this case. We may
 330          * be in an interrupt or a critical region, and should
 331          * only copy the information from the master page table,
 332          * nothing more.
 333          *
 334          * This verifies that the fault happens in kernel space
 335          * (error_code & 4) == 0, and that the fault was not a
 336          * protection error (error_code & 9) == 0.
 337          */
 338         if (unlikely(address >= TASK_SIZE64)) {
 339                 /*
 340                  * Don't check for the module range here: its PML4
 341                  * is always initialized because it's shared with the main
 342                  * kernel text. Only vmalloc may need PML4 syncups.
 343                  */
 344                 if (!(error_code & (PF_RSVD|PF_USER|PF_PROT)) &&
 345                       ((address >= VMALLOC_START && address < VMALLOC_END))) {
 346                         if (vmalloc_fault(address) >= 0)
 347                                 return;
 348                 }
 349                 if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
 350                                                 SIGSEGV) == NOTIFY_STOP)
 351                         return;
 352                 /*
 353                  * Don't take the mm semaphore here. If we fixup a prefetch
 354                  * fault we could otherwise deadlock.
 355                  */
 356                 goto bad_area_nosemaphore;
 357         }
 358
 359         if (notify_die(DIE_PAGE_FAULT, "page fault", regs, error_code, 14,
 360                                         SIGSEGV) == NOTIFY_STOP)
 361                 return;
 362
 363         if (likely(regs->eflags & X86_EFLAGS_IF))
 364                 local_irq_enable();
 365
 366         if (unlikely(page_fault_trace))
 367                 printk("pagefault rip:%lx rsp:%lx cs:%lu ss:%lu address %lx error %lx\n",
 368                        regs->rip,regs->rsp,regs->cs,regs->ss,address,error_code);
 369
 370         if (unlikely(error_code & PF_RSVD))
 371                 pgtable_bad(address, regs, error_code);
 372
 373         /*
 374          * If we're in an interrupt or have no user
 375          * context, we must not take the fault..
 376          */
 377         if (unlikely(in_atomic() || !mm))
 378                 goto bad_area_nosemaphore;
 379
 380  again:
 381         /* When running in the kernel we expect faults to occur only to
 382          * addresses in user space.  All other faults represent errors in the
 383          * kernel and should generate an OOPS.  Unfortunatly, in the case of an
 384          * erroneous fault occuring in a code path which already holds mmap_sem
 385          * we will deadlock attempting to validate the fault against the
 386          * address space.  Luckily the kernel only validly references user
 387          * space from well defined areas of code, which are listed in the
 388          * exceptions table.
 389          *
 390          * As the vast majority of faults will be valid we will only perform
 391          * the source reference check when there is a possibilty of a deadlock.
 392          * Attempt to lock the address space, if we cannot we then validate the
 393          * source.  If this is invalid we can skip the address space check,
 394          * thus avoiding the deadlock.
 395          */
 396         if (!down_read_trylock(&mm->mmap_sem)) {
 397                 if ((error_code & PF_USER) == 0 &&
 398                     !search_exception_tables(regs->rip))
 399                         goto bad_area_nosemaphore;
 400                 down_read(&mm->mmap_sem);
 401         }
 402
 403         vma = find_vma(mm, address);
 404         if (!vma)
 405                 goto bad_area;
 406         if (likely(vma->vm_start <= address))
 407                 goto good_area;
 408         if (!(vma->vm_flags & VM_GROWSDOWN))
 409                 goto bad_area;
 410         if (error_code & 4) {
 411                 // XXX: align red zone size with ABI
 412                 if (address + 128 < regs->rsp)
 413                         goto bad_area;
 414         }
 415         if (expand_stack(vma, address))
 416                 goto bad_area;
 417 /*
 418  * Ok, we have a good vm_area for this memory access, so
 419  * we can handle it..
 420  */
 421 good_area:
 422         info.si_code = SEGV_ACCERR;
 423         write = 0;
 424         switch (error_code & (PF_PROT|PF_WRITE)) {
 425                 default:        /* 3: write, present */
 426                         /* fall through */
 427                 case PF_WRITE:          /* write, not present */
 428                         if (!(vma->vm_flags & VM_WRITE))
 429                                 goto bad_area;
 430                         write++;
 431                         break;
 432                 case PF_PROT:           /* read, present */
 433                         goto bad_area;
 434                 case 0:                 /* read, not present */
 435                         if (!(vma->vm_flags & (VM_READ | VM_EXEC)))
 436                                 goto bad_area;
 437         }
 438
 439         /*
 440          * If for any reason at all we couldn't handle the fault,
 441          * make sure we exit gracefully rather than endlessly redo
 442          * the fault.
 443          */
 444         switch (handle_mm_fault(mm, vma, address, write)) {
 445         case VM_FAULT_MINOR:
 446                 tsk->min_flt++;
 447                 break;
 448         case VM_FAULT_MAJOR:
 449                 tsk->maj_flt++;
 450                 break;
 451         case VM_FAULT_SIGBUS:
 452                 goto do_sigbus;
 453         default:
 454                 goto out_of_memory;
 455         }
 456
 457         up_read(&mm->mmap_sem);
 458         return;
 459
 460 /*
 461  * Something tried to access memory that isn't in our memory map..
 462  * Fix it, but check if it's kernel or user first..
 463  */
 464 bad_area:
 465         up_read(&mm->mmap_sem);
 466
 467 bad_area_nosemaphore:
 468         /* User mode accesses just cause a SIGSEGV */
 469         if (error_code & PF_USER) {
 470                 if (is_prefetch(regs, address, error_code))
 471                         return;
 472
 473                 /* Work around K8 erratum #100 K8 in compat mode
 474                    occasionally jumps to illegal addresses >4GB.  We
 475                    catch this here in the page fault handler because
 476                    these addresses are not reachable. Just detect this
 477                    case and return.  Any code segment in LDT is
 478                    compatibility mode. */
 479                 if ((regs->cs == __USER32_CS || (regs->cs & (1<<2))) &&
 480                     (address >> 32))
 481                         return;
 482
 483                 if (exception_trace && unhandled_signal(tsk, SIGSEGV)) {
 484                         printk(
 485                        "%s%s[%d]: segfault at %016lx rip %016lx rsp %016lx error %lx\n",
 486                                         tsk->pid > 1 ? KERN_INFO : KERN_EMERG,
 487                                         tsk->comm, tsk->pid, address, regs->rip,
 488                                         regs->rsp, error_code);
 489                 }
 490
 491                 tsk->thread.cr2 = address;
 492                 /* Kernel addresses are always protection faults */
 493                 tsk->thread.error_code = error_code | (address >= TASK_SIZE);
 494                 tsk->thread.trap_no = 14;
 495                 info.si_signo = SIGSEGV;
 496                 info.si_errno = 0;
 497                 /* info.si_code has been set above */
 498                 info.si_addr = (void __user *)address;
 499                 force_sig_info(SIGSEGV, &info, tsk);
 500                 return;
 501         }
 502
 503 no_context:
 504
 505         /* Are we prepared to handle this kernel fault?  */
 506         fixup = search_exception_tables(regs->rip);
 507         if (fixup) {
 508                 regs->rip = fixup->fixup;
 509                 return;
 510         }
 511
 512         /*
 513          * Hall of shame of CPU/BIOS bugs.
 514          */
 515
 516         if (is_prefetch(regs, address, error_code))
 517                 return;
 518
 519         if (is_errata93(regs, address))
 520                 return;
 521
 522 /*
 523  * Oops. The kernel tried to access some bad page. We'll have to
 524  * terminate things with extreme prejudice.
 525  */
 526
 527         flags = oops_begin();
 528
 529         if (address < PAGE_SIZE)
 530                 printk(KERN_ALERT "Unable to handle kernel NULL pointer dereference");
 531         else
 532                 printk(KERN_ALERT "Unable to handle kernel paging request");
 533         printk(" at %016lx RIP: \n" KERN_ALERT,address);
 534         printk_address(regs->rip);
 535         printk("\n");
 536         dump_pagetable(address);
 537         tsk->thread.cr2 = address;
 538         tsk->thread.trap_no = 14;
 539         tsk->thread.error_code = error_code;
 540         __die("Oops", regs, error_code);
 541         /* Executive summary in case the body of the oops scrolled away */
 542         printk(KERN_EMERG "CR2: %016lx\n", address);
 543         oops_end(flags);
 544         do_exit(SIGKILL);
 545
 546 /*
 547  * We ran out of memory, or some other thing happened to us that made
 548  * us unable to handle the page fault gracefully.
 549  */
 550 out_of_memory:
 551         up_read(&mm->mmap_sem);
 552         if (current->pid == 1) {
 553                 yield();
 554                 goto again;
 555         }
 556         printk("VM: killing process %s\n", tsk->comm);
 557         if (error_code & 4)
 558                 do_exit(SIGKILL);
 559         goto no_context;
 560
 561 do_sigbus:
 562         up_read(&mm->mmap_sem);
 563
 564         /* Kernel mode? Handle exceptions or die */
 565         if (!(error_code & PF_USER))
 566                 goto no_context;
 567
 568         tsk->thread.cr2 = address;
 569         tsk->thread.error_code = error_code;
 570         tsk->thread.trap_no = 14;
 571         info.si_signo = SIGBUS;
 572         info.si_errno = 0;
 573         info.si_code = BUS_ADRERR;
 574         info.si_addr = (void __user *)address;
 575         force_sig_info(SIGBUS, &info, tsk);
 576         return;
 577 }
 578
 579 DEFINE_SPINLOCK(pgd_lock);
 580 struct page *pgd_list;
 581
 582 void vmalloc_sync_all(void)
 583 {
 584         /* Note that races in the updates of insync and start aren't
 585            problematic:
 586            insync can only get set bits added, and updates to start are only
 587            improving performance (without affecting correctness if undone). */
 588         static DECLARE_BITMAP(insync, PTRS_PER_PGD);
 589         static unsigned long start = VMALLOC_START & PGDIR_MASK;
 590         unsigned long address;
 591
 592         for (address = start; address <= VMALLOC_END; address += PGDIR_SIZE) {
 593                 if (!test_bit(pgd_index(address), insync)) {
 594                         const pgd_t *pgd_ref = pgd_offset_k(address);
 595                         struct page *page;
 596
 597                         if (pgd_none(*pgd_ref))
 598                                 continue;
 599                         spin_lock(&pgd_lock);
 600                         for (page = pgd_list; page;
 601                              page = (struct page *)page->index) {
 602                                 pgd_t *pgd;
 603                                 pgd = (pgd_t *)page_address(page) + pgd_index(address);
 604                                 if (pgd_none(*pgd))
 605                                         set_pgd(pgd, *pgd_ref);
 606                                 else
 607                                         BUG_ON(pgd_page(*pgd) != pgd_page(*pgd_ref));
 608                         }
 609                         spin_unlock(&pgd_lock);
 610                         set_bit(pgd_index(address), insync);
 611                 }
 612                 if (address == start)
 613                         start = address + PGDIR_SIZE;
 614         }
 615         /* Check that there is no need to do the same for the modules area. */
 616         BUILD_BUG_ON(!(MODULES_VADDR > __START_KERNEL));
 617         BUILD_BUG_ON(!(((MODULES_END - 1) & PGDIR_MASK) ==
 618                                 (__START_KERNEL & PGDIR_MASK)));
 619 }
 620
 621 static int __init enable_pagefaulttrace(char *str)
 622 {
 623         page_fault_trace = 1;
 624         return 0;
 625 }
 626 __setup("pagefaulttrace", enable_pagefaulttrace);