arch/x86/mm/pgtable.c

   1 #include <linux/mm.h>
   2 #include <linux/gfp.h>
   3 #include <asm/pgalloc.h>
   4 #include <asm/pgtable.h>
   5 #include <asm/tlb.h>
   6 #include <asm/fixmap.h>
   7
   8 #define PGALLOC_GFP GFP_KERNEL | __GFP_NOTRACK | __GFP_REPEAT | __GFP_ZERO
   9
  10 #ifdef CONFIG_HIGHPTE
  11 #define PGALLOC_USER_GFP __GFP_HIGHMEM
  12 #else
  13 #define PGALLOC_USER_GFP 0
  14 #endif
  15
  16 gfp_t __userpte_alloc_gfp = PGALLOC_GFP | PGALLOC_USER_GFP;
  17
  18 pte_t *pte_alloc_one_kernel(struct mm_struct *mm, unsigned long address)
  19 {
  20         return (pte_t *)__get_free_page(PGALLOC_GFP);
  21 }
  22
  23 pgtable_t pte_alloc_one(struct mm_struct *mm, unsigned long address)
  24 {
  25         struct page *pte;
  26
  27         pte = alloc_pages(__userpte_alloc_gfp, 0);
  28         if (!pte)
  29                 return NULL;
  30         if (!pgtable_page_ctor(pte)) {
  31                 __free_page(pte);
  32                 return NULL;
  33         }
  34         return pte;
  35 }
  36
  37 static int __init setup_userpte(char *arg)
  38 {
  39         if (!arg)
  40                 return -EINVAL;
  41
  42         /*
  43          * "userpte=nohigh" disables allocation of user pagetables in
  44          * high memory.
  45          */
  46         if (strcmp(arg, "nohigh") == 0)
  47                 __userpte_alloc_gfp &= ~__GFP_HIGHMEM;
  48         else
  49                 return -EINVAL;
  50         return 0;
  51 }
  52 early_param("userpte", setup_userpte);
  53
  54 void ___pte_free_tlb(struct mmu_gather *tlb, struct page *pte)
  55 {
  56         pgtable_page_dtor(pte);
  57         paravirt_release_pte(page_to_pfn(pte));
  58         tlb_remove_page(tlb, pte);
  59 }
  60
  61 #if PAGETABLE_LEVELS > 2
  62 void ___pmd_free_tlb(struct mmu_gather *tlb, pmd_t *pmd)
  63 {
  64         paravirt_release_pmd(__pa(pmd) >> PAGE_SHIFT);
  65         /*
  66          * NOTE! For PAE, any changes to the top page-directory-pointer-table
  67          * entries need a full cr3 reload to flush.
  68          */
  69 #ifdef CONFIG_X86_PAE
  70         tlb->need_flush_all = 1;
  71 #endif
  72         tlb_remove_page(tlb, virt_to_page(pmd));
  73 }
  74
  75 #if PAGETABLE_LEVELS > 3
  76 void ___pud_free_tlb(struct mmu_gather *tlb, pud_t *pud)
  77 {
  78         paravirt_release_pud(__pa(pud) >> PAGE_SHIFT);
  79         tlb_remove_page(tlb, virt_to_page(pud));
  80 }
  81 #endif  /* PAGETABLE_LEVELS > 3 */
  82 #endif  /* PAGETABLE_LEVELS > 2 */
  83
  84 static inline void pgd_list_add(pgd_t *pgd)
  85 {
  86         struct page *page = virt_to_page(pgd);
  87
  88         list_add(&page->lru, &pgd_list);
  89 }
  90
  91 static inline void pgd_list_del(pgd_t *pgd)
  92 {
  93         struct page *page = virt_to_page(pgd);
  94
  95         list_del(&page->lru);
  96 }
  97
  98 #define UNSHARED_PTRS_PER_PGD                           \
  99         (SHARED_KERNEL_PMD ? KERNEL_PGD_BOUNDARY : PTRS_PER_PGD)
 100
 101
 102 static void pgd_set_mm(pgd_t *pgd, struct mm_struct *mm)
 103 {
 104         BUILD_BUG_ON(sizeof(virt_to_page(pgd)->index) < sizeof(mm));
 105         virt_to_page(pgd)->index = (pgoff_t)mm;
 106 }
 107
 108 struct mm_struct *pgd_page_get_mm(struct page *page)
 109 {
 110         return (struct mm_struct *)page->index;
 111 }
 112
 113 static void pgd_ctor(struct mm_struct *mm, pgd_t *pgd)
 114 {
 115         /* If the pgd points to a shared pagetable level (either the
 116            ptes in non-PAE, or shared PMD in PAE), then just copy the
 117            references from swapper_pg_dir. */
 118         if (PAGETABLE_LEVELS == 2 ||
 119             (PAGETABLE_LEVELS == 3 && SHARED_KERNEL_PMD) ||
 120             PAGETABLE_LEVELS == 4) {
 121                 clone_pgd_range(pgd + KERNEL_PGD_BOUNDARY,
 122                                 swapper_pg_dir + KERNEL_PGD_BOUNDARY,
 123                                 KERNEL_PGD_PTRS);
 124         }
 125
 126         /* list required to sync kernel mapping updates */
 127         if (!SHARED_KERNEL_PMD) {
 128                 pgd_set_mm(pgd, mm);
 129                 pgd_list_add(pgd);
 130         }
 131 }
 132
 133 static void pgd_dtor(pgd_t *pgd)
 134 {
 135         if (SHARED_KERNEL_PMD)
 136                 return;
 137
 138         spin_lock(&pgd_lock);
 139         pgd_list_del(pgd);
 140         spin_unlock(&pgd_lock);
 141 }
 142
 143 /*
 144  * List of all pgd's needed for non-PAE so it can invalidate entries
 145  * in both cached and uncached pgd's; not needed for PAE since the
 146  * kernel pmd is shared. If PAE were not to share the pmd a similar
 147  * tactic would be needed. This is essentially codepath-based locking
 148  * against pageattr.c; it is the unique case in which a valid change
 149  * of kernel pagetables can't be lazily synchronized by vmalloc faults.
 150  * vmalloc faults work because attached pagetables are never freed.
 151  * -- nyc
 152  */
 153
 154 #ifdef CONFIG_X86_PAE
 155 /*
 156  * In PAE mode, we need to do a cr3 reload (=tlb flush) when
 157  * updating the top-level pagetable entries to guarantee the
 158  * processor notices the update.  Since this is expensive, and
 159  * all 4 top-level entries are used almost immediately in a
 160  * new process's life, we just pre-populate them here.
 161  *
 162  * Also, if we're in a paravirt environment where the kernel pmd is
 163  * not shared between pagetables (!SHARED_KERNEL_PMDS), we allocate
 164  * and initialize the kernel pmds here.
 165  */
 166 #define PREALLOCATED_PMDS       UNSHARED_PTRS_PER_PGD
 167
 168 void pud_populate(struct mm_struct *mm, pud_t *pudp, pmd_t *pmd)
 169 {
 170         paravirt_alloc_pmd(mm, __pa(pmd) >> PAGE_SHIFT);
 171
 172         /* Note: almost everything apart from _PAGE_PRESENT is
 173            reserved at the pmd (PDPT) level. */
 174         set_pud(pudp, __pud(__pa(pmd) | _PAGE_PRESENT));
 175
 176         /*
 177          * According to Intel App note "TLBs, Paging-Structure Caches,
 178          * and Their Invalidation", April 2007, document 317080-001,
 179          * section 8.1: in PAE mode we explicitly have to flush the
 180          * TLB via cr3 if the top-level pgd is changed...
 181          */
 182         flush_tlb_mm(mm);
 183 }
 184 #else  /* !CONFIG_X86_PAE */
 185
 186 /* No need to prepopulate any pagetable entries in non-PAE modes. */
 187 #define PREALLOCATED_PMDS       0
 188
 189 #endif  /* CONFIG_X86_PAE */
 190
 191 static void free_pmds(pmd_t *pmds[])
 192 {
 193         int i;
 194
 195         for(i = 0; i < PREALLOCATED_PMDS; i++)
 196                 if (pmds[i]) {
 197                         pgtable_pmd_page_dtor(virt_to_page(pmds[i]));
 198                         free_page((unsigned long)pmds[i]);
 199                 }
 200 }
 201
 202 static int preallocate_pmds(pmd_t *pmds[])
 203 {
 204         int i;
 205         bool failed = false;
 206
 207         for(i = 0; i < PREALLOCATED_PMDS; i++) {
 208                 pmd_t *pmd = (pmd_t *)__get_free_page(PGALLOC_GFP);
 209                 if (!pmd)
 210                         failed = true;
 211                 if (pmd && !pgtable_pmd_page_ctor(virt_to_page(pmd))) {
 212                         free_page((unsigned long)pmds[i]);
 213                         pmd = NULL;
 214                         failed = true;
 215                 }
 216                 pmds[i] = pmd;
 217         }
 218
 219         if (failed) {
 220                 free_pmds(pmds);
 221                 return -ENOMEM;
 222         }
 223
 224         return 0;
 225 }
 226
 227 /*
 228  * Mop up any pmd pages which may still be attached to the pgd.
 229  * Normally they will be freed by munmap/exit_mmap, but any pmd we
 230  * preallocate which never got a corresponding vma will need to be
 231  * freed manually.
 232  */
 233 static void pgd_mop_up_pmds(struct mm_struct *mm, pgd_t *pgdp)
 234 {
 235         int i;
 236
 237         for(i = 0; i < PREALLOCATED_PMDS; i++) {
 238                 pgd_t pgd = pgdp[i];
 239
 240                 if (pgd_val(pgd) != 0) {
 241                         pmd_t *pmd = (pmd_t *)pgd_page_vaddr(pgd);
 242
 243                         pgdp[i] = native_make_pgd(0);
 244
 245                         paravirt_release_pmd(pgd_val(pgd) >> PAGE_SHIFT);
 246                         pmd_free(mm, pmd);
 247                 }
 248         }
 249 }
 250
 251 static void pgd_prepopulate_pmd(struct mm_struct *mm, pgd_t *pgd, pmd_t *pmds[])
 252 {
 253         pud_t *pud;
 254         int i;
 255
 256         if (PREALLOCATED_PMDS == 0) /* Work around gcc-3.4.x bug */
 257                 return;
 258
 259         pud = pud_offset(pgd, 0);
 260
 261         for (i = 0; i < PREALLOCATED_PMDS; i++, pud++) {
 262                 pmd_t *pmd = pmds[i];
 263
 264                 if (i >= KERNEL_PGD_BOUNDARY)
 265                         memcpy(pmd, (pmd_t *)pgd_page_vaddr(swapper_pg_dir[i]),
 266                                sizeof(pmd_t) * PTRS_PER_PMD);
 267
 268                 pud_populate(mm, pud, pmd);
 269         }
 270 }
 271
 272 pgd_t *pgd_alloc(struct mm_struct *mm)
 273 {
 274         pgd_t *pgd;
 275         pmd_t *pmds[PREALLOCATED_PMDS];
 276
 277         pgd = (pgd_t *)__get_free_page(PGALLOC_GFP);
 278
 279         if (pgd == NULL)
 280                 goto out;
 281
 282         mm->pgd = pgd;
 283
 284         if (preallocate_pmds(pmds) != 0)
 285                 goto out_free_pgd;
 286
 287         if (paravirt_pgd_alloc(mm) != 0)
 288                 goto out_free_pmds;
 289
 290         /*
 291          * Make sure that pre-populating the pmds is atomic with
 292          * respect to anything walking the pgd_list, so that they
 293          * never see a partially populated pgd.
 294          */
 295         spin_lock(&pgd_lock);
 296
 297         pgd_ctor(mm, pgd);
 298         pgd_prepopulate_pmd(mm, pgd, pmds);
 299
 300         spin_unlock(&pgd_lock);
 301
 302         return pgd;
 303
 304 out_free_pmds:
 305         free_pmds(pmds);
 306 out_free_pgd:
 307         free_page((unsigned long)pgd);
 308 out:
 309         return NULL;
 310 }
 311
 312 void pgd_free(struct mm_struct *mm, pgd_t *pgd)
 313 {
 314         pgd_mop_up_pmds(mm, pgd);
 315         pgd_dtor(pgd);
 316         paravirt_pgd_free(mm, pgd);
 317         free_page((unsigned long)pgd);
 318 }
 319
 320 /*
 321  * Used to set accessed or dirty bits in the page table entries
 322  * on other architectures. On x86, the accessed and dirty bits
 323  * are tracked by hardware. However, do_wp_page calls this function
 324  * to also make the pte writeable at the same time the dirty bit is
 325  * set. In that case we do actually need to write the PTE.
 326  */
 327 int ptep_set_access_flags(struct vm_area_struct *vma,
 328                           unsigned long address, pte_t *ptep,
 329                           pte_t entry, int dirty)
 330 {
 331         int changed = !pte_same(*ptep, entry);
 332
 333         if (changed && dirty) {
 334                 *ptep = entry;
 335                 pte_update_defer(vma->vm_mm, address, ptep);
 336         }
 337
 338         return changed;
 339 }
 340
 341 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 342 int pmdp_set_access_flags(struct vm_area_struct *vma,
 343                           unsigned long address, pmd_t *pmdp,
 344                           pmd_t entry, int dirty)
 345 {
 346         int changed = !pmd_same(*pmdp, entry);
 347
 348         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 349
 350         if (changed && dirty) {
 351                 *pmdp = entry;
 352                 pmd_update_defer(vma->vm_mm, address, pmdp);
 353                 /*
 354                  * We had a write-protection fault here and changed the pmd
 355                  * to to more permissive. No need to flush the TLB for that,
 356                  * #PF is architecturally guaranteed to do that and in the
 357                  * worst-case we'll generate a spurious fault.
 358                  */
 359         }
 360
 361         return changed;
 362 }
 363 #endif
 364
 365 int ptep_test_and_clear_young(struct vm_area_struct *vma,
 366                               unsigned long addr, pte_t *ptep)
 367 {
 368         int ret = 0;
 369
 370         if (pte_young(*ptep))
 371                 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
 372                                          (unsigned long *) &ptep->pte);
 373
 374         if (ret)
 375                 pte_update(vma->vm_mm, addr, ptep);
 376
 377         return ret;
 378 }
 379
 380 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 381 int pmdp_test_and_clear_young(struct vm_area_struct *vma,
 382                               unsigned long addr, pmd_t *pmdp)
 383 {
 384         int ret = 0;
 385
 386         if (pmd_young(*pmdp))
 387                 ret = test_and_clear_bit(_PAGE_BIT_ACCESSED,
 388                                          (unsigned long *)pmdp);
 389
 390         if (ret)
 391                 pmd_update(vma->vm_mm, addr, pmdp);
 392
 393         return ret;
 394 }
 395 #endif
 396
 397 int ptep_clear_flush_young(struct vm_area_struct *vma,
 398                            unsigned long address, pte_t *ptep)
 399 {
 400         int young;
 401
 402         young = ptep_test_and_clear_young(vma, address, ptep);
 403         if (young)
 404                 flush_tlb_page(vma, address);
 405
 406         return young;
 407 }
 408
 409 #ifdef CONFIG_TRANSPARENT_HUGEPAGE
 410 int pmdp_clear_flush_young(struct vm_area_struct *vma,
 411                            unsigned long address, pmd_t *pmdp)
 412 {
 413         int young;
 414
 415         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 416
 417         young = pmdp_test_and_clear_young(vma, address, pmdp);
 418         if (young)
 419                 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
 420
 421         return young;
 422 }
 423
 424 void pmdp_splitting_flush(struct vm_area_struct *vma,
 425                           unsigned long address, pmd_t *pmdp)
 426 {
 427         int set;
 428         VM_BUG_ON(address & ~HPAGE_PMD_MASK);
 429         set = !test_and_set_bit(_PAGE_BIT_SPLITTING,
 430                                 (unsigned long *)pmdp);
 431         if (set) {
 432                 pmd_update(vma->vm_mm, address, pmdp);
 433                 /* need tlb flush only to serialize against gup-fast */
 434                 flush_tlb_range(vma, address, address + HPAGE_PMD_SIZE);
 435         }
 436 }
 437 #endif
 438
 439 /**
 440  * reserve_top_address - reserves a hole in the top of kernel address space
 441  * @reserve - size of hole to reserve
 442  *
 443  * Can be used to relocate the fixmap area and poke a hole in the top
 444  * of kernel address space to make room for a hypervisor.
 445  */
 446 void __init reserve_top_address(unsigned long reserve)
 447 {
 448 #ifdef CONFIG_X86_32
 449         BUG_ON(fixmaps_set > 0);
 450         printk(KERN_INFO "Reserving virtual address space above 0x%08x\n",
 451                (int)-reserve);
 452         __FIXADDR_TOP = -reserve - PAGE_SIZE;
 453 #endif
 454 }
 455
 456 int fixmaps_set;
 457
 458 void __native_set_fixmap(enum fixed_addresses idx, pte_t pte)
 459 {
 460         unsigned long address = __fix_to_virt(idx);
 461
 462         if (idx >= __end_of_fixed_addresses) {
 463                 BUG();
 464                 return;
 465         }
 466         set_pte_vaddr(address, pte);
 467         fixmaps_set++;
 468 }
 469
 470 void native_set_fixmap(enum fixed_addresses idx, phys_addr_t phys,
 471                        pgprot_t flags)
 472 {
 473         __native_set_fixmap(idx, pfn_pte(phys >> PAGE_SHIFT, flags));
 474 }