arch/s390/mm/hugetlbpage.c

   1 /*
   2  *  IBM System z Huge TLB Page Support for Kernel.
   3  *
   4  *    Copyright IBM Corp. 2007
   5  *    Author(s): Gerald Schaefer <gerald.schaefer@de.ibm.com>
   6  */
   7
   8 #include <linux/mm.h>
   9 #include <linux/hugetlb.h>
  10
  11 static inline pmd_t __pte_to_pmd(pte_t pte)
  12 {
  13         pmd_t pmd;
  14
  15         /*
  16          * Convert encoding               pte bits         pmd bits
  17          *                              lIR.uswrdy.p    dy..R...I...wr
  18          * empty                        010.000000.0 -> 00..0...1...00
  19          * prot-none, clean, old        111.000000.1 -> 00..1...1...00
  20          * prot-none, clean, young      111.000001.1 -> 01..1...1...00
  21          * prot-none, dirty, old        111.000010.1 -> 10..1...1...00
  22          * prot-none, dirty, young      111.000011.1 -> 11..1...1...00
  23          * read-only, clean, old        111.000100.1 -> 00..1...1...01
  24          * read-only, clean, young      101.000101.1 -> 01..1...0...01
  25          * read-only, dirty, old        111.000110.1 -> 10..1...1...01
  26          * read-only, dirty, young      101.000111.1 -> 11..1...0...01
  27          * read-write, clean, old       111.001100.1 -> 00..1...1...11
  28          * read-write, clean, young     101.001101.1 -> 01..1...0...11
  29          * read-write, dirty, old       110.001110.1 -> 10..0...1...11
  30          * read-write, dirty, young     100.001111.1 -> 11..0...0...11
  31          * HW-bits: R read-only, I invalid
  32          * SW-bits: p present, y young, d dirty, r read, w write, s special,
  33          *          u unused, l large
  34          */
  35         if (pte_present(pte)) {
  36                 pmd_val(pmd) = pte_val(pte) & PAGE_MASK;
  37                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_READ) >> 4;
  38                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_WRITE) >> 4;
  39                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_INVALID) >> 5;
  40                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_PROTECT);
  41                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_DIRTY) << 10;
  42                 pmd_val(pmd) |= (pte_val(pte) & _PAGE_YOUNG) << 10;
  43         } else
  44                 pmd_val(pmd) = _SEGMENT_ENTRY_INVALID;
  45         return pmd;
  46 }
  47
  48 static inline pte_t __pmd_to_pte(pmd_t pmd)
  49 {
  50         pte_t pte;
  51
  52         /*
  53          * Convert encoding                pmd bits         pte bits
  54          *                              dy..R...I...wr    lIR.uswrdy.p
  55          * empty                        00..0...1...00 -> 010.000000.0
  56          * prot-none, clean, old        00..1...1...00 -> 111.000000.1
  57          * prot-none, clean, young      01..1...1...00 -> 111.000001.1
  58          * prot-none, dirty, old        10..1...1...00 -> 111.000010.1
  59          * prot-none, dirty, young      11..1...1...00 -> 111.000011.1
  60          * read-only, clean, old        00..1...1...01 -> 111.000100.1
  61          * read-only, clean, young      01..1...0...01 -> 101.000101.1
  62          * read-only, dirty, old        10..1...1...01 -> 111.000110.1
  63          * read-only, dirty, young      11..1...0...01 -> 101.000111.1
  64          * read-write, clean, old       00..1...1...11 -> 111.001100.1
  65          * read-write, clean, young     01..1...0...11 -> 101.001101.1
  66          * read-write, dirty, old       10..0...1...11 -> 110.001110.1
  67          * read-write, dirty, young     11..0...0...11 -> 100.001111.1
  68          * HW-bits: R read-only, I invalid
  69          * SW-bits: p present, y young, d dirty, r read, w write, s special,
  70          *          u unused, l large
  71          */
  72         if (pmd_present(pmd)) {
  73                 pte_val(pte) = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN_LARGE;
  74                 pte_val(pte) |= _PAGE_LARGE | _PAGE_PRESENT;
  75                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_READ) << 4;
  76                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_WRITE) << 4;
  77                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_INVALID) << 5;
  78                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_PROTECT);
  79                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_DIRTY) >> 10;
  80                 pte_val(pte) |= (pmd_val(pmd) & _SEGMENT_ENTRY_YOUNG) >> 10;
  81         } else
  82                 pte_val(pte) = _PAGE_INVALID;
  83         return pte;
  84 }
  85
  86 void set_huge_pte_at(struct mm_struct *mm, unsigned long addr,
  87                      pte_t *ptep, pte_t pte)
  88 {
  89         pmd_t pmd;
  90
  91         pmd = __pte_to_pmd(pte);
  92         if (!MACHINE_HAS_HPAGE) {
  93                 /* Emulated huge ptes loose the dirty and young bit */
  94                 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
  95                 pmd_val(pmd) |= pte_page(pte)[1].index;
  96         } else
  97                 pmd_val(pmd) |= _SEGMENT_ENTRY_LARGE;
  98         *(pmd_t *) ptep = pmd;
  99 }
 100
 101 pte_t huge_ptep_get(pte_t *ptep)
 102 {
 103         unsigned long origin;
 104         pmd_t pmd;
 105
 106         pmd = *(pmd_t *) ptep;
 107         if (!MACHINE_HAS_HPAGE && pmd_present(pmd)) {
 108                 origin = pmd_val(pmd) & _SEGMENT_ENTRY_ORIGIN;
 109                 pmd_val(pmd) &= ~_SEGMENT_ENTRY_ORIGIN;
 110                 pmd_val(pmd) |= *(unsigned long *) origin;
 111                 /* Emulated huge ptes are young and dirty by definition */
 112                 pmd_val(pmd) |= _SEGMENT_ENTRY_YOUNG | _SEGMENT_ENTRY_DIRTY;
 113         }
 114         return __pmd_to_pte(pmd);
 115 }
 116
 117 pte_t huge_ptep_get_and_clear(struct mm_struct *mm,
 118                               unsigned long addr, pte_t *ptep)
 119 {
 120         pmd_t *pmdp = (pmd_t *) ptep;
 121         pte_t pte = huge_ptep_get(ptep);
 122
 123         pmdp_flush_direct(mm, addr, pmdp);
 124         pmd_val(*pmdp) = _SEGMENT_ENTRY_EMPTY;
 125         return pte;
 126 }
 127
 128 int arch_prepare_hugepage(struct page *page)
 129 {
 130         unsigned long addr = page_to_phys(page);
 131         pte_t pte;
 132         pte_t *ptep;
 133         int i;
 134
 135         if (MACHINE_HAS_HPAGE)
 136                 return 0;
 137
 138         ptep = (pte_t *) pte_alloc_one(&init_mm, addr);
 139         if (!ptep)
 140                 return -ENOMEM;
 141
 142         pte_val(pte) = addr;
 143         for (i = 0; i < PTRS_PER_PTE; i++) {
 144                 set_pte_at(&init_mm, addr + i * PAGE_SIZE, ptep + i, pte);
 145                 pte_val(pte) += PAGE_SIZE;
 146         }
 147         page[1].index = (unsigned long) ptep;
 148         return 0;
 149 }
 150
 151 void arch_release_hugepage(struct page *page)
 152 {
 153         pte_t *ptep;
 154
 155         if (MACHINE_HAS_HPAGE)
 156                 return;
 157
 158         ptep = (pte_t *) page[1].index;
 159         if (!ptep)
 160                 return;
 161         clear_table((unsigned long *) ptep, _PAGE_INVALID,
 162                     PTRS_PER_PTE * sizeof(pte_t));
 163         page_table_free(&init_mm, (unsigned long *) ptep);
 164         page[1].index = 0;
 165 }
 166
 167 pte_t *huge_pte_alloc(struct mm_struct *mm,
 168                         unsigned long addr, unsigned long sz)
 169 {
 170         pgd_t *pgdp;
 171         pud_t *pudp;
 172         pmd_t *pmdp = NULL;
 173
 174         pgdp = pgd_offset(mm, addr);
 175         pudp = pud_alloc(mm, pgdp, addr);
 176         if (pudp)
 177                 pmdp = pmd_alloc(mm, pudp, addr);
 178         return (pte_t *) pmdp;
 179 }
 180
 181 pte_t *huge_pte_offset(struct mm_struct *mm, unsigned long addr)
 182 {
 183         pgd_t *pgdp;
 184         pud_t *pudp;
 185         pmd_t *pmdp = NULL;
 186
 187         pgdp = pgd_offset(mm, addr);
 188         if (pgd_present(*pgdp)) {
 189                 pudp = pud_offset(pgdp, addr);
 190                 if (pud_present(*pudp))
 191                         pmdp = pmd_offset(pudp, addr);
 192         }
 193         return (pte_t *) pmdp;
 194 }
 195
 196 int pmd_huge(pmd_t pmd)
 197 {
 198         if (!MACHINE_HAS_HPAGE)
 199                 return 0;
 200
 201         return !!(pmd_val(pmd) & _SEGMENT_ENTRY_LARGE);
 202 }
 203
 204 int pud_huge(pud_t pud)
 205 {
 206         return 0;
 207 }