2 * Page table allocation functions
4 * Copyright IBM Corp. 2016
5 * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com>
9 #include <linux/sysctl.h>
10 #include <asm/mmu_context.h>
11 #include <asm/pgalloc.h>
14 #include <asm/tlbflush.h>
18 static int page_table_allocate_pgste_min
= 0;
19 static int page_table_allocate_pgste_max
= 1;
20 int page_table_allocate_pgste
= 0;
21 EXPORT_SYMBOL(page_table_allocate_pgste
);
23 static struct ctl_table page_table_sysctl
[] = {
25 .procname
= "allocate_pgste",
26 .data
= &page_table_allocate_pgste
,
27 .maxlen
= sizeof(int),
28 .mode
= S_IRUGO
| S_IWUSR
,
29 .proc_handler
= proc_dointvec
,
30 .extra1
= &page_table_allocate_pgste_min
,
31 .extra2
= &page_table_allocate_pgste_max
,
36 static struct ctl_table page_table_sysctl_dir
[] = {
41 .child
= page_table_sysctl
,
46 static int __init
page_table_register_sysctl(void)
48 return register_sysctl_table(page_table_sysctl_dir
) ? 0 : -ENOMEM
;
50 __initcall(page_table_register_sysctl
);
52 #endif /* CONFIG_PGSTE */
54 unsigned long *crst_table_alloc(struct mm_struct
*mm
)
56 struct page
*page
= alloc_pages(GFP_KERNEL
, 2);
60 return (unsigned long *) page_to_phys(page
);
63 void crst_table_free(struct mm_struct
*mm
, unsigned long *table
)
65 free_pages((unsigned long) table
, 2);
68 static void __crst_table_upgrade(void *arg
)
70 struct mm_struct
*mm
= arg
;
72 if (current
->active_mm
== mm
) {
79 int crst_table_upgrade(struct mm_struct
*mm
)
81 unsigned long *table
, *pgd
;
83 /* upgrade should only happen from 3 to 4 levels */
84 BUG_ON(mm
->context
.asce_limit
!= (1UL << 42));
86 table
= crst_table_alloc(mm
);
90 spin_lock_bh(&mm
->page_table_lock
);
91 pgd
= (unsigned long *) mm
->pgd
;
92 crst_table_init(table
, _REGION2_ENTRY_EMPTY
);
93 pgd_populate(mm
, (pgd_t
*) table
, (pud_t
*) pgd
);
94 mm
->pgd
= (pgd_t
*) table
;
95 mm
->context
.asce_limit
= 1UL << 53;
96 mm
->context
.asce
= __pa(mm
->pgd
) | _ASCE_TABLE_LENGTH
|
97 _ASCE_USER_BITS
| _ASCE_TYPE_REGION2
;
98 mm
->task_size
= mm
->context
.asce_limit
;
99 spin_unlock_bh(&mm
->page_table_lock
);
101 on_each_cpu(__crst_table_upgrade
, mm
, 0);
105 void crst_table_downgrade(struct mm_struct
*mm
)
109 /* downgrade should only happen from 3 to 2 levels (compat only) */
110 BUG_ON(mm
->context
.asce_limit
!= (1UL << 42));
112 if (current
->active_mm
== mm
) {
118 mm
->pgd
= (pgd_t
*) (pgd_val(*pgd
) & _REGION_ENTRY_ORIGIN
);
119 mm
->context
.asce_limit
= 1UL << 31;
120 mm
->context
.asce
= __pa(mm
->pgd
) | _ASCE_TABLE_LENGTH
|
121 _ASCE_USER_BITS
| _ASCE_TYPE_SEGMENT
;
122 mm
->task_size
= mm
->context
.asce_limit
;
123 crst_table_free(mm
, (unsigned long *) pgd
);
125 if (current
->active_mm
== mm
)
129 static inline unsigned int atomic_xor_bits(atomic_t
*v
, unsigned int bits
)
131 unsigned int old
, new;
134 old
= atomic_read(v
);
136 } while (atomic_cmpxchg(v
, old
, new) != old
);
141 * page table entry allocation/free routines.
143 unsigned long *page_table_alloc(struct mm_struct
*mm
)
145 unsigned long *table
;
147 unsigned int mask
, bit
;
149 /* Try to get a fragment of a 4K page as a 2K page table */
150 if (!mm_alloc_pgste(mm
)) {
152 spin_lock_bh(&mm
->context
.list_lock
);
153 if (!list_empty(&mm
->context
.pgtable_list
)) {
154 page
= list_first_entry(&mm
->context
.pgtable_list
,
156 mask
= atomic_read(&page
->_mapcount
);
157 mask
= (mask
| (mask
>> 4)) & 3;
159 table
= (unsigned long *) page_to_phys(page
);
160 bit
= mask
& 1; /* =1 -> second 2K */
162 table
+= PTRS_PER_PTE
;
163 atomic_xor_bits(&page
->_mapcount
, 1U << bit
);
164 list_del(&page
->lru
);
167 spin_unlock_bh(&mm
->context
.list_lock
);
171 /* Allocate a fresh page */
172 page
= alloc_page(GFP_KERNEL
|__GFP_REPEAT
);
175 if (!pgtable_page_ctor(page
)) {
179 /* Initialize page table */
180 table
= (unsigned long *) page_to_phys(page
);
181 if (mm_alloc_pgste(mm
)) {
182 /* Return 4K page table with PGSTEs */
183 atomic_set(&page
->_mapcount
, 3);
184 clear_table(table
, _PAGE_INVALID
, PAGE_SIZE
/2);
185 clear_table(table
+ PTRS_PER_PTE
, 0, PAGE_SIZE
/2);
187 /* Return the first 2K fragment of the page */
188 atomic_set(&page
->_mapcount
, 1);
189 clear_table(table
, _PAGE_INVALID
, PAGE_SIZE
);
190 spin_lock_bh(&mm
->context
.list_lock
);
191 list_add(&page
->lru
, &mm
->context
.pgtable_list
);
192 spin_unlock_bh(&mm
->context
.list_lock
);
197 void page_table_free(struct mm_struct
*mm
, unsigned long *table
)
200 unsigned int bit
, mask
;
202 page
= pfn_to_page(__pa(table
) >> PAGE_SHIFT
);
203 if (!mm_alloc_pgste(mm
)) {
204 /* Free 2K page table fragment of a 4K page */
205 bit
= (__pa(table
) & ~PAGE_MASK
)/(PTRS_PER_PTE
*sizeof(pte_t
));
206 spin_lock_bh(&mm
->context
.list_lock
);
207 mask
= atomic_xor_bits(&page
->_mapcount
, 1U << bit
);
209 list_add(&page
->lru
, &mm
->context
.pgtable_list
);
211 list_del(&page
->lru
);
212 spin_unlock_bh(&mm
->context
.list_lock
);
217 pgtable_page_dtor(page
);
218 atomic_set(&page
->_mapcount
, -1);
222 void page_table_free_rcu(struct mmu_gather
*tlb
, unsigned long *table
,
223 unsigned long vmaddr
)
225 struct mm_struct
*mm
;
227 unsigned int bit
, mask
;
230 page
= pfn_to_page(__pa(table
) >> PAGE_SHIFT
);
231 if (mm_alloc_pgste(mm
)) {
232 gmap_unlink(mm
, table
, vmaddr
);
233 table
= (unsigned long *) (__pa(table
) | 3);
234 tlb_remove_table(tlb
, table
);
237 bit
= (__pa(table
) & ~PAGE_MASK
) / (PTRS_PER_PTE
*sizeof(pte_t
));
238 spin_lock_bh(&mm
->context
.list_lock
);
239 mask
= atomic_xor_bits(&page
->_mapcount
, 0x11U
<< bit
);
241 list_add_tail(&page
->lru
, &mm
->context
.pgtable_list
);
243 list_del(&page
->lru
);
244 spin_unlock_bh(&mm
->context
.list_lock
);
245 table
= (unsigned long *) (__pa(table
) | (1U << bit
));
246 tlb_remove_table(tlb
, table
);
249 static void __tlb_remove_table(void *_table
)
251 unsigned int mask
= (unsigned long) _table
& 3;
252 void *table
= (void *)((unsigned long) _table
^ mask
);
253 struct page
*page
= pfn_to_page(__pa(table
) >> PAGE_SHIFT
);
256 case 0: /* pmd or pud */
257 free_pages((unsigned long) table
, 2);
259 case 1: /* lower 2K of a 4K page table */
260 case 2: /* higher 2K of a 4K page table */
261 if (atomic_xor_bits(&page
->_mapcount
, mask
<< 4) != 0)
264 case 3: /* 4K page table with pgstes */
265 pgtable_page_dtor(page
);
266 atomic_set(&page
->_mapcount
, -1);
272 static void tlb_remove_table_smp_sync(void *arg
)
274 /* Simply deliver the interrupt */
277 static void tlb_remove_table_one(void *table
)
280 * This isn't an RCU grace period and hence the page-tables cannot be
281 * assumed to be actually RCU-freed.
283 * It is however sufficient for software page-table walkers that rely
284 * on IRQ disabling. See the comment near struct mmu_table_batch.
286 smp_call_function(tlb_remove_table_smp_sync
, NULL
, 1);
287 __tlb_remove_table(table
);
290 static void tlb_remove_table_rcu(struct rcu_head
*head
)
292 struct mmu_table_batch
*batch
;
295 batch
= container_of(head
, struct mmu_table_batch
, rcu
);
297 for (i
= 0; i
< batch
->nr
; i
++)
298 __tlb_remove_table(batch
->tables
[i
]);
300 free_page((unsigned long)batch
);
303 void tlb_table_flush(struct mmu_gather
*tlb
)
305 struct mmu_table_batch
**batch
= &tlb
->batch
;
308 call_rcu_sched(&(*batch
)->rcu
, tlb_remove_table_rcu
);
313 void tlb_remove_table(struct mmu_gather
*tlb
, void *table
)
315 struct mmu_table_batch
**batch
= &tlb
->batch
;
317 tlb
->mm
->context
.flush_mm
= 1;
318 if (*batch
== NULL
) {
319 *batch
= (struct mmu_table_batch
*)
320 __get_free_page(GFP_NOWAIT
| __GFP_NOWARN
);
321 if (*batch
== NULL
) {
322 __tlb_flush_mm_lazy(tlb
->mm
);
323 tlb_remove_table_one(table
);
328 (*batch
)->tables
[(*batch
)->nr
++] = table
;
329 if ((*batch
)->nr
== MAX_TABLE_BATCH
)