Commit | Line | Data |
---|---|---|
3610cce8 | 1 | /* |
a53c8fab | 2 | * Copyright IBM Corp. 2007, 2011 |
3610cce8 MS |
3 | * Author(s): Martin Schwidefsky <schwidefsky@de.ibm.com> |
4 | */ | |
5 | ||
6 | #include <linux/sched.h> | |
7 | #include <linux/kernel.h> | |
8 | #include <linux/errno.h> | |
5a0e3ad6 | 9 | #include <linux/gfp.h> |
3610cce8 MS |
10 | #include <linux/mm.h> |
11 | #include <linux/swap.h> | |
12 | #include <linux/smp.h> | |
3610cce8 | 13 | #include <linux/spinlock.h> |
80217147 | 14 | #include <linux/rcupdate.h> |
e5992f2e | 15 | #include <linux/slab.h> |
b31288fa | 16 | #include <linux/swapops.h> |
0b46e0a3 | 17 | #include <linux/sysctl.h> |
3ac8e380 DD |
18 | #include <linux/ksm.h> |
19 | #include <linux/mman.h> | |
3610cce8 | 20 | |
3610cce8 MS |
21 | #include <asm/pgtable.h> |
22 | #include <asm/pgalloc.h> | |
23 | #include <asm/tlb.h> | |
24 | #include <asm/tlbflush.h> | |
6252d702 | 25 | #include <asm/mmu_context.h> |
3610cce8 | 26 | |
ebde765c MS |
27 | static inline pte_t ptep_flush_direct(struct mm_struct *mm, |
28 | unsigned long addr, pte_t *ptep) | |
29 | { | |
ebde765c MS |
30 | pte_t old; |
31 | ||
32 | old = *ptep; | |
33 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
34 | return old; | |
64f31d58 MS |
35 | atomic_inc(&mm->context.flush_count); |
36 | if (MACHINE_HAS_TLB_LC && | |
ebde765c MS |
37 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
38 | __ptep_ipte_local(addr, ptep); | |
39 | else | |
40 | __ptep_ipte(addr, ptep); | |
64f31d58 | 41 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
42 | return old; |
43 | } | |
44 | ||
45 | static inline pte_t ptep_flush_lazy(struct mm_struct *mm, | |
46 | unsigned long addr, pte_t *ptep) | |
47 | { | |
ebde765c MS |
48 | pte_t old; |
49 | ||
50 | old = *ptep; | |
51 | if (unlikely(pte_val(old) & _PAGE_INVALID)) | |
52 | return old; | |
64f31d58 MS |
53 | atomic_inc(&mm->context.flush_count); |
54 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
55 | cpumask_of(smp_processor_id()))) { | |
ebde765c MS |
56 | pte_val(*ptep) |= _PAGE_INVALID; |
57 | mm->context.flush_mm = 1; | |
58 | } else | |
59 | __ptep_ipte(addr, ptep); | |
64f31d58 | 60 | atomic_dec(&mm->context.flush_count); |
ebde765c MS |
61 | return old; |
62 | } | |
63 | ||
1e133ab2 MS |
64 | static inline pgste_t pgste_get_lock(pte_t *ptep) |
65 | { | |
66 | unsigned long new = 0; | |
67 | #ifdef CONFIG_PGSTE | |
68 | unsigned long old; | |
69 | ||
1e133ab2 MS |
70 | asm( |
71 | " lg %0,%2\n" | |
72 | "0: lgr %1,%0\n" | |
73 | " nihh %0,0xff7f\n" /* clear PCL bit in old */ | |
74 | " oihh %1,0x0080\n" /* set PCL bit in new */ | |
75 | " csg %0,%1,%2\n" | |
76 | " jl 0b\n" | |
77 | : "=&d" (old), "=&d" (new), "=Q" (ptep[PTRS_PER_PTE]) | |
78 | : "Q" (ptep[PTRS_PER_PTE]) : "cc", "memory"); | |
79 | #endif | |
80 | return __pgste(new); | |
81 | } | |
82 | ||
83 | static inline void pgste_set_unlock(pte_t *ptep, pgste_t pgste) | |
84 | { | |
85 | #ifdef CONFIG_PGSTE | |
86 | asm( | |
87 | " nihh %1,0xff7f\n" /* clear PCL bit */ | |
88 | " stg %1,%0\n" | |
89 | : "=Q" (ptep[PTRS_PER_PTE]) | |
90 | : "d" (pgste_val(pgste)), "Q" (ptep[PTRS_PER_PTE]) | |
91 | : "cc", "memory"); | |
1e133ab2 MS |
92 | #endif |
93 | } | |
94 | ||
95 | static inline pgste_t pgste_get(pte_t *ptep) | |
96 | { | |
97 | unsigned long pgste = 0; | |
98 | #ifdef CONFIG_PGSTE | |
99 | pgste = *(unsigned long *)(ptep + PTRS_PER_PTE); | |
100 | #endif | |
101 | return __pgste(pgste); | |
102 | } | |
103 | ||
104 | static inline void pgste_set(pte_t *ptep, pgste_t pgste) | |
105 | { | |
106 | #ifdef CONFIG_PGSTE | |
107 | *(pgste_t *)(ptep + PTRS_PER_PTE) = pgste; | |
108 | #endif | |
109 | } | |
110 | ||
ebde765c MS |
111 | static inline pgste_t pgste_update_all(pte_t pte, pgste_t pgste, |
112 | struct mm_struct *mm) | |
113 | { | |
114 | #ifdef CONFIG_PGSTE | |
115 | unsigned long address, bits, skey; | |
116 | ||
117 | if (!mm_use_skey(mm) || pte_val(pte) & _PAGE_INVALID) | |
118 | return pgste; | |
119 | address = pte_val(pte) & PAGE_MASK; | |
120 | skey = (unsigned long) page_get_storage_key(address); | |
121 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
122 | /* Transfer page changed & referenced bit to guest bits in pgste */ | |
123 | pgste_val(pgste) |= bits << 48; /* GR bit & GC bit */ | |
124 | /* Copy page access key and fetch protection bit to pgste */ | |
125 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
126 | pgste_val(pgste) |= (skey & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
127 | #endif | |
128 | return pgste; | |
129 | ||
130 | } | |
131 | ||
132 | static inline void pgste_set_key(pte_t *ptep, pgste_t pgste, pte_t entry, | |
133 | struct mm_struct *mm) | |
134 | { | |
135 | #ifdef CONFIG_PGSTE | |
136 | unsigned long address; | |
137 | unsigned long nkey; | |
138 | ||
139 | if (!mm_use_skey(mm) || pte_val(entry) & _PAGE_INVALID) | |
140 | return; | |
141 | VM_BUG_ON(!(pte_val(*ptep) & _PAGE_INVALID)); | |
142 | address = pte_val(entry) & PAGE_MASK; | |
143 | /* | |
144 | * Set page access key and fetch protection bit from pgste. | |
145 | * The guest C/R information is still in the PGSTE, set real | |
146 | * key C/R to 0. | |
147 | */ | |
148 | nkey = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; | |
149 | nkey |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; | |
150 | page_set_storage_key(address, nkey, 0); | |
151 | #endif | |
152 | } | |
153 | ||
154 | static inline pgste_t pgste_set_pte(pte_t *ptep, pgste_t pgste, pte_t entry) | |
155 | { | |
156 | #ifdef CONFIG_PGSTE | |
157 | if ((pte_val(entry) & _PAGE_PRESENT) && | |
158 | (pte_val(entry) & _PAGE_WRITE) && | |
159 | !(pte_val(entry) & _PAGE_INVALID)) { | |
160 | if (!MACHINE_HAS_ESOP) { | |
161 | /* | |
162 | * Without enhanced suppression-on-protection force | |
163 | * the dirty bit on for all writable ptes. | |
164 | */ | |
165 | pte_val(entry) |= _PAGE_DIRTY; | |
166 | pte_val(entry) &= ~_PAGE_PROTECT; | |
167 | } | |
168 | if (!(pte_val(entry) & _PAGE_PROTECT)) | |
169 | /* This pte allows write access, set user-dirty */ | |
170 | pgste_val(pgste) |= PGSTE_UC_BIT; | |
171 | } | |
172 | #endif | |
173 | *ptep = entry; | |
174 | return pgste; | |
175 | } | |
176 | ||
b2d73b2a MS |
177 | static inline pgste_t pgste_pte_notify(struct mm_struct *mm, |
178 | unsigned long addr, | |
179 | pte_t *ptep, pgste_t pgste) | |
ebde765c MS |
180 | { |
181 | #ifdef CONFIG_PGSTE | |
4be130a0 MS |
182 | unsigned long bits; |
183 | ||
184 | bits = pgste_val(pgste) & (PGSTE_IN_BIT | PGSTE_VSIE_BIT); | |
185 | if (bits) { | |
186 | pgste_val(pgste) ^= bits; | |
187 | ptep_notify(mm, addr, ptep, bits); | |
ebde765c MS |
188 | } |
189 | #endif | |
190 | return pgste; | |
191 | } | |
192 | ||
ebde765c MS |
193 | static inline pgste_t ptep_xchg_start(struct mm_struct *mm, |
194 | unsigned long addr, pte_t *ptep) | |
195 | { | |
196 | pgste_t pgste = __pgste(0); | |
197 | ||
198 | if (mm_has_pgste(mm)) { | |
199 | pgste = pgste_get_lock(ptep); | |
b2d73b2a | 200 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
ebde765c MS |
201 | } |
202 | return pgste; | |
203 | } | |
204 | ||
205 | static inline void ptep_xchg_commit(struct mm_struct *mm, | |
206 | unsigned long addr, pte_t *ptep, | |
207 | pgste_t pgste, pte_t old, pte_t new) | |
208 | { | |
209 | if (mm_has_pgste(mm)) { | |
210 | if (pte_val(old) & _PAGE_INVALID) | |
211 | pgste_set_key(ptep, pgste, new, mm); | |
212 | if (pte_val(new) & _PAGE_INVALID) { | |
213 | pgste = pgste_update_all(old, pgste, mm); | |
214 | if ((pgste_val(pgste) & _PGSTE_GPS_USAGE_MASK) == | |
215 | _PGSTE_GPS_USAGE_UNUSED) | |
216 | pte_val(old) |= _PAGE_UNUSED; | |
217 | } | |
218 | pgste = pgste_set_pte(ptep, pgste, new); | |
219 | pgste_set_unlock(ptep, pgste); | |
220 | } else { | |
221 | *ptep = new; | |
222 | } | |
223 | } | |
224 | ||
225 | pte_t ptep_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
226 | pte_t *ptep, pte_t new) | |
227 | { | |
228 | pgste_t pgste; | |
229 | pte_t old; | |
230 | ||
a9809407 | 231 | preempt_disable(); |
ebde765c MS |
232 | pgste = ptep_xchg_start(mm, addr, ptep); |
233 | old = ptep_flush_direct(mm, addr, ptep); | |
234 | ptep_xchg_commit(mm, addr, ptep, pgste, old, new); | |
a9809407 | 235 | preempt_enable(); |
ebde765c MS |
236 | return old; |
237 | } | |
238 | EXPORT_SYMBOL(ptep_xchg_direct); | |
239 | ||
240 | pte_t ptep_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
241 | pte_t *ptep, pte_t new) | |
242 | { | |
243 | pgste_t pgste; | |
244 | pte_t old; | |
245 | ||
a9809407 | 246 | preempt_disable(); |
ebde765c MS |
247 | pgste = ptep_xchg_start(mm, addr, ptep); |
248 | old = ptep_flush_lazy(mm, addr, ptep); | |
249 | ptep_xchg_commit(mm, addr, ptep, pgste, old, new); | |
a9809407 | 250 | preempt_enable(); |
ebde765c MS |
251 | return old; |
252 | } | |
253 | EXPORT_SYMBOL(ptep_xchg_lazy); | |
254 | ||
255 | pte_t ptep_modify_prot_start(struct mm_struct *mm, unsigned long addr, | |
256 | pte_t *ptep) | |
257 | { | |
258 | pgste_t pgste; | |
259 | pte_t old; | |
260 | ||
a9809407 | 261 | preempt_disable(); |
ebde765c MS |
262 | pgste = ptep_xchg_start(mm, addr, ptep); |
263 | old = ptep_flush_lazy(mm, addr, ptep); | |
264 | if (mm_has_pgste(mm)) { | |
265 | pgste = pgste_update_all(old, pgste, mm); | |
266 | pgste_set(ptep, pgste); | |
267 | } | |
268 | return old; | |
269 | } | |
270 | EXPORT_SYMBOL(ptep_modify_prot_start); | |
271 | ||
272 | void ptep_modify_prot_commit(struct mm_struct *mm, unsigned long addr, | |
273 | pte_t *ptep, pte_t pte) | |
274 | { | |
275 | pgste_t pgste; | |
276 | ||
277 | if (mm_has_pgste(mm)) { | |
278 | pgste = pgste_get(ptep); | |
279 | pgste_set_key(ptep, pgste, pte, mm); | |
280 | pgste = pgste_set_pte(ptep, pgste, pte); | |
281 | pgste_set_unlock(ptep, pgste); | |
282 | } else { | |
283 | *ptep = pte; | |
284 | } | |
a9809407 | 285 | preempt_enable(); |
ebde765c MS |
286 | } |
287 | EXPORT_SYMBOL(ptep_modify_prot_commit); | |
288 | ||
227be799 MS |
289 | static inline pmd_t pmdp_flush_direct(struct mm_struct *mm, |
290 | unsigned long addr, pmd_t *pmdp) | |
291 | { | |
227be799 MS |
292 | pmd_t old; |
293 | ||
294 | old = *pmdp; | |
295 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
296 | return old; | |
297 | if (!MACHINE_HAS_IDTE) { | |
298 | __pmdp_csp(pmdp); | |
299 | return old; | |
300 | } | |
64f31d58 MS |
301 | atomic_inc(&mm->context.flush_count); |
302 | if (MACHINE_HAS_TLB_LC && | |
227be799 MS |
303 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) |
304 | __pmdp_idte_local(addr, pmdp); | |
305 | else | |
306 | __pmdp_idte(addr, pmdp); | |
64f31d58 | 307 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
308 | return old; |
309 | } | |
310 | ||
311 | static inline pmd_t pmdp_flush_lazy(struct mm_struct *mm, | |
312 | unsigned long addr, pmd_t *pmdp) | |
313 | { | |
227be799 MS |
314 | pmd_t old; |
315 | ||
316 | old = *pmdp; | |
317 | if (pmd_val(old) & _SEGMENT_ENTRY_INVALID) | |
318 | return old; | |
64f31d58 MS |
319 | atomic_inc(&mm->context.flush_count); |
320 | if (cpumask_equal(&mm->context.cpu_attach_mask, | |
321 | cpumask_of(smp_processor_id()))) { | |
227be799 MS |
322 | pmd_val(*pmdp) |= _SEGMENT_ENTRY_INVALID; |
323 | mm->context.flush_mm = 1; | |
324 | } else if (MACHINE_HAS_IDTE) | |
325 | __pmdp_idte(addr, pmdp); | |
326 | else | |
327 | __pmdp_csp(pmdp); | |
64f31d58 | 328 | atomic_dec(&mm->context.flush_count); |
227be799 MS |
329 | return old; |
330 | } | |
331 | ||
332 | pmd_t pmdp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
333 | pmd_t *pmdp, pmd_t new) | |
334 | { | |
335 | pmd_t old; | |
336 | ||
a9809407 | 337 | preempt_disable(); |
227be799 MS |
338 | old = pmdp_flush_direct(mm, addr, pmdp); |
339 | *pmdp = new; | |
a9809407 | 340 | preempt_enable(); |
227be799 MS |
341 | return old; |
342 | } | |
343 | EXPORT_SYMBOL(pmdp_xchg_direct); | |
344 | ||
345 | pmd_t pmdp_xchg_lazy(struct mm_struct *mm, unsigned long addr, | |
346 | pmd_t *pmdp, pmd_t new) | |
347 | { | |
348 | pmd_t old; | |
349 | ||
a9809407 | 350 | preempt_disable(); |
227be799 MS |
351 | old = pmdp_flush_lazy(mm, addr, pmdp); |
352 | *pmdp = new; | |
a9809407 | 353 | preempt_enable(); |
227be799 MS |
354 | return old; |
355 | } | |
356 | EXPORT_SYMBOL(pmdp_xchg_lazy); | |
357 | ||
d08de8e2 GS |
358 | static inline pud_t pudp_flush_direct(struct mm_struct *mm, |
359 | unsigned long addr, pud_t *pudp) | |
360 | { | |
361 | pud_t old; | |
362 | ||
363 | old = *pudp; | |
364 | if (pud_val(old) & _REGION_ENTRY_INVALID) | |
365 | return old; | |
366 | if (!MACHINE_HAS_IDTE) { | |
367 | /* | |
368 | * Invalid bit position is the same for pmd and pud, so we can | |
369 | * re-use _pmd_csp() here | |
370 | */ | |
371 | __pmdp_csp((pmd_t *) pudp); | |
372 | return old; | |
373 | } | |
374 | atomic_inc(&mm->context.flush_count); | |
375 | if (MACHINE_HAS_TLB_LC && | |
376 | cpumask_equal(mm_cpumask(mm), cpumask_of(smp_processor_id()))) | |
377 | __pudp_idte_local(addr, pudp); | |
378 | else | |
379 | __pudp_idte(addr, pudp); | |
380 | atomic_dec(&mm->context.flush_count); | |
381 | return old; | |
382 | } | |
383 | ||
384 | pud_t pudp_xchg_direct(struct mm_struct *mm, unsigned long addr, | |
385 | pud_t *pudp, pud_t new) | |
386 | { | |
387 | pud_t old; | |
388 | ||
389 | preempt_disable(); | |
390 | old = pudp_flush_direct(mm, addr, pudp); | |
391 | *pudp = new; | |
392 | preempt_enable(); | |
393 | return old; | |
394 | } | |
395 | EXPORT_SYMBOL(pudp_xchg_direct); | |
396 | ||
75077afb | 397 | #ifdef CONFIG_TRANSPARENT_HUGEPAGE |
6b0b50b0 AK |
398 | void pgtable_trans_huge_deposit(struct mm_struct *mm, pmd_t *pmdp, |
399 | pgtable_t pgtable) | |
9501d09f GS |
400 | { |
401 | struct list_head *lh = (struct list_head *) pgtable; | |
402 | ||
ec66ad66 | 403 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
404 | |
405 | /* FIFO */ | |
c389a250 | 406 | if (!pmd_huge_pte(mm, pmdp)) |
9501d09f GS |
407 | INIT_LIST_HEAD(lh); |
408 | else | |
c389a250 KS |
409 | list_add(lh, (struct list_head *) pmd_huge_pte(mm, pmdp)); |
410 | pmd_huge_pte(mm, pmdp) = pgtable; | |
9501d09f GS |
411 | } |
412 | ||
6b0b50b0 | 413 | pgtable_t pgtable_trans_huge_withdraw(struct mm_struct *mm, pmd_t *pmdp) |
9501d09f GS |
414 | { |
415 | struct list_head *lh; | |
416 | pgtable_t pgtable; | |
417 | pte_t *ptep; | |
418 | ||
ec66ad66 | 419 | assert_spin_locked(pmd_lockptr(mm, pmdp)); |
9501d09f GS |
420 | |
421 | /* FIFO */ | |
c389a250 | 422 | pgtable = pmd_huge_pte(mm, pmdp); |
9501d09f GS |
423 | lh = (struct list_head *) pgtable; |
424 | if (list_empty(lh)) | |
c389a250 | 425 | pmd_huge_pte(mm, pmdp) = NULL; |
9501d09f | 426 | else { |
c389a250 | 427 | pmd_huge_pte(mm, pmdp) = (pgtable_t) lh->next; |
9501d09f GS |
428 | list_del(lh); |
429 | } | |
430 | ptep = (pte_t *) pgtable; | |
e5098611 | 431 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f | 432 | ptep++; |
e5098611 | 433 | pte_val(*ptep) = _PAGE_INVALID; |
9501d09f GS |
434 | return pgtable; |
435 | } | |
75077afb | 436 | #endif /* CONFIG_TRANSPARENT_HUGEPAGE */ |
1e133ab2 MS |
437 | |
438 | #ifdef CONFIG_PGSTE | |
439 | void ptep_set_pte_at(struct mm_struct *mm, unsigned long addr, | |
440 | pte_t *ptep, pte_t entry) | |
441 | { | |
442 | pgste_t pgste; | |
443 | ||
444 | /* the mm_has_pgste() check is done in set_pte_at() */ | |
a9809407 | 445 | preempt_disable(); |
1e133ab2 MS |
446 | pgste = pgste_get_lock(ptep); |
447 | pgste_val(pgste) &= ~_PGSTE_GPS_ZERO; | |
448 | pgste_set_key(ptep, pgste, entry, mm); | |
449 | pgste = pgste_set_pte(ptep, pgste, entry); | |
450 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 451 | preempt_enable(); |
1e133ab2 MS |
452 | } |
453 | ||
454 | void ptep_set_notify(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
455 | { | |
456 | pgste_t pgste; | |
457 | ||
a9809407 | 458 | preempt_disable(); |
1e133ab2 MS |
459 | pgste = pgste_get_lock(ptep); |
460 | pgste_val(pgste) |= PGSTE_IN_BIT; | |
461 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 462 | preempt_enable(); |
1e133ab2 MS |
463 | } |
464 | ||
b2d73b2a MS |
465 | /** |
466 | * ptep_force_prot - change access rights of a locked pte | |
467 | * @mm: pointer to the process mm_struct | |
468 | * @addr: virtual address in the guest address space | |
469 | * @ptep: pointer to the page table entry | |
470 | * @prot: indicates guest access rights: PROT_NONE, PROT_READ or PROT_WRITE | |
4be130a0 | 471 | * @bit: pgste bit to set (e.g. for notification) |
b2d73b2a MS |
472 | * |
473 | * Returns 0 if the access rights were changed and -EAGAIN if the current | |
474 | * and requested access rights are incompatible. | |
475 | */ | |
476 | int ptep_force_prot(struct mm_struct *mm, unsigned long addr, | |
4be130a0 | 477 | pte_t *ptep, int prot, unsigned long bit) |
b2d73b2a MS |
478 | { |
479 | pte_t entry; | |
480 | pgste_t pgste; | |
481 | int pte_i, pte_p; | |
482 | ||
483 | pgste = pgste_get_lock(ptep); | |
484 | entry = *ptep; | |
485 | /* Check pte entry after all locks have been acquired */ | |
486 | pte_i = pte_val(entry) & _PAGE_INVALID; | |
487 | pte_p = pte_val(entry) & _PAGE_PROTECT; | |
488 | if ((pte_i && (prot != PROT_NONE)) || | |
489 | (pte_p && (prot & PROT_WRITE))) { | |
490 | pgste_set_unlock(ptep, pgste); | |
491 | return -EAGAIN; | |
492 | } | |
4be130a0 | 493 | /* Change access rights and set pgste bit */ |
b2d73b2a MS |
494 | if (prot == PROT_NONE && !pte_i) { |
495 | ptep_flush_direct(mm, addr, ptep); | |
496 | pgste = pgste_update_all(entry, pgste, mm); | |
497 | pte_val(entry) |= _PAGE_INVALID; | |
498 | } | |
499 | if (prot == PROT_READ && !pte_p) { | |
500 | ptep_flush_direct(mm, addr, ptep); | |
501 | pte_val(entry) &= ~_PAGE_INVALID; | |
502 | pte_val(entry) |= _PAGE_PROTECT; | |
503 | } | |
4be130a0 | 504 | pgste_val(pgste) |= bit; |
b2d73b2a MS |
505 | pgste = pgste_set_pte(ptep, pgste, entry); |
506 | pgste_set_unlock(ptep, pgste); | |
507 | return 0; | |
508 | } | |
509 | ||
4be130a0 | 510 | int ptep_shadow_pte(struct mm_struct *mm, unsigned long saddr, |
a9d23e71 | 511 | pte_t *sptep, pte_t *tptep, pte_t pte) |
4be130a0 MS |
512 | { |
513 | pgste_t spgste, tpgste; | |
514 | pte_t spte, tpte; | |
515 | int rc = -EAGAIN; | |
516 | ||
a9d23e71 DH |
517 | if (!(pte_val(*tptep) & _PAGE_INVALID)) |
518 | return 0; /* already shadowed */ | |
4be130a0 MS |
519 | spgste = pgste_get_lock(sptep); |
520 | spte = *sptep; | |
521 | if (!(pte_val(spte) & _PAGE_INVALID) && | |
a9d23e71 DH |
522 | !((pte_val(spte) & _PAGE_PROTECT) && |
523 | !(pte_val(pte) & _PAGE_PROTECT))) { | |
4be130a0 MS |
524 | pgste_val(spgste) |= PGSTE_VSIE_BIT; |
525 | tpgste = pgste_get_lock(tptep); | |
526 | pte_val(tpte) = (pte_val(spte) & PAGE_MASK) | | |
a9d23e71 | 527 | (pte_val(pte) & _PAGE_PROTECT); |
4be130a0 MS |
528 | /* don't touch the storage key - it belongs to parent pgste */ |
529 | tpgste = pgste_set_pte(tptep, tpgste, tpte); | |
530 | pgste_set_unlock(tptep, tpgste); | |
a9d23e71 | 531 | rc = 1; |
4be130a0 MS |
532 | } |
533 | pgste_set_unlock(sptep, spgste); | |
534 | return rc; | |
535 | } | |
536 | ||
537 | void ptep_unshadow_pte(struct mm_struct *mm, unsigned long saddr, pte_t *ptep) | |
538 | { | |
539 | pgste_t pgste; | |
540 | ||
541 | pgste = pgste_get_lock(ptep); | |
542 | /* notifier is called by the caller */ | |
543 | ptep_flush_direct(mm, saddr, ptep); | |
544 | /* don't touch the storage key - it belongs to parent pgste */ | |
545 | pgste = pgste_set_pte(ptep, pgste, __pte(_PAGE_INVALID)); | |
546 | pgste_set_unlock(ptep, pgste); | |
547 | } | |
548 | ||
1e133ab2 MS |
549 | static void ptep_zap_swap_entry(struct mm_struct *mm, swp_entry_t entry) |
550 | { | |
551 | if (!non_swap_entry(entry)) | |
552 | dec_mm_counter(mm, MM_SWAPENTS); | |
553 | else if (is_migration_entry(entry)) { | |
554 | struct page *page = migration_entry_to_page(entry); | |
555 | ||
556 | dec_mm_counter(mm, mm_counter(page)); | |
557 | } | |
558 | free_swap_and_cache(entry); | |
559 | } | |
560 | ||
561 | void ptep_zap_unused(struct mm_struct *mm, unsigned long addr, | |
562 | pte_t *ptep, int reset) | |
563 | { | |
564 | unsigned long pgstev; | |
565 | pgste_t pgste; | |
566 | pte_t pte; | |
567 | ||
568 | /* Zap unused and logically-zero pages */ | |
a9809407 | 569 | preempt_disable(); |
1e133ab2 MS |
570 | pgste = pgste_get_lock(ptep); |
571 | pgstev = pgste_val(pgste); | |
572 | pte = *ptep; | |
1c343f7b | 573 | if (!reset && pte_swap(pte) && |
1e133ab2 MS |
574 | ((pgstev & _PGSTE_GPS_USAGE_MASK) == _PGSTE_GPS_USAGE_UNUSED || |
575 | (pgstev & _PGSTE_GPS_ZERO))) { | |
576 | ptep_zap_swap_entry(mm, pte_to_swp_entry(pte)); | |
577 | pte_clear(mm, addr, ptep); | |
578 | } | |
579 | if (reset) | |
580 | pgste_val(pgste) &= ~_PGSTE_GPS_USAGE_MASK; | |
581 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 582 | preempt_enable(); |
1e133ab2 MS |
583 | } |
584 | ||
585 | void ptep_zap_key(struct mm_struct *mm, unsigned long addr, pte_t *ptep) | |
586 | { | |
587 | unsigned long ptev; | |
588 | pgste_t pgste; | |
589 | ||
590 | /* Clear storage key */ | |
a9809407 | 591 | preempt_disable(); |
1e133ab2 MS |
592 | pgste = pgste_get_lock(ptep); |
593 | pgste_val(pgste) &= ~(PGSTE_ACC_BITS | PGSTE_FP_BIT | | |
594 | PGSTE_GR_BIT | PGSTE_GC_BIT); | |
595 | ptev = pte_val(*ptep); | |
596 | if (!(ptev & _PAGE_INVALID) && (ptev & _PAGE_WRITE)) | |
597 | page_set_storage_key(ptev & PAGE_MASK, PAGE_DEFAULT_KEY, 1); | |
598 | pgste_set_unlock(ptep, pgste); | |
a9809407 | 599 | preempt_enable(); |
1e133ab2 MS |
600 | } |
601 | ||
602 | /* | |
603 | * Test and reset if a guest page is dirty | |
604 | */ | |
605 | bool test_and_clear_guest_dirty(struct mm_struct *mm, unsigned long addr) | |
606 | { | |
607 | spinlock_t *ptl; | |
608 | pgste_t pgste; | |
609 | pte_t *ptep; | |
610 | pte_t pte; | |
611 | bool dirty; | |
612 | ||
613 | ptep = get_locked_pte(mm, addr, &ptl); | |
614 | if (unlikely(!ptep)) | |
615 | return false; | |
616 | ||
617 | pgste = pgste_get_lock(ptep); | |
618 | dirty = !!(pgste_val(pgste) & PGSTE_UC_BIT); | |
619 | pgste_val(pgste) &= ~PGSTE_UC_BIT; | |
620 | pte = *ptep; | |
621 | if (dirty && (pte_val(pte) & _PAGE_PRESENT)) { | |
b2d73b2a | 622 | pgste = pgste_pte_notify(mm, addr, ptep, pgste); |
1e133ab2 MS |
623 | __ptep_ipte(addr, ptep); |
624 | if (MACHINE_HAS_ESOP || !(pte_val(pte) & _PAGE_WRITE)) | |
625 | pte_val(pte) |= _PAGE_PROTECT; | |
626 | else | |
627 | pte_val(pte) |= _PAGE_INVALID; | |
628 | *ptep = pte; | |
629 | } | |
630 | pgste_set_unlock(ptep, pgste); | |
631 | ||
632 | spin_unlock(ptl); | |
633 | return dirty; | |
634 | } | |
635 | EXPORT_SYMBOL_GPL(test_and_clear_guest_dirty); | |
636 | ||
637 | int set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
638 | unsigned char key, bool nq) | |
639 | { | |
640 | unsigned long keyul; | |
641 | spinlock_t *ptl; | |
642 | pgste_t old, new; | |
643 | pte_t *ptep; | |
644 | ||
1e133ab2 | 645 | ptep = get_locked_pte(mm, addr, &ptl); |
d3ed1cee | 646 | if (unlikely(!ptep)) |
1e133ab2 | 647 | return -EFAULT; |
1e133ab2 MS |
648 | |
649 | new = old = pgste_get_lock(ptep); | |
650 | pgste_val(new) &= ~(PGSTE_GR_BIT | PGSTE_GC_BIT | | |
651 | PGSTE_ACC_BITS | PGSTE_FP_BIT); | |
652 | keyul = (unsigned long) key; | |
653 | pgste_val(new) |= (keyul & (_PAGE_CHANGED | _PAGE_REFERENCED)) << 48; | |
654 | pgste_val(new) |= (keyul & (_PAGE_ACC_BITS | _PAGE_FP_BIT)) << 56; | |
655 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { | |
656 | unsigned long address, bits, skey; | |
657 | ||
658 | address = pte_val(*ptep) & PAGE_MASK; | |
659 | skey = (unsigned long) page_get_storage_key(address); | |
660 | bits = skey & (_PAGE_CHANGED | _PAGE_REFERENCED); | |
661 | skey = key & (_PAGE_ACC_BITS | _PAGE_FP_BIT); | |
662 | /* Set storage key ACC and FP */ | |
663 | page_set_storage_key(address, skey, !nq); | |
664 | /* Merge host changed & referenced into pgste */ | |
665 | pgste_val(new) |= bits << 52; | |
666 | } | |
667 | /* changing the guest storage key is considered a change of the page */ | |
668 | if ((pgste_val(new) ^ pgste_val(old)) & | |
669 | (PGSTE_ACC_BITS | PGSTE_FP_BIT | PGSTE_GR_BIT | PGSTE_GC_BIT)) | |
670 | pgste_val(new) |= PGSTE_UC_BIT; | |
671 | ||
672 | pgste_set_unlock(ptep, new); | |
673 | pte_unmap_unlock(ptep, ptl); | |
1e133ab2 MS |
674 | return 0; |
675 | } | |
676 | EXPORT_SYMBOL(set_guest_storage_key); | |
677 | ||
1824c723 DH |
678 | /** |
679 | * Conditionally set a guest storage key (handling csske). | |
680 | * oldkey will be updated when either mr or mc is set and a pointer is given. | |
681 | * | |
682 | * Returns 0 if a guests storage key update wasn't necessary, 1 if the guest | |
683 | * storage key was updated and -EFAULT on access errors. | |
684 | */ | |
685 | int cond_set_guest_storage_key(struct mm_struct *mm, unsigned long addr, | |
686 | unsigned char key, unsigned char *oldkey, | |
687 | bool nq, bool mr, bool mc) | |
688 | { | |
689 | unsigned char tmp, mask = _PAGE_ACC_BITS | _PAGE_FP_BIT; | |
690 | int rc; | |
691 | ||
692 | /* we can drop the pgste lock between getting and setting the key */ | |
693 | if (mr | mc) { | |
694 | rc = get_guest_storage_key(current->mm, addr, &tmp); | |
695 | if (rc) | |
696 | return rc; | |
697 | if (oldkey) | |
698 | *oldkey = tmp; | |
699 | if (!mr) | |
700 | mask |= _PAGE_REFERENCED; | |
701 | if (!mc) | |
702 | mask |= _PAGE_CHANGED; | |
703 | if (!((tmp ^ key) & mask)) | |
704 | return 0; | |
705 | } | |
706 | rc = set_guest_storage_key(current->mm, addr, key, nq); | |
707 | return rc < 0 ? rc : 1; | |
708 | } | |
709 | EXPORT_SYMBOL(cond_set_guest_storage_key); | |
710 | ||
a7e19ab5 DH |
711 | /** |
712 | * Reset a guest reference bit (rrbe), returning the reference and changed bit. | |
713 | * | |
714 | * Returns < 0 in case of error, otherwise the cc to be reported to the guest. | |
715 | */ | |
716 | int reset_guest_reference_bit(struct mm_struct *mm, unsigned long addr) | |
1e133ab2 | 717 | { |
1e133ab2 | 718 | spinlock_t *ptl; |
a7e19ab5 | 719 | pgste_t old, new; |
1e133ab2 | 720 | pte_t *ptep; |
a7e19ab5 | 721 | int cc = 0; |
1e133ab2 | 722 | |
1e133ab2 | 723 | ptep = get_locked_pte(mm, addr, &ptl); |
a7e19ab5 | 724 | if (unlikely(!ptep)) |
1e133ab2 | 725 | return -EFAULT; |
1e133ab2 | 726 | |
a7e19ab5 DH |
727 | new = old = pgste_get_lock(ptep); |
728 | /* Reset guest reference bit only */ | |
729 | pgste_val(new) &= ~PGSTE_GR_BIT; | |
1e133ab2 | 730 | |
a7e19ab5 DH |
731 | if (!(pte_val(*ptep) & _PAGE_INVALID)) { |
732 | cc = page_reset_referenced(pte_val(*ptep) & PAGE_MASK); | |
733 | /* Merge real referenced bit into host-set */ | |
734 | pgste_val(new) |= ((unsigned long) cc << 53) & PGSTE_HR_BIT; | |
1e133ab2 | 735 | } |
a7e19ab5 DH |
736 | /* Reflect guest's logical view, not physical */ |
737 | cc |= (pgste_val(old) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 49; | |
738 | /* Changing the guest storage key is considered a change of the page */ | |
739 | if ((pgste_val(new) ^ pgste_val(old)) & PGSTE_GR_BIT) | |
740 | pgste_val(new) |= PGSTE_UC_BIT; | |
741 | ||
742 | pgste_set_unlock(ptep, new); | |
743 | pte_unmap_unlock(ptep, ptl); | |
744 | return 0; | |
745 | } | |
746 | EXPORT_SYMBOL(reset_guest_reference_bit); | |
747 | ||
154c8c19 DH |
748 | int get_guest_storage_key(struct mm_struct *mm, unsigned long addr, |
749 | unsigned char *key) | |
1e133ab2 | 750 | { |
1e133ab2 MS |
751 | spinlock_t *ptl; |
752 | pgste_t pgste; | |
753 | pte_t *ptep; | |
1e133ab2 | 754 | |
1e133ab2 | 755 | ptep = get_locked_pte(mm, addr, &ptl); |
d3ed1cee | 756 | if (unlikely(!ptep)) |
1e133ab2 | 757 | return -EFAULT; |
1e133ab2 | 758 | |
d3ed1cee | 759 | pgste = pgste_get_lock(ptep); |
154c8c19 | 760 | *key = (pgste_val(pgste) & (PGSTE_ACC_BITS | PGSTE_FP_BIT)) >> 56; |
8d6037a7 | 761 | if (!(pte_val(*ptep) & _PAGE_INVALID)) |
154c8c19 | 762 | *key = page_get_storage_key(pte_val(*ptep) & PAGE_MASK); |
8d6037a7 | 763 | /* Reflect guest's logical view, not physical */ |
154c8c19 | 764 | *key |= (pgste_val(pgste) & (PGSTE_GR_BIT | PGSTE_GC_BIT)) >> 48; |
1e133ab2 MS |
765 | pgste_set_unlock(ptep, pgste); |
766 | pte_unmap_unlock(ptep, ptl); | |
154c8c19 | 767 | return 0; |
1e133ab2 MS |
768 | } |
769 | EXPORT_SYMBOL(get_guest_storage_key); | |
770 | #endif |