Commit | Line | Data |
---|---|---|
305b1523 JG |
1 | /* |
2 | * Copyright IBM Corp. 2011 | |
3 | * Author(s): Jan Glauber <jang@linux.vnet.ibm.com> | |
4 | */ | |
6b70a920 | 5 | #include <linux/hugetlb.h> |
305b1523 JG |
6 | #include <linux/module.h> |
7 | #include <linux/mm.h> | |
638ad34a | 8 | #include <asm/cacheflush.h> |
cfb0b241 | 9 | #include <asm/facility.h> |
305b1523 | 10 | #include <asm/pgtable.h> |
6b70a920 HC |
11 | #include <asm/page.h> |
12 | ||
f7f8d7e5 HC |
13 | static inline unsigned long sske_frame(unsigned long addr, unsigned char skey) |
14 | { | |
15 | asm volatile(".insn rrf,0xb22b0000,%[skey],%[addr],9,0" | |
16 | : [addr] "+a" (addr) : [skey] "d" (skey)); | |
17 | return addr; | |
18 | } | |
19 | ||
127c1fef | 20 | void __storage_key_init_range(unsigned long start, unsigned long end) |
6b70a920 | 21 | { |
f7f8d7e5 | 22 | unsigned long boundary, size; |
6b70a920 | 23 | |
de3fa841 HC |
24 | if (!PAGE_DEFAULT_KEY) |
25 | return; | |
6b70a920 | 26 | while (start < end) { |
6b70a920 HC |
27 | if (MACHINE_HAS_EDAT1) { |
28 | /* set storage keys for a 1MB frame */ | |
6b70a920 HC |
29 | size = 1UL << 20; |
30 | boundary = (start + size) & ~(size - 1); | |
31 | if (boundary <= end) { | |
32 | do { | |
f7f8d7e5 | 33 | start = sske_frame(start, PAGE_DEFAULT_KEY); |
6b70a920 HC |
34 | } while (start < boundary); |
35 | continue; | |
36 | } | |
37 | } | |
38 | page_set_storage_key(start, PAGE_DEFAULT_KEY, 0); | |
39 | start += PAGE_SIZE; | |
40 | } | |
41 | } | |
305b1523 | 42 | |
37cd944c HC |
43 | #ifdef CONFIG_PROC_FS |
44 | atomic_long_t direct_pages_count[PG_DIRECT_MAP_MAX]; | |
45 | ||
46 | void arch_report_meminfo(struct seq_file *m) | |
47 | { | |
48 | seq_printf(m, "DirectMap4k: %8lu kB\n", | |
49 | atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_4K]) << 2); | |
50 | seq_printf(m, "DirectMap1M: %8lu kB\n", | |
51 | atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_1M]) << 10); | |
52 | seq_printf(m, "DirectMap2G: %8lu kB\n", | |
53 | atomic_long_read(&direct_pages_count[PG_DIRECT_MAP_2G]) << 21); | |
54 | } | |
55 | #endif /* CONFIG_PROC_FS */ | |
56 | ||
e8a97e42 HC |
57 | static void pgt_set(unsigned long *old, unsigned long new, unsigned long addr, |
58 | unsigned long dtt) | |
5b1ba9e3 | 59 | { |
e8a97e42 HC |
60 | unsigned long table, mask; |
61 | ||
62 | mask = 0; | |
63 | if (MACHINE_HAS_EDAT2) { | |
64 | switch (dtt) { | |
65 | case CRDTE_DTT_REGION3: | |
66 | mask = ~(PTRS_PER_PUD * sizeof(pud_t) - 1); | |
67 | break; | |
68 | case CRDTE_DTT_SEGMENT: | |
69 | mask = ~(PTRS_PER_PMD * sizeof(pmd_t) - 1); | |
70 | break; | |
71 | case CRDTE_DTT_PAGE: | |
72 | mask = ~(PTRS_PER_PTE * sizeof(pte_t) - 1); | |
73 | break; | |
74 | } | |
75 | table = (unsigned long)old & mask; | |
76 | crdte(*old, new, table, dtt, addr, S390_lowcore.kernel_asce); | |
77 | } else if (MACHINE_HAS_IDTE) { | |
78 | cspg(old, *old, new); | |
79 | } else { | |
80 | csp((unsigned int *)old + 1, *old, new); | |
81 | } | |
82 | } | |
83 | ||
84 | struct cpa { | |
85 | unsigned int set_ro : 1; | |
86 | unsigned int clear_ro : 1; | |
87 | }; | |
88 | ||
89 | static int walk_pte_level(pmd_t *pmdp, unsigned long addr, unsigned long end, | |
90 | struct cpa cpa) | |
91 | { | |
92 | pte_t *ptep, new; | |
93 | ||
94 | ptep = pte_offset(pmdp, addr); | |
95 | do { | |
96 | if (pte_none(*ptep)) | |
97 | return -EINVAL; | |
98 | if (cpa.set_ro) | |
99 | new = pte_wrprotect(*ptep); | |
100 | else if (cpa.clear_ro) | |
101 | new = pte_mkwrite(pte_mkdirty(*ptep)); | |
102 | pgt_set((unsigned long *)ptep, pte_val(new), addr, CRDTE_DTT_PAGE); | |
103 | ptep++; | |
104 | addr += PAGE_SIZE; | |
105 | cond_resched(); | |
106 | } while (addr < end); | |
107 | return 0; | |
108 | } | |
109 | ||
110 | static int split_pmd_page(pmd_t *pmdp, unsigned long addr) | |
111 | { | |
112 | unsigned long pte_addr, prot; | |
113 | pte_t *pt_dir, *ptep; | |
114 | pmd_t new; | |
115 | int i, ro; | |
116 | ||
117 | pt_dir = vmem_pte_alloc(); | |
118 | if (!pt_dir) | |
119 | return -ENOMEM; | |
120 | pte_addr = pmd_pfn(*pmdp) << PAGE_SHIFT; | |
121 | ro = !!(pmd_val(*pmdp) & _SEGMENT_ENTRY_PROTECT); | |
122 | prot = pgprot_val(ro ? PAGE_KERNEL_RO : PAGE_KERNEL); | |
123 | ptep = pt_dir; | |
124 | for (i = 0; i < PTRS_PER_PTE; i++) { | |
125 | pte_val(*ptep) = pte_addr | prot; | |
126 | pte_addr += PAGE_SIZE; | |
127 | ptep++; | |
128 | } | |
129 | pmd_val(new) = __pa(pt_dir) | _SEGMENT_ENTRY; | |
130 | pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); | |
37cd944c HC |
131 | update_page_count(PG_DIRECT_MAP_4K, PTRS_PER_PTE); |
132 | update_page_count(PG_DIRECT_MAP_1M, -1); | |
e8a97e42 HC |
133 | return 0; |
134 | } | |
135 | ||
136 | static void modify_pmd_page(pmd_t *pmdp, unsigned long addr, struct cpa cpa) | |
137 | { | |
138 | pmd_t new; | |
139 | ||
140 | if (cpa.set_ro) | |
141 | new = pmd_wrprotect(*pmdp); | |
142 | else if (cpa.clear_ro) | |
143 | new = pmd_mkwrite(pmd_mkdirty(*pmdp)); | |
144 | pgt_set((unsigned long *)pmdp, pmd_val(new), addr, CRDTE_DTT_SEGMENT); | |
145 | } | |
146 | ||
147 | static int walk_pmd_level(pud_t *pudp, unsigned long addr, unsigned long end, | |
148 | struct cpa cpa) | |
149 | { | |
150 | unsigned long next; | |
5b1ba9e3 | 151 | pmd_t *pmdp; |
e8a97e42 | 152 | int rc = 0; |
5b1ba9e3 | 153 | |
5b1ba9e3 | 154 | pmdp = pmd_offset(pudp, addr); |
e8a97e42 HC |
155 | do { |
156 | if (pmd_none(*pmdp)) | |
157 | return -EINVAL; | |
158 | next = pmd_addr_end(addr, end); | |
159 | if (pmd_large(*pmdp)) { | |
160 | if (addr & ~PMD_MASK || addr + PMD_SIZE > next) { | |
161 | rc = split_pmd_page(pmdp, addr); | |
162 | if (rc) | |
163 | return rc; | |
164 | continue; | |
165 | } | |
166 | modify_pmd_page(pmdp, addr, cpa); | |
167 | } else { | |
168 | rc = walk_pte_level(pmdp, addr, next, cpa); | |
169 | if (rc) | |
170 | return rc; | |
171 | } | |
172 | pmdp++; | |
173 | addr = next; | |
174 | cond_resched(); | |
175 | } while (addr < end); | |
176 | return rc; | |
5b1ba9e3 HC |
177 | } |
178 | ||
e8a97e42 | 179 | static int split_pud_page(pud_t *pudp, unsigned long addr) |
305b1523 | 180 | { |
e8a97e42 HC |
181 | unsigned long pmd_addr, prot; |
182 | pmd_t *pm_dir, *pmdp; | |
183 | pud_t new; | |
184 | int i, ro; | |
305b1523 | 185 | |
e8a97e42 HC |
186 | pm_dir = vmem_pmd_alloc(); |
187 | if (!pm_dir) | |
188 | return -ENOMEM; | |
189 | pmd_addr = pud_pfn(*pudp) << PAGE_SHIFT; | |
190 | ro = !!(pud_val(*pudp) & _REGION_ENTRY_PROTECT); | |
191 | prot = pgprot_val(ro ? SEGMENT_KERNEL_RO : SEGMENT_KERNEL); | |
192 | pmdp = pm_dir; | |
193 | for (i = 0; i < PTRS_PER_PMD; i++) { | |
194 | pmd_val(*pmdp) = pmd_addr | prot; | |
195 | pmd_addr += PMD_SIZE; | |
196 | pmdp++; | |
305b1523 | 197 | } |
e8a97e42 HC |
198 | pud_val(new) = __pa(pm_dir) | _REGION3_ENTRY; |
199 | pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); | |
37cd944c HC |
200 | update_page_count(PG_DIRECT_MAP_1M, PTRS_PER_PMD); |
201 | update_page_count(PG_DIRECT_MAP_2G, -1); | |
e8a97e42 HC |
202 | return 0; |
203 | } | |
204 | ||
205 | static void modify_pud_page(pud_t *pudp, unsigned long addr, struct cpa cpa) | |
206 | { | |
207 | pud_t new; | |
208 | ||
209 | if (cpa.set_ro) | |
210 | new = pud_wrprotect(*pudp); | |
211 | else if (cpa.clear_ro) | |
212 | new = pud_mkwrite(pud_mkdirty(*pudp)); | |
213 | pgt_set((unsigned long *)pudp, pud_val(new), addr, CRDTE_DTT_REGION3); | |
214 | } | |
215 | ||
216 | static int walk_pud_level(pgd_t *pgd, unsigned long addr, unsigned long end, | |
217 | struct cpa cpa) | |
218 | { | |
219 | unsigned long next; | |
220 | pud_t *pudp; | |
221 | int rc = 0; | |
222 | ||
223 | pudp = pud_offset(pgd, addr); | |
224 | do { | |
225 | if (pud_none(*pudp)) | |
226 | return -EINVAL; | |
227 | next = pud_addr_end(addr, end); | |
228 | if (pud_large(*pudp)) { | |
229 | if (addr & ~PUD_MASK || addr + PUD_SIZE > next) { | |
230 | rc = split_pud_page(pudp, addr); | |
231 | if (rc) | |
232 | break; | |
233 | continue; | |
234 | } | |
235 | modify_pud_page(pudp, addr, cpa); | |
236 | } else { | |
237 | rc = walk_pmd_level(pudp, addr, next, cpa); | |
238 | } | |
239 | pudp++; | |
240 | addr = next; | |
241 | cond_resched(); | |
242 | } while (addr < end && !rc); | |
243 | return rc; | |
244 | } | |
245 | ||
246 | static DEFINE_MUTEX(cpa_mutex); | |
247 | ||
248 | static int change_page_attr(unsigned long addr, unsigned long end, | |
249 | struct cpa cpa) | |
250 | { | |
251 | unsigned long next; | |
252 | int rc = -EINVAL; | |
253 | pgd_t *pgdp; | |
254 | ||
4d81aaa5 HC |
255 | if (addr == end) |
256 | return 0; | |
e8a97e42 HC |
257 | if (end >= MODULES_END) |
258 | return -EINVAL; | |
259 | mutex_lock(&cpa_mutex); | |
260 | pgdp = pgd_offset_k(addr); | |
261 | do { | |
262 | if (pgd_none(*pgdp)) | |
263 | break; | |
264 | next = pgd_addr_end(addr, end); | |
265 | rc = walk_pud_level(pgdp, addr, next, cpa); | |
266 | if (rc) | |
267 | break; | |
268 | cond_resched(); | |
269 | } while (pgdp++, addr = next, addr < end && !rc); | |
270 | mutex_unlock(&cpa_mutex); | |
271 | return rc; | |
305b1523 JG |
272 | } |
273 | ||
274 | int set_memory_ro(unsigned long addr, int numpages) | |
275 | { | |
e8a97e42 HC |
276 | struct cpa cpa = { |
277 | .set_ro = 1, | |
278 | }; | |
279 | ||
280 | addr &= PAGE_MASK; | |
281 | return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa); | |
305b1523 | 282 | } |
305b1523 JG |
283 | |
284 | int set_memory_rw(unsigned long addr, int numpages) | |
285 | { | |
e8a97e42 HC |
286 | struct cpa cpa = { |
287 | .clear_ro = 1, | |
288 | }; | |
289 | ||
290 | addr &= PAGE_MASK; | |
291 | return change_page_attr(addr, addr + numpages * PAGE_SIZE, cpa); | |
305b1523 | 292 | } |
305b1523 JG |
293 | |
294 | /* not possible */ | |
295 | int set_memory_nx(unsigned long addr, int numpages) | |
296 | { | |
297 | return 0; | |
298 | } | |
448694a1 JG |
299 | |
300 | int set_memory_x(unsigned long addr, int numpages) | |
301 | { | |
302 | return 0; | |
303 | } | |
0a4ccc99 HC |
304 | |
305 | #ifdef CONFIG_DEBUG_PAGEALLOC | |
cfb0b241 HC |
306 | |
307 | static void ipte_range(pte_t *pte, unsigned long address, int nr) | |
308 | { | |
309 | int i; | |
310 | ||
5a79859a | 311 | if (test_facility(13)) { |
34eeaf37 | 312 | __ptep_ipte_range(address, nr - 1, pte, IPTE_GLOBAL); |
cfb0b241 HC |
313 | return; |
314 | } | |
315 | for (i = 0; i < nr; i++) { | |
34eeaf37 | 316 | __ptep_ipte(address, pte, IPTE_GLOBAL); |
cfb0b241 HC |
317 | address += PAGE_SIZE; |
318 | pte++; | |
319 | } | |
320 | } | |
321 | ||
031bc574 | 322 | void __kernel_map_pages(struct page *page, int numpages, int enable) |
0a4ccc99 HC |
323 | { |
324 | unsigned long address; | |
cfb0b241 | 325 | int nr, i, j; |
0a4ccc99 HC |
326 | pgd_t *pgd; |
327 | pud_t *pud; | |
328 | pmd_t *pmd; | |
329 | pte_t *pte; | |
0a4ccc99 | 330 | |
cfb0b241 | 331 | for (i = 0; i < numpages;) { |
0a4ccc99 HC |
332 | address = page_to_phys(page + i); |
333 | pgd = pgd_offset_k(address); | |
334 | pud = pud_offset(pgd, address); | |
335 | pmd = pmd_offset(pud, address); | |
336 | pte = pte_offset_kernel(pmd, address); | |
cfb0b241 HC |
337 | nr = (unsigned long)pte >> ilog2(sizeof(long)); |
338 | nr = PTRS_PER_PTE - (nr & (PTRS_PER_PTE - 1)); | |
339 | nr = min(numpages - i, nr); | |
340 | if (enable) { | |
341 | for (j = 0; j < nr; j++) { | |
3e76ee99 | 342 | pte_val(*pte) = address | pgprot_val(PAGE_KERNEL); |
cfb0b241 HC |
343 | address += PAGE_SIZE; |
344 | pte++; | |
345 | } | |
346 | } else { | |
347 | ipte_range(pte, address, nr); | |
0a4ccc99 | 348 | } |
cfb0b241 | 349 | i += nr; |
0a4ccc99 HC |
350 | } |
351 | } | |
352 | ||
353 | #ifdef CONFIG_HIBERNATION | |
354 | bool kernel_page_present(struct page *page) | |
355 | { | |
356 | unsigned long addr; | |
357 | int cc; | |
358 | ||
359 | addr = page_to_phys(page); | |
360 | asm volatile( | |
361 | " lra %1,0(%1)\n" | |
362 | " ipm %0\n" | |
363 | " srl %0,28" | |
364 | : "=d" (cc), "+a" (addr) : : "cc"); | |
365 | return cc == 0; | |
366 | } | |
367 | #endif /* CONFIG_HIBERNATION */ | |
368 | ||
369 | #endif /* CONFIG_DEBUG_PAGEALLOC */ |