mm: thp: Use more portable PMD clearing sequenece in zap_huge_pmd().
[deliverable/linux.git] / arch / sparc / mm / tsb.c
CommitLineData
74bf4312
DM
1/* arch/sparc64/mm/tsb.c
2 *
a3cf5e6b 3 * Copyright (C) 2006, 2008 David S. Miller <davem@davemloft.net>
74bf4312
DM
4 */
5
6#include <linux/kernel.h>
a3cf5e6b 7#include <linux/preempt.h>
5a0e3ad6 8#include <linux/slab.h>
74bf4312
DM
9#include <asm/page.h>
10#include <asm/tlbflush.h>
11#include <asm/tlb.h>
09f94287 12#include <asm/mmu_context.h>
98c5584c 13#include <asm/pgtable.h>
bd40791e 14#include <asm/tsb.h>
9b4006dc 15#include <asm/oplib.h>
74bf4312 16
74bf4312
DM
17extern struct tsb swapper_tsb[KERNEL_TSB_NENTRIES];
18
dcc1e8dd 19static inline unsigned long tsb_hash(unsigned long vaddr, unsigned long hash_shift, unsigned long nentries)
74bf4312 20{
dcc1e8dd 21 vaddr >>= hash_shift;
98c5584c 22 return vaddr & (nentries - 1);
74bf4312
DM
23}
24
8b234274 25static inline int tag_compare(unsigned long tag, unsigned long vaddr)
74bf4312 26{
8b234274 27 return (tag == (vaddr >> 22));
74bf4312
DM
28}
29
30/* TSB flushes need only occur on the processor initiating the address
31 * space modification, not on each cpu the address space has run on.
32 * Only the TLB flush needs that treatment.
33 */
34
35void flush_tsb_kernel_range(unsigned long start, unsigned long end)
36{
37 unsigned long v;
38
39 for (v = start; v < end; v += PAGE_SIZE) {
dcc1e8dd
DM
40 unsigned long hash = tsb_hash(v, PAGE_SHIFT,
41 KERNEL_TSB_NENTRIES);
98c5584c 42 struct tsb *ent = &swapper_tsb[hash];
74bf4312 43
293666b7 44 if (tag_compare(ent->tag, v))
8b234274 45 ent->tag = (1UL << TSB_TAG_INVALID_BIT);
74bf4312
DM
46 }
47}
48
90f08e39
PZ
49static void __flush_tsb_one(struct tlb_batch *tb, unsigned long hash_shift,
50 unsigned long tsb, unsigned long nentries)
74bf4312 51{
dcc1e8dd 52 unsigned long i;
7a1ac526 53
90f08e39
PZ
54 for (i = 0; i < tb->tlb_nr; i++) {
55 unsigned long v = tb->vaddrs[i];
517af332 56 unsigned long tag, ent, hash;
74bf4312
DM
57
58 v &= ~0x1UL;
59
dcc1e8dd
DM
60 hash = tsb_hash(v, hash_shift, nentries);
61 ent = tsb + (hash * sizeof(struct tsb));
8b234274 62 tag = (v >> 22UL);
517af332
DM
63
64 tsb_flush(ent, tag);
74bf4312 65 }
dcc1e8dd
DM
66}
67
90f08e39 68void flush_tsb_user(struct tlb_batch *tb)
dcc1e8dd 69{
90f08e39 70 struct mm_struct *mm = tb->mm;
dcc1e8dd
DM
71 unsigned long nentries, base, flags;
72
73 spin_lock_irqsave(&mm->context.lock, flags);
7a1ac526 74
dcc1e8dd
DM
75 base = (unsigned long) mm->context.tsb_block[MM_TSB_BASE].tsb;
76 nentries = mm->context.tsb_block[MM_TSB_BASE].tsb_nentries;
77 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
78 base = __pa(base);
90f08e39 79 __flush_tsb_one(tb, PAGE_SHIFT, base, nentries);
dcc1e8dd
DM
80
81#ifdef CONFIG_HUGETLB_PAGE
82 if (mm->context.tsb_block[MM_TSB_HUGE].tsb) {
83 base = (unsigned long) mm->context.tsb_block[MM_TSB_HUGE].tsb;
84 nentries = mm->context.tsb_block[MM_TSB_HUGE].tsb_nentries;
85 if (tlb_type == cheetah_plus || tlb_type == hypervisor)
86 base = __pa(base);
90f08e39 87 __flush_tsb_one(tb, HPAGE_SHIFT, base, nentries);
dcc1e8dd
DM
88 }
89#endif
7a1ac526 90 spin_unlock_irqrestore(&mm->context.lock, flags);
74bf4312 91}
09f94287 92
dcc1e8dd
DM
93#define HV_PGSZ_IDX_BASE HV_PGSZ_IDX_8K
94#define HV_PGSZ_MASK_BASE HV_PGSZ_MASK_8K
dcc1e8dd
DM
95
96#ifdef CONFIG_HUGETLB_PAGE
dcc1e8dd
DM
97#define HV_PGSZ_IDX_HUGE HV_PGSZ_IDX_4MB
98#define HV_PGSZ_MASK_HUGE HV_PGSZ_MASK_4MB
dcc1e8dd
DM
99#endif
100
101static void setup_tsb_params(struct mm_struct *mm, unsigned long tsb_idx, unsigned long tsb_bytes)
98c5584c
DM
102{
103 unsigned long tsb_reg, base, tsb_paddr;
104 unsigned long page_sz, tte;
105
dcc1e8dd
DM
106 mm->context.tsb_block[tsb_idx].tsb_nentries =
107 tsb_bytes / sizeof(struct tsb);
98c5584c
DM
108
109 base = TSBMAP_BASE;
c4bce90e 110 tte = pgprot_val(PAGE_KERNEL_LOCKED);
dcc1e8dd 111 tsb_paddr = __pa(mm->context.tsb_block[tsb_idx].tsb);
517af332 112 BUG_ON(tsb_paddr & (tsb_bytes - 1UL));
98c5584c
DM
113
114 /* Use the smallest page size that can map the whole TSB
115 * in one TLB entry.
116 */
117 switch (tsb_bytes) {
118 case 8192 << 0:
119 tsb_reg = 0x0UL;
120#ifdef DCACHE_ALIASING_POSSIBLE
121 base += (tsb_paddr & 8192);
122#endif
98c5584c
DM
123 page_sz = 8192;
124 break;
125
126 case 8192 << 1:
127 tsb_reg = 0x1UL;
98c5584c
DM
128 page_sz = 64 * 1024;
129 break;
130
131 case 8192 << 2:
132 tsb_reg = 0x2UL;
98c5584c
DM
133 page_sz = 64 * 1024;
134 break;
135
136 case 8192 << 3:
137 tsb_reg = 0x3UL;
98c5584c
DM
138 page_sz = 64 * 1024;
139 break;
140
141 case 8192 << 4:
142 tsb_reg = 0x4UL;
98c5584c
DM
143 page_sz = 512 * 1024;
144 break;
145
146 case 8192 << 5:
147 tsb_reg = 0x5UL;
98c5584c
DM
148 page_sz = 512 * 1024;
149 break;
150
151 case 8192 << 6:
152 tsb_reg = 0x6UL;
98c5584c
DM
153 page_sz = 512 * 1024;
154 break;
155
156 case 8192 << 7:
157 tsb_reg = 0x7UL;
98c5584c
DM
158 page_sz = 4 * 1024 * 1024;
159 break;
bd40791e
DM
160
161 default:
7e5766fa
DM
162 printk(KERN_ERR "TSB[%s:%d]: Impossible TSB size %lu, killing process.\n",
163 current->comm, current->pid, tsb_bytes);
164 do_exit(SIGSEGV);
6cb79b3f 165 }
c4bce90e 166 tte |= pte_sz_bits(page_sz);
98c5584c 167
618e9ed9 168 if (tlb_type == cheetah_plus || tlb_type == hypervisor) {
517af332
DM
169 /* Physical mapping, no locked TLB entry for TSB. */
170 tsb_reg |= tsb_paddr;
171
dcc1e8dd
DM
172 mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg;
173 mm->context.tsb_block[tsb_idx].tsb_map_vaddr = 0;
174 mm->context.tsb_block[tsb_idx].tsb_map_pte = 0;
517af332
DM
175 } else {
176 tsb_reg |= base;
177 tsb_reg |= (tsb_paddr & (page_sz - 1UL));
178 tte |= (tsb_paddr & ~(page_sz - 1UL));
179
dcc1e8dd
DM
180 mm->context.tsb_block[tsb_idx].tsb_reg_val = tsb_reg;
181 mm->context.tsb_block[tsb_idx].tsb_map_vaddr = base;
182 mm->context.tsb_block[tsb_idx].tsb_map_pte = tte;
517af332 183 }
98c5584c 184
618e9ed9
DM
185 /* Setup the Hypervisor TSB descriptor. */
186 if (tlb_type == hypervisor) {
dcc1e8dd 187 struct hv_tsb_descr *hp = &mm->context.tsb_descr[tsb_idx];
618e9ed9 188
dcc1e8dd
DM
189 switch (tsb_idx) {
190 case MM_TSB_BASE:
191 hp->pgsz_idx = HV_PGSZ_IDX_BASE;
618e9ed9 192 break;
dcc1e8dd
DM
193#ifdef CONFIG_HUGETLB_PAGE
194 case MM_TSB_HUGE:
195 hp->pgsz_idx = HV_PGSZ_IDX_HUGE;
618e9ed9 196 break;
dcc1e8dd
DM
197#endif
198 default:
199 BUG();
6cb79b3f 200 }
618e9ed9
DM
201 hp->assoc = 1;
202 hp->num_ttes = tsb_bytes / 16;
203 hp->ctx_idx = 0;
dcc1e8dd
DM
204 switch (tsb_idx) {
205 case MM_TSB_BASE:
206 hp->pgsz_mask = HV_PGSZ_MASK_BASE;
618e9ed9 207 break;
dcc1e8dd
DM
208#ifdef CONFIG_HUGETLB_PAGE
209 case MM_TSB_HUGE:
210 hp->pgsz_mask = HV_PGSZ_MASK_HUGE;
618e9ed9 211 break;
dcc1e8dd
DM
212#endif
213 default:
214 BUG();
6cb79b3f 215 }
618e9ed9
DM
216 hp->tsb_base = tsb_paddr;
217 hp->resv = 0;
218 }
98c5584c
DM
219}
220
4dedbf8d
DM
221struct kmem_cache *pgtable_cache __read_mostly;
222
e18b890b 223static struct kmem_cache *tsb_caches[8] __read_mostly;
9b4006dc
DM
224
225static const char *tsb_cache_names[8] = {
226 "tsb_8KB",
227 "tsb_16KB",
228 "tsb_32KB",
229 "tsb_64KB",
230 "tsb_128KB",
231 "tsb_256KB",
232 "tsb_512KB",
233 "tsb_1MB",
234};
235
3a2cba99 236void __init pgtable_cache_init(void)
9b4006dc
DM
237{
238 unsigned long i;
239
4dedbf8d
DM
240 pgtable_cache = kmem_cache_create("pgtable_cache",
241 PAGE_SIZE, PAGE_SIZE,
242 0,
243 _clear_page);
244 if (!pgtable_cache) {
245 prom_printf("pgtable_cache_init(): Could not create!\n");
246 prom_halt();
247 }
248
9b4006dc
DM
249 for (i = 0; i < 8; i++) {
250 unsigned long size = 8192 << i;
251 const char *name = tsb_cache_names[i];
252
253 tsb_caches[i] = kmem_cache_create(name,
254 size, size,
20c2df83 255 0, NULL);
9b4006dc
DM
256 if (!tsb_caches[i]) {
257 prom_printf("Could not create %s cache\n", name);
258 prom_halt();
259 }
260 }
261}
262
0871420f
DM
263int sysctl_tsb_ratio = -2;
264
265static unsigned long tsb_size_to_rss_limit(unsigned long new_size)
266{
267 unsigned long num_ents = (new_size / sizeof(struct tsb));
268
269 if (sysctl_tsb_ratio < 0)
270 return num_ents - (num_ents >> -sysctl_tsb_ratio);
271 else
272 return num_ents + (num_ents >> sysctl_tsb_ratio);
273}
274
dcc1e8dd
DM
275/* When the RSS of an address space exceeds tsb_rss_limit for a TSB,
276 * do_sparc64_fault() invokes this routine to try and grow it.
7a1ac526 277 *
bd40791e 278 * When we reach the maximum TSB size supported, we stick ~0UL into
dcc1e8dd 279 * tsb_rss_limit for that TSB so the grow checks in do_sparc64_fault()
bd40791e
DM
280 * will not trigger any longer.
281 *
282 * The TSB can be anywhere from 8K to 1MB in size, in increasing powers
283 * of two. The TSB must be aligned to it's size, so f.e. a 512K TSB
b52439c2
DM
284 * must be 512K aligned. It also must be physically contiguous, so we
285 * cannot use vmalloc().
bd40791e
DM
286 *
287 * The idea here is to grow the TSB when the RSS of the process approaches
288 * the number of entries that the current TSB can hold at once. Currently,
289 * we trigger when the RSS hits 3/4 of the TSB capacity.
290 */
dcc1e8dd 291void tsb_grow(struct mm_struct *mm, unsigned long tsb_index, unsigned long rss)
bd40791e
DM
292{
293 unsigned long max_tsb_size = 1 * 1024 * 1024;
9b4006dc 294 unsigned long new_size, old_size, flags;
7a1ac526 295 struct tsb *old_tsb, *new_tsb;
9b4006dc
DM
296 unsigned long new_cache_index, old_cache_index;
297 unsigned long new_rss_limit;
b52439c2 298 gfp_t gfp_flags;
bd40791e
DM
299
300 if (max_tsb_size > (PAGE_SIZE << MAX_ORDER))
301 max_tsb_size = (PAGE_SIZE << MAX_ORDER);
302
9b4006dc
DM
303 new_cache_index = 0;
304 for (new_size = 8192; new_size < max_tsb_size; new_size <<= 1UL) {
0871420f
DM
305 new_rss_limit = tsb_size_to_rss_limit(new_size);
306 if (new_rss_limit > rss)
bd40791e 307 break;
9b4006dc 308 new_cache_index++;
bd40791e
DM
309 }
310
9b4006dc 311 if (new_size == max_tsb_size)
b52439c2 312 new_rss_limit = ~0UL;
b52439c2 313
9b4006dc 314retry_tsb_alloc:
b52439c2 315 gfp_flags = GFP_KERNEL;
9b4006dc 316 if (new_size > (PAGE_SIZE * 2))
b52439c2
DM
317 gfp_flags = __GFP_NOWARN | __GFP_NORETRY;
318
1f261ef5
DM
319 new_tsb = kmem_cache_alloc_node(tsb_caches[new_cache_index],
320 gfp_flags, numa_node_id());
9b4006dc 321 if (unlikely(!new_tsb)) {
b52439c2
DM
322 /* Not being able to fork due to a high-order TSB
323 * allocation failure is very bad behavior. Just back
324 * down to a 0-order allocation and force no TSB
325 * growing for this address space.
326 */
dcc1e8dd
DM
327 if (mm->context.tsb_block[tsb_index].tsb == NULL &&
328 new_cache_index > 0) {
9b4006dc
DM
329 new_cache_index = 0;
330 new_size = 8192;
b52439c2 331 new_rss_limit = ~0UL;
9b4006dc 332 goto retry_tsb_alloc;
b52439c2
DM
333 }
334
335 /* If we failed on a TSB grow, we are under serious
336 * memory pressure so don't try to grow any more.
337 */
dcc1e8dd
DM
338 if (mm->context.tsb_block[tsb_index].tsb != NULL)
339 mm->context.tsb_block[tsb_index].tsb_rss_limit = ~0UL;
bd40791e 340 return;
b52439c2 341 }
bd40791e 342
8b234274 343 /* Mark all tags as invalid. */
bb8646d8 344 tsb_init(new_tsb, new_size);
7a1ac526
DM
345
346 /* Ok, we are about to commit the changes. If we are
347 * growing an existing TSB the locking is very tricky,
348 * so WATCH OUT!
349 *
350 * We have to hold mm->context.lock while committing to the
351 * new TSB, this synchronizes us with processors in
352 * flush_tsb_user() and switch_mm() for this address space.
353 *
354 * But even with that lock held, processors run asynchronously
355 * accessing the old TSB via TLB miss handling. This is OK
356 * because those actions are just propagating state from the
357 * Linux page tables into the TSB, page table mappings are not
358 * being changed. If a real fault occurs, the processor will
359 * synchronize with us when it hits flush_tsb_user(), this is
360 * also true for the case where vmscan is modifying the page
361 * tables. The only thing we need to be careful with is to
362 * skip any locked TSB entries during copy_tsb().
363 *
364 * When we finish committing to the new TSB, we have to drop
365 * the lock and ask all other cpus running this address space
366 * to run tsb_context_switch() to see the new TSB table.
367 */
368 spin_lock_irqsave(&mm->context.lock, flags);
369
dcc1e8dd
DM
370 old_tsb = mm->context.tsb_block[tsb_index].tsb;
371 old_cache_index =
372 (mm->context.tsb_block[tsb_index].tsb_reg_val & 0x7UL);
373 old_size = (mm->context.tsb_block[tsb_index].tsb_nentries *
374 sizeof(struct tsb));
7a1ac526 375
9b4006dc 376
7a1ac526
DM
377 /* Handle multiple threads trying to grow the TSB at the same time.
378 * One will get in here first, and bump the size and the RSS limit.
379 * The others will get in here next and hit this check.
380 */
dcc1e8dd
DM
381 if (unlikely(old_tsb &&
382 (rss < mm->context.tsb_block[tsb_index].tsb_rss_limit))) {
7a1ac526
DM
383 spin_unlock_irqrestore(&mm->context.lock, flags);
384
9b4006dc 385 kmem_cache_free(tsb_caches[new_cache_index], new_tsb);
7a1ac526
DM
386 return;
387 }
8b234274 388
dcc1e8dd 389 mm->context.tsb_block[tsb_index].tsb_rss_limit = new_rss_limit;
bd40791e 390
7a1ac526
DM
391 if (old_tsb) {
392 extern void copy_tsb(unsigned long old_tsb_base,
393 unsigned long old_tsb_size,
394 unsigned long new_tsb_base,
395 unsigned long new_tsb_size);
396 unsigned long old_tsb_base = (unsigned long) old_tsb;
397 unsigned long new_tsb_base = (unsigned long) new_tsb;
398
399 if (tlb_type == cheetah_plus || tlb_type == hypervisor) {
400 old_tsb_base = __pa(old_tsb_base);
401 new_tsb_base = __pa(new_tsb_base);
402 }
9b4006dc 403 copy_tsb(old_tsb_base, old_size, new_tsb_base, new_size);
7a1ac526 404 }
bd40791e 405
dcc1e8dd
DM
406 mm->context.tsb_block[tsb_index].tsb = new_tsb;
407 setup_tsb_params(mm, tsb_index, new_size);
bd40791e 408
7a1ac526
DM
409 spin_unlock_irqrestore(&mm->context.lock, flags);
410
bd40791e
DM
411 /* If old_tsb is NULL, we're being invoked for the first time
412 * from init_new_context().
413 */
414 if (old_tsb) {
7a1ac526 415 /* Reload it on the local cpu. */
bd40791e
DM
416 tsb_context_switch(mm);
417
7a1ac526 418 /* Now force other processors to do the same. */
a3cf5e6b 419 preempt_disable();
7a1ac526 420 smp_tsb_sync(mm);
a3cf5e6b 421 preempt_enable();
7a1ac526
DM
422
423 /* Now it is safe to free the old tsb. */
9b4006dc 424 kmem_cache_free(tsb_caches[old_cache_index], old_tsb);
bd40791e
DM
425 }
426}
427
09f94287
DM
428int init_new_context(struct task_struct *tsk, struct mm_struct *mm)
429{
dcc1e8dd
DM
430#ifdef CONFIG_HUGETLB_PAGE
431 unsigned long huge_pte_count;
432#endif
433 unsigned int i;
434
a77754b4 435 spin_lock_init(&mm->context.lock);
09f94287
DM
436
437 mm->context.sparc64_ctx_val = 0UL;
09f94287 438
dcc1e8dd
DM
439#ifdef CONFIG_HUGETLB_PAGE
440 /* We reset it to zero because the fork() page copying
441 * will re-increment the counters as the parent PTEs are
442 * copied into the child address space.
443 */
444 huge_pte_count = mm->context.huge_pte_count;
445 mm->context.huge_pte_count = 0;
446#endif
447
c460bec7
DM
448 mm->context.pgtable_page = NULL;
449
bd40791e
DM
450 /* copy_mm() copies over the parent's mm_struct before calling
451 * us, so we need to zero out the TSB pointer or else tsb_grow()
452 * will be confused and think there is an older TSB to free up.
453 */
dcc1e8dd
DM
454 for (i = 0; i < MM_NUM_TSBS; i++)
455 mm->context.tsb_block[i].tsb = NULL;
7a1ac526
DM
456
457 /* If this is fork, inherit the parent's TSB size. We would
458 * grow it to that size on the first page fault anyways.
459 */
dcc1e8dd 460 tsb_grow(mm, MM_TSB_BASE, get_mm_rss(mm));
bd40791e 461
dcc1e8dd
DM
462#ifdef CONFIG_HUGETLB_PAGE
463 if (unlikely(huge_pte_count))
464 tsb_grow(mm, MM_TSB_HUGE, huge_pte_count);
465#endif
466
467 if (unlikely(!mm->context.tsb_block[MM_TSB_BASE].tsb))
bd40791e 468 return -ENOMEM;
09f94287
DM
469
470 return 0;
471}
472
dcc1e8dd 473static void tsb_destroy_one(struct tsb_config *tp)
09f94287 474{
dcc1e8dd 475 unsigned long cache_index;
bd40791e 476
dcc1e8dd
DM
477 if (!tp->tsb)
478 return;
479 cache_index = tp->tsb_reg_val & 0x7UL;
480 kmem_cache_free(tsb_caches[cache_index], tp->tsb);
481 tp->tsb = NULL;
482 tp->tsb_reg_val = 0UL;
483}
98c5584c 484
dcc1e8dd
DM
485void destroy_context(struct mm_struct *mm)
486{
487 unsigned long flags, i;
c460bec7 488 struct page *page;
dcc1e8dd
DM
489
490 for (i = 0; i < MM_NUM_TSBS; i++)
491 tsb_destroy_one(&mm->context.tsb_block[i]);
09f94287 492
c460bec7
DM
493 page = mm->context.pgtable_page;
494 if (page && put_page_testzero(page)) {
495 pgtable_page_dtor(page);
496 free_hot_cold_page(page, 0);
497 }
498
77b838fa 499 spin_lock_irqsave(&ctx_alloc_lock, flags);
09f94287
DM
500
501 if (CTX_VALID(mm->context)) {
502 unsigned long nr = CTX_NRBITS(mm->context);
503 mmu_context_bmap[nr>>6] &= ~(1UL << (nr & 63));
504 }
505
77b838fa 506 spin_unlock_irqrestore(&ctx_alloc_lock, flags);
09f94287 507}
This page took 0.577663 seconds and 5 git commands to generate.