Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * native hashtable management. | |
3 | * | |
4 | * SMP scalability work: | |
5 | * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU General Public License | |
9 | * as published by the Free Software Foundation; either version | |
10 | * 2 of the License, or (at your option) any later version. | |
11 | */ | |
12 | #include <linux/spinlock.h> | |
13 | #include <linux/bitops.h> | |
14 | #include <linux/threads.h> | |
15 | #include <linux/smp.h> | |
16 | ||
17 | #include <asm/abs_addr.h> | |
18 | #include <asm/machdep.h> | |
19 | #include <asm/mmu.h> | |
20 | #include <asm/mmu_context.h> | |
21 | #include <asm/pgtable.h> | |
22 | #include <asm/tlbflush.h> | |
23 | #include <asm/tlb.h> | |
24 | #include <asm/cputable.h> | |
25 | ||
26 | #define HPTE_LOCK_BIT 3 | |
27 | ||
28 | static DEFINE_SPINLOCK(native_tlbie_lock); | |
29 | ||
96e28449 | 30 | static inline void native_lock_hpte(hpte_t *hptep) |
1da177e4 | 31 | { |
96e28449 | 32 | unsigned long *word = &hptep->v; |
1da177e4 LT |
33 | |
34 | while (1) { | |
35 | if (!test_and_set_bit(HPTE_LOCK_BIT, word)) | |
36 | break; | |
37 | while(test_bit(HPTE_LOCK_BIT, word)) | |
38 | cpu_relax(); | |
39 | } | |
40 | } | |
41 | ||
96e28449 | 42 | static inline void native_unlock_hpte(hpte_t *hptep) |
1da177e4 | 43 | { |
96e28449 | 44 | unsigned long *word = &hptep->v; |
1da177e4 LT |
45 | |
46 | asm volatile("lwsync":::"memory"); | |
47 | clear_bit(HPTE_LOCK_BIT, word); | |
48 | } | |
49 | ||
50 | long native_hpte_insert(unsigned long hpte_group, unsigned long va, | |
96e28449 DG |
51 | unsigned long prpn, unsigned long vflags, |
52 | unsigned long rflags) | |
1da177e4 LT |
53 | { |
54 | unsigned long arpn = physRpn_to_absRpn(prpn); | |
96e28449 DG |
55 | hpte_t *hptep = htab_address + hpte_group; |
56 | unsigned long hpte_v, hpte_r; | |
1da177e4 LT |
57 | int i; |
58 | ||
59 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
96e28449 | 60 | if (! (hptep->v & HPTE_V_VALID)) { |
1da177e4 LT |
61 | /* retry with lock held */ |
62 | native_lock_hpte(hptep); | |
96e28449 | 63 | if (! (hptep->v & HPTE_V_VALID)) |
1da177e4 LT |
64 | break; |
65 | native_unlock_hpte(hptep); | |
66 | } | |
67 | ||
68 | hptep++; | |
69 | } | |
70 | ||
71 | if (i == HPTES_PER_GROUP) | |
72 | return -1; | |
73 | ||
96e28449 DG |
74 | hpte_v = (va >> 23) << HPTE_V_AVPN_SHIFT | vflags | HPTE_V_VALID; |
75 | if (vflags & HPTE_V_LARGE) | |
76 | va &= ~(1UL << HPTE_V_AVPN_SHIFT); | |
77 | hpte_r = (arpn << HPTE_R_RPN_SHIFT) | rflags; | |
1da177e4 | 78 | |
96e28449 | 79 | hptep->r = hpte_r; |
1da177e4 LT |
80 | /* Guarantee the second dword is visible before the valid bit */ |
81 | __asm__ __volatile__ ("eieio" : : : "memory"); | |
1da177e4 LT |
82 | /* |
83 | * Now set the first dword including the valid bit | |
84 | * NOTE: this also unlocks the hpte | |
85 | */ | |
96e28449 | 86 | hptep->v = hpte_v; |
1da177e4 LT |
87 | |
88 | __asm__ __volatile__ ("ptesync" : : : "memory"); | |
89 | ||
96e28449 | 90 | return i | (!!(vflags & HPTE_V_SECONDARY) << 3); |
1da177e4 LT |
91 | } |
92 | ||
93 | static long native_hpte_remove(unsigned long hpte_group) | |
94 | { | |
96e28449 | 95 | hpte_t *hptep; |
1da177e4 LT |
96 | int i; |
97 | int slot_offset; | |
96e28449 | 98 | unsigned long hpte_v; |
1da177e4 LT |
99 | |
100 | /* pick a random entry to start at */ | |
101 | slot_offset = mftb() & 0x7; | |
102 | ||
103 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
104 | hptep = htab_address + hpte_group + slot_offset; | |
96e28449 | 105 | hpte_v = hptep->v; |
1da177e4 | 106 | |
96e28449 | 107 | if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) { |
1da177e4 LT |
108 | /* retry with lock held */ |
109 | native_lock_hpte(hptep); | |
96e28449 DG |
110 | hpte_v = hptep->v; |
111 | if ((hpte_v & HPTE_V_VALID) | |
112 | && !(hpte_v & HPTE_V_BOLTED)) | |
1da177e4 LT |
113 | break; |
114 | native_unlock_hpte(hptep); | |
115 | } | |
116 | ||
117 | slot_offset++; | |
118 | slot_offset &= 0x7; | |
119 | } | |
120 | ||
121 | if (i == HPTES_PER_GROUP) | |
122 | return -1; | |
123 | ||
124 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 125 | hptep->v = 0; |
1da177e4 LT |
126 | |
127 | return i; | |
128 | } | |
129 | ||
96e28449 | 130 | static inline void set_pp_bit(unsigned long pp, hpte_t *addr) |
1da177e4 LT |
131 | { |
132 | unsigned long old; | |
96e28449 | 133 | unsigned long *p = &addr->r; |
1da177e4 LT |
134 | |
135 | __asm__ __volatile__( | |
136 | "1: ldarx %0,0,%3\n\ | |
137 | rldimi %0,%2,0,61\n\ | |
138 | stdcx. %0,0,%3\n\ | |
139 | bne 1b" | |
140 | : "=&r" (old), "=m" (*p) | |
141 | : "r" (pp), "r" (p), "m" (*p) | |
142 | : "cc"); | |
143 | } | |
144 | ||
145 | /* | |
146 | * Only works on small pages. Yes its ugly to have to check each slot in | |
147 | * the group but we only use this during bootup. | |
148 | */ | |
149 | static long native_hpte_find(unsigned long vpn) | |
150 | { | |
96e28449 | 151 | hpte_t *hptep; |
1da177e4 LT |
152 | unsigned long hash; |
153 | unsigned long i, j; | |
154 | long slot; | |
96e28449 | 155 | unsigned long hpte_v; |
1da177e4 LT |
156 | |
157 | hash = hpt_hash(vpn, 0); | |
158 | ||
159 | for (j = 0; j < 2; j++) { | |
160 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
161 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
162 | hptep = htab_address + slot; | |
96e28449 | 163 | hpte_v = hptep->v; |
1da177e4 | 164 | |
96e28449 DG |
165 | if ((HPTE_V_AVPN_VAL(hpte_v) == (vpn >> 11)) |
166 | && (hpte_v & HPTE_V_VALID) | |
167 | && ( !!(hpte_v & HPTE_V_SECONDARY) == j)) { | |
1da177e4 LT |
168 | /* HPTE matches */ |
169 | if (j) | |
170 | slot = -slot; | |
171 | return slot; | |
172 | } | |
173 | ++slot; | |
174 | } | |
175 | hash = ~hash; | |
176 | } | |
177 | ||
178 | return -1; | |
179 | } | |
180 | ||
181 | static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, | |
182 | unsigned long va, int large, int local) | |
183 | { | |
96e28449 DG |
184 | hpte_t *hptep = htab_address + slot; |
185 | unsigned long hpte_v; | |
1da177e4 LT |
186 | unsigned long avpn = va >> 23; |
187 | int ret = 0; | |
188 | ||
189 | if (large) | |
96e28449 | 190 | avpn &= ~1; |
1da177e4 LT |
191 | |
192 | native_lock_hpte(hptep); | |
193 | ||
96e28449 | 194 | hpte_v = hptep->v; |
1da177e4 LT |
195 | |
196 | /* Even if we miss, we need to invalidate the TLB */ | |
96e28449 DG |
197 | if ((HPTE_V_AVPN_VAL(hpte_v) != avpn) |
198 | || !(hpte_v & HPTE_V_VALID)) { | |
1da177e4 LT |
199 | native_unlock_hpte(hptep); |
200 | ret = -1; | |
201 | } else { | |
202 | set_pp_bit(newpp, hptep); | |
203 | native_unlock_hpte(hptep); | |
204 | } | |
205 | ||
206 | /* Ensure it is out of the tlb too */ | |
207 | if (cpu_has_feature(CPU_FTR_TLBIEL) && !large && local) { | |
208 | tlbiel(va); | |
209 | } else { | |
210 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); | |
211 | ||
212 | if (lock_tlbie) | |
213 | spin_lock(&native_tlbie_lock); | |
214 | tlbie(va, large); | |
215 | if (lock_tlbie) | |
216 | spin_unlock(&native_tlbie_lock); | |
217 | } | |
218 | ||
219 | return ret; | |
220 | } | |
221 | ||
222 | /* | |
223 | * Update the page protection bits. Intended to be used to create | |
224 | * guard pages for kernel data structures on pages which are bolted | |
225 | * in the HPT. Assumes pages being operated on will not be stolen. | |
226 | * Does not work on large pages. | |
227 | * | |
228 | * No need to lock here because we should be the only user. | |
229 | */ | |
230 | static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea) | |
231 | { | |
232 | unsigned long vsid, va, vpn, flags = 0; | |
233 | long slot; | |
96e28449 | 234 | hpte_t *hptep; |
1da177e4 LT |
235 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); |
236 | ||
237 | vsid = get_kernel_vsid(ea); | |
238 | va = (vsid << 28) | (ea & 0x0fffffff); | |
239 | vpn = va >> PAGE_SHIFT; | |
240 | ||
241 | slot = native_hpte_find(vpn); | |
242 | if (slot == -1) | |
243 | panic("could not find page to bolt\n"); | |
244 | hptep = htab_address + slot; | |
245 | ||
246 | set_pp_bit(newpp, hptep); | |
247 | ||
248 | /* Ensure it is out of the tlb too */ | |
249 | if (lock_tlbie) | |
250 | spin_lock_irqsave(&native_tlbie_lock, flags); | |
251 | tlbie(va, 0); | |
252 | if (lock_tlbie) | |
253 | spin_unlock_irqrestore(&native_tlbie_lock, flags); | |
254 | } | |
255 | ||
256 | static void native_hpte_invalidate(unsigned long slot, unsigned long va, | |
257 | int large, int local) | |
258 | { | |
96e28449 DG |
259 | hpte_t *hptep = htab_address + slot; |
260 | unsigned long hpte_v; | |
1da177e4 LT |
261 | unsigned long avpn = va >> 23; |
262 | unsigned long flags; | |
263 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); | |
264 | ||
265 | if (large) | |
96e28449 | 266 | avpn &= ~1; |
1da177e4 LT |
267 | |
268 | local_irq_save(flags); | |
269 | native_lock_hpte(hptep); | |
270 | ||
96e28449 | 271 | hpte_v = hptep->v; |
1da177e4 LT |
272 | |
273 | /* Even if we miss, we need to invalidate the TLB */ | |
96e28449 DG |
274 | if ((HPTE_V_AVPN_VAL(hpte_v) != avpn) |
275 | || !(hpte_v & HPTE_V_VALID)) { | |
1da177e4 LT |
276 | native_unlock_hpte(hptep); |
277 | } else { | |
278 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 279 | hptep->v = 0; |
1da177e4 LT |
280 | } |
281 | ||
282 | /* Invalidate the tlb */ | |
283 | if (cpu_has_feature(CPU_FTR_TLBIEL) && !large && local) { | |
284 | tlbiel(va); | |
285 | } else { | |
286 | if (lock_tlbie) | |
287 | spin_lock(&native_tlbie_lock); | |
288 | tlbie(va, large); | |
289 | if (lock_tlbie) | |
290 | spin_unlock(&native_tlbie_lock); | |
291 | } | |
292 | local_irq_restore(flags); | |
293 | } | |
294 | ||
f4c82d51 S |
295 | /* |
296 | * clear all mappings on kexec. All cpus are in real mode (or they will | |
297 | * be when they isi), and we are the only one left. We rely on our kernel | |
298 | * mapping being 0xC0's and the hardware ignoring those two real bits. | |
299 | * | |
300 | * TODO: add batching support when enabled. remember, no dynamic memory here, | |
301 | * athough there is the control page available... | |
302 | */ | |
303 | static void native_hpte_clear(void) | |
304 | { | |
305 | unsigned long slot, slots, flags; | |
96e28449 DG |
306 | hpte_t *hptep = htab_address; |
307 | unsigned long hpte_v; | |
f4c82d51 S |
308 | unsigned long pteg_count; |
309 | ||
310 | pteg_count = htab_hash_mask + 1; | |
311 | ||
312 | local_irq_save(flags); | |
313 | ||
314 | /* we take the tlbie lock and hold it. Some hardware will | |
315 | * deadlock if we try to tlbie from two processors at once. | |
316 | */ | |
317 | spin_lock(&native_tlbie_lock); | |
318 | ||
319 | slots = pteg_count * HPTES_PER_GROUP; | |
320 | ||
321 | for (slot = 0; slot < slots; slot++, hptep++) { | |
322 | /* | |
323 | * we could lock the pte here, but we are the only cpu | |
324 | * running, right? and for crash dump, we probably | |
325 | * don't want to wait for a maybe bad cpu. | |
326 | */ | |
96e28449 | 327 | hpte_v = hptep->v; |
f4c82d51 | 328 | |
96e28449 DG |
329 | if (hpte_v & HPTE_V_VALID) { |
330 | hptep->v = 0; | |
331 | tlbie(slot2va(hpte_v, slot), hpte_v & HPTE_V_LARGE); | |
f4c82d51 S |
332 | } |
333 | } | |
334 | ||
335 | spin_unlock(&native_tlbie_lock); | |
336 | local_irq_restore(flags); | |
337 | } | |
338 | ||
1da177e4 LT |
339 | static void native_flush_hash_range(unsigned long context, |
340 | unsigned long number, int local) | |
341 | { | |
342 | unsigned long vsid, vpn, va, hash, secondary, slot, flags, avpn; | |
343 | int i, j; | |
96e28449 DG |
344 | hpte_t *hptep; |
345 | unsigned long hpte_v; | |
1da177e4 LT |
346 | struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); |
347 | ||
348 | /* XXX fix for large ptes */ | |
349 | unsigned long large = 0; | |
350 | ||
351 | local_irq_save(flags); | |
352 | ||
353 | j = 0; | |
354 | for (i = 0; i < number; i++) { | |
1f8d419e | 355 | if (batch->addr[i] < KERNELBASE) |
1da177e4 LT |
356 | vsid = get_vsid(context, batch->addr[i]); |
357 | else | |
358 | vsid = get_kernel_vsid(batch->addr[i]); | |
359 | ||
360 | va = (vsid << 28) | (batch->addr[i] & 0x0fffffff); | |
361 | batch->vaddr[j] = va; | |
362 | if (large) | |
363 | vpn = va >> HPAGE_SHIFT; | |
364 | else | |
365 | vpn = va >> PAGE_SHIFT; | |
366 | hash = hpt_hash(vpn, large); | |
367 | secondary = (pte_val(batch->pte[i]) & _PAGE_SECONDARY) >> 15; | |
368 | if (secondary) | |
369 | hash = ~hash; | |
370 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
371 | slot += (pte_val(batch->pte[i]) & _PAGE_GROUP_IX) >> 12; | |
372 | ||
373 | hptep = htab_address + slot; | |
374 | ||
375 | avpn = va >> 23; | |
376 | if (large) | |
377 | avpn &= ~0x1UL; | |
378 | ||
379 | native_lock_hpte(hptep); | |
380 | ||
96e28449 | 381 | hpte_v = hptep->v; |
1da177e4 LT |
382 | |
383 | /* Even if we miss, we need to invalidate the TLB */ | |
96e28449 DG |
384 | if ((HPTE_V_AVPN_VAL(hpte_v) != avpn) |
385 | || !(hpte_v & HPTE_V_VALID)) { | |
1da177e4 LT |
386 | native_unlock_hpte(hptep); |
387 | } else { | |
388 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 389 | hptep->v = 0; |
1da177e4 LT |
390 | } |
391 | ||
392 | j++; | |
393 | } | |
394 | ||
395 | if (cpu_has_feature(CPU_FTR_TLBIEL) && !large && local) { | |
396 | asm volatile("ptesync":::"memory"); | |
397 | ||
398 | for (i = 0; i < j; i++) | |
399 | __tlbiel(batch->vaddr[i]); | |
400 | ||
401 | asm volatile("ptesync":::"memory"); | |
402 | } else { | |
403 | int lock_tlbie = !cpu_has_feature(CPU_FTR_LOCKLESS_TLBIE); | |
404 | ||
405 | if (lock_tlbie) | |
406 | spin_lock(&native_tlbie_lock); | |
407 | ||
408 | asm volatile("ptesync":::"memory"); | |
409 | ||
410 | for (i = 0; i < j; i++) | |
411 | __tlbie(batch->vaddr[i], 0); | |
412 | ||
413 | asm volatile("eieio; tlbsync; ptesync":::"memory"); | |
414 | ||
415 | if (lock_tlbie) | |
416 | spin_unlock(&native_tlbie_lock); | |
417 | } | |
418 | ||
419 | local_irq_restore(flags); | |
420 | } | |
421 | ||
422 | #ifdef CONFIG_PPC_PSERIES | |
423 | /* Disable TLB batching on nighthawk */ | |
424 | static inline int tlb_batching_enabled(void) | |
425 | { | |
426 | struct device_node *root = of_find_node_by_path("/"); | |
427 | int enabled = 1; | |
428 | ||
429 | if (root) { | |
430 | const char *model = get_property(root, "model", NULL); | |
431 | if (model && !strcmp(model, "IBM,9076-N81")) | |
432 | enabled = 0; | |
433 | of_node_put(root); | |
434 | } | |
435 | ||
436 | return enabled; | |
437 | } | |
438 | #else | |
439 | static inline int tlb_batching_enabled(void) | |
440 | { | |
441 | return 1; | |
442 | } | |
443 | #endif | |
444 | ||
445 | void hpte_init_native(void) | |
446 | { | |
447 | ppc_md.hpte_invalidate = native_hpte_invalidate; | |
448 | ppc_md.hpte_updatepp = native_hpte_updatepp; | |
449 | ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp; | |
450 | ppc_md.hpte_insert = native_hpte_insert; | |
f4c82d51 S |
451 | ppc_md.hpte_remove = native_hpte_remove; |
452 | ppc_md.hpte_clear_all = native_hpte_clear; | |
1da177e4 LT |
453 | if (tlb_batching_enabled()) |
454 | ppc_md.flush_hash_range = native_flush_hash_range; | |
455 | htab_finish_init(); | |
456 | } |