Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * native hashtable management. | |
3 | * | |
4 | * SMP scalability work: | |
5 | * Copyright (C) 2001 Anton Blanchard <anton@au.ibm.com>, IBM | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or | |
8 | * modify it under the terms of the GNU General Public License | |
9 | * as published by the Free Software Foundation; either version | |
10 | * 2 of the License, or (at your option) any later version. | |
11 | */ | |
3c726f8d BH |
12 | |
13 | #undef DEBUG_LOW | |
14 | ||
1da177e4 LT |
15 | #include <linux/spinlock.h> |
16 | #include <linux/bitops.h> | |
beacc6da | 17 | #include <linux/of.h> |
1da177e4 LT |
18 | #include <linux/threads.h> |
19 | #include <linux/smp.h> | |
20 | ||
1da177e4 LT |
21 | #include <asm/machdep.h> |
22 | #include <asm/mmu.h> | |
23 | #include <asm/mmu_context.h> | |
24 | #include <asm/pgtable.h> | |
25 | #include <asm/tlbflush.h> | |
26 | #include <asm/tlb.h> | |
27 | #include <asm/cputable.h> | |
3c726f8d | 28 | #include <asm/udbg.h> |
71bf08b6 | 29 | #include <asm/kexec.h> |
60dbf438 | 30 | #include <asm/ppc-opcode.h> |
3c726f8d BH |
31 | |
32 | #ifdef DEBUG_LOW | |
33 | #define DBG_LOW(fmt...) udbg_printf(fmt) | |
34 | #else | |
35 | #define DBG_LOW(fmt...) | |
36 | #endif | |
1da177e4 LT |
37 | |
38 | #define HPTE_LOCK_BIT 3 | |
39 | ||
9e368f29 | 40 | DEFINE_RAW_SPINLOCK(native_tlbie_lock); |
1da177e4 | 41 | |
b1022fbd | 42 | static inline void __tlbie(unsigned long vpn, int psize, int apsize, int ssize) |
3c726f8d | 43 | { |
5524a27d | 44 | unsigned long va; |
3c726f8d BH |
45 | unsigned int penc; |
46 | ||
5524a27d AK |
47 | /* |
48 | * We need 14 to 65 bits of va for a tlibe of 4K page | |
49 | * With vpn we ignore the lower VPN_SHIFT bits already. | |
50 | * And top two bits are already ignored because we can | |
51 | * only accomadate 76 bits in a 64 bit vpn with a VPN_SHIFT | |
52 | * of 12. | |
53 | */ | |
54 | va = vpn << VPN_SHIFT; | |
55 | /* | |
56 | * clear top 16 bits of 64bit va, non SLS segment | |
57 | * Older versions of the architecture (2.02 and earler) require the | |
58 | * masking of the top 16 bits. | |
59 | */ | |
3c726f8d BH |
60 | va &= ~(0xffffULL << 48); |
61 | ||
62 | switch (psize) { | |
63 | case MMU_PAGE_4K: | |
1189be65 | 64 | va |= ssize << 8; |
a32e252f | 65 | asm volatile(ASM_FTR_IFCLR("tlbie %0,0", PPC_TLBIE(%1,%0), %2) |
969391c5 | 66 | : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) |
60dbf438 | 67 | : "memory"); |
3c726f8d BH |
68 | break; |
69 | default: | |
5524a27d | 70 | /* We need 14 to 14 + i bits of va */ |
b1022fbd | 71 | penc = mmu_psize_defs[psize].penc[apsize]; |
3c726f8d | 72 | va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); |
19242b24 | 73 | va |= penc << 12; |
1189be65 | 74 | va |= ssize << 8; |
60dbf438 | 75 | va |= 1; /* L */ |
a32e252f | 76 | asm volatile(ASM_FTR_IFCLR("tlbie %0,1", PPC_TLBIE(%1,%0), %2) |
969391c5 | 77 | : : "r" (va), "r"(0), "i" (CPU_FTR_ARCH_206) |
60dbf438 | 78 | : "memory"); |
3c726f8d BH |
79 | break; |
80 | } | |
81 | } | |
82 | ||
b1022fbd | 83 | static inline void __tlbiel(unsigned long vpn, int psize, int apsize, int ssize) |
3c726f8d | 84 | { |
5524a27d | 85 | unsigned long va; |
3c726f8d BH |
86 | unsigned int penc; |
87 | ||
5524a27d AK |
88 | /* VPN_SHIFT can be atmost 12 */ |
89 | va = vpn << VPN_SHIFT; | |
90 | /* | |
91 | * clear top 16 bits of 64 bit va, non SLS segment | |
92 | * Older versions of the architecture (2.02 and earler) require the | |
93 | * masking of the top 16 bits. | |
94 | */ | |
3c726f8d BH |
95 | va &= ~(0xffffULL << 48); |
96 | ||
97 | switch (psize) { | |
98 | case MMU_PAGE_4K: | |
1189be65 | 99 | va |= ssize << 8; |
3c726f8d BH |
100 | asm volatile(".long 0x7c000224 | (%0 << 11) | (0 << 21)" |
101 | : : "r"(va) : "memory"); | |
102 | break; | |
103 | default: | |
5524a27d | 104 | /* We need 14 to 14 + i bits of va */ |
b1022fbd | 105 | penc = mmu_psize_defs[psize].penc[apsize]; |
3c726f8d | 106 | va &= ~((1ul << mmu_psize_defs[psize].shift) - 1); |
19242b24 | 107 | va |= penc << 12; |
1189be65 | 108 | va |= ssize << 8; |
60dbf438 | 109 | va |= 1; /* L */ |
3c726f8d BH |
110 | asm volatile(".long 0x7c000224 | (%0 << 11) | (1 << 21)" |
111 | : : "r"(va) : "memory"); | |
112 | break; | |
113 | } | |
114 | ||
115 | } | |
116 | ||
b1022fbd AK |
117 | static inline void tlbie(unsigned long vpn, int psize, int apsize, |
118 | int ssize, int local) | |
3c726f8d | 119 | { |
44ae3ab3 ME |
120 | unsigned int use_local = local && mmu_has_feature(MMU_FTR_TLBIEL); |
121 | int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); | |
3c726f8d BH |
122 | |
123 | if (use_local) | |
124 | use_local = mmu_psize_defs[psize].tlbiel; | |
125 | if (lock_tlbie && !use_local) | |
6b9c9b8a | 126 | raw_spin_lock(&native_tlbie_lock); |
3c726f8d BH |
127 | asm volatile("ptesync": : :"memory"); |
128 | if (use_local) { | |
b1022fbd | 129 | __tlbiel(vpn, psize, apsize, ssize); |
3c726f8d BH |
130 | asm volatile("ptesync": : :"memory"); |
131 | } else { | |
b1022fbd | 132 | __tlbie(vpn, psize, apsize, ssize); |
3c726f8d BH |
133 | asm volatile("eieio; tlbsync; ptesync": : :"memory"); |
134 | } | |
135 | if (lock_tlbie && !use_local) | |
6b9c9b8a | 136 | raw_spin_unlock(&native_tlbie_lock); |
3c726f8d BH |
137 | } |
138 | ||
8e561e7e | 139 | static inline void native_lock_hpte(struct hash_pte *hptep) |
1da177e4 | 140 | { |
96e28449 | 141 | unsigned long *word = &hptep->v; |
1da177e4 LT |
142 | |
143 | while (1) { | |
66d99b88 | 144 | if (!test_and_set_bit_lock(HPTE_LOCK_BIT, word)) |
1da177e4 LT |
145 | break; |
146 | while(test_bit(HPTE_LOCK_BIT, word)) | |
147 | cpu_relax(); | |
148 | } | |
149 | } | |
150 | ||
8e561e7e | 151 | static inline void native_unlock_hpte(struct hash_pte *hptep) |
1da177e4 | 152 | { |
96e28449 | 153 | unsigned long *word = &hptep->v; |
1da177e4 | 154 | |
66d99b88 | 155 | clear_bit_unlock(HPTE_LOCK_BIT, word); |
1da177e4 LT |
156 | } |
157 | ||
5524a27d | 158 | static long native_hpte_insert(unsigned long hpte_group, unsigned long vpn, |
3c726f8d | 159 | unsigned long pa, unsigned long rflags, |
b1022fbd | 160 | unsigned long vflags, int psize, int apsize, int ssize) |
1da177e4 | 161 | { |
8e561e7e | 162 | struct hash_pte *hptep = htab_address + hpte_group; |
96e28449 | 163 | unsigned long hpte_v, hpte_r; |
1da177e4 LT |
164 | int i; |
165 | ||
3c726f8d | 166 | if (!(vflags & HPTE_V_BOLTED)) { |
5524a27d | 167 | DBG_LOW(" insert(group=%lx, vpn=%016lx, pa=%016lx," |
3c726f8d | 168 | " rflags=%lx, vflags=%lx, psize=%d)\n", |
5524a27d | 169 | hpte_group, vpn, pa, rflags, vflags, psize); |
3c726f8d BH |
170 | } |
171 | ||
1da177e4 | 172 | for (i = 0; i < HPTES_PER_GROUP; i++) { |
96e28449 | 173 | if (! (hptep->v & HPTE_V_VALID)) { |
1da177e4 LT |
174 | /* retry with lock held */ |
175 | native_lock_hpte(hptep); | |
96e28449 | 176 | if (! (hptep->v & HPTE_V_VALID)) |
1da177e4 LT |
177 | break; |
178 | native_unlock_hpte(hptep); | |
179 | } | |
180 | ||
181 | hptep++; | |
182 | } | |
183 | ||
184 | if (i == HPTES_PER_GROUP) | |
185 | return -1; | |
186 | ||
b1022fbd AK |
187 | hpte_v = hpte_encode_v(vpn, psize, apsize, ssize) | vflags | HPTE_V_VALID; |
188 | hpte_r = hpte_encode_r(pa, psize, apsize) | rflags; | |
3c726f8d BH |
189 | |
190 | if (!(vflags & HPTE_V_BOLTED)) { | |
191 | DBG_LOW(" i=%x hpte_v=%016lx, hpte_r=%016lx\n", | |
192 | i, hpte_v, hpte_r); | |
193 | } | |
1da177e4 | 194 | |
96e28449 | 195 | hptep->r = hpte_r; |
1da177e4 | 196 | /* Guarantee the second dword is visible before the valid bit */ |
74a0ba61 | 197 | eieio(); |
1da177e4 LT |
198 | /* |
199 | * Now set the first dword including the valid bit | |
200 | * NOTE: this also unlocks the hpte | |
201 | */ | |
96e28449 | 202 | hptep->v = hpte_v; |
1da177e4 LT |
203 | |
204 | __asm__ __volatile__ ("ptesync" : : : "memory"); | |
205 | ||
96e28449 | 206 | return i | (!!(vflags & HPTE_V_SECONDARY) << 3); |
1da177e4 LT |
207 | } |
208 | ||
209 | static long native_hpte_remove(unsigned long hpte_group) | |
210 | { | |
8e561e7e | 211 | struct hash_pte *hptep; |
1da177e4 LT |
212 | int i; |
213 | int slot_offset; | |
96e28449 | 214 | unsigned long hpte_v; |
1da177e4 | 215 | |
3c726f8d BH |
216 | DBG_LOW(" remove(group=%lx)\n", hpte_group); |
217 | ||
1da177e4 LT |
218 | /* pick a random entry to start at */ |
219 | slot_offset = mftb() & 0x7; | |
220 | ||
221 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
222 | hptep = htab_address + hpte_group + slot_offset; | |
96e28449 | 223 | hpte_v = hptep->v; |
1da177e4 | 224 | |
96e28449 | 225 | if ((hpte_v & HPTE_V_VALID) && !(hpte_v & HPTE_V_BOLTED)) { |
1da177e4 LT |
226 | /* retry with lock held */ |
227 | native_lock_hpte(hptep); | |
96e28449 DG |
228 | hpte_v = hptep->v; |
229 | if ((hpte_v & HPTE_V_VALID) | |
230 | && !(hpte_v & HPTE_V_BOLTED)) | |
1da177e4 LT |
231 | break; |
232 | native_unlock_hpte(hptep); | |
233 | } | |
234 | ||
235 | slot_offset++; | |
236 | slot_offset &= 0x7; | |
237 | } | |
238 | ||
239 | if (i == HPTES_PER_GROUP) | |
240 | return -1; | |
241 | ||
242 | /* Invalidate the hpte. NOTE: this also unlocks it */ | |
96e28449 | 243 | hptep->v = 0; |
1da177e4 LT |
244 | |
245 | return i; | |
246 | } | |
247 | ||
7e74c392 | 248 | static inline int __hpte_actual_psize(unsigned int lp, int psize) |
b1022fbd AK |
249 | { |
250 | int i, shift; | |
251 | unsigned int mask; | |
b1022fbd AK |
252 | |
253 | /* start from 1 ignoring MMU_PAGE_4K */ | |
254 | for (i = 1; i < MMU_PAGE_COUNT; i++) { | |
255 | ||
256 | /* invalid penc */ | |
257 | if (mmu_psize_defs[psize].penc[i] == -1) | |
258 | continue; | |
259 | /* | |
260 | * encoding bits per actual page size | |
261 | * PTE LP actual page size | |
262 | * rrrr rrrz >=8KB | |
263 | * rrrr rrzz >=16KB | |
264 | * rrrr rzzz >=32KB | |
265 | * rrrr zzzz >=64KB | |
266 | * ....... | |
267 | */ | |
268 | shift = mmu_psize_defs[i].shift - LP_SHIFT; | |
269 | if (shift > LP_BITS) | |
270 | shift = LP_BITS; | |
271 | mask = (1 << shift) - 1; | |
272 | if ((lp & mask) == mmu_psize_defs[psize].penc[i]) | |
273 | return i; | |
274 | } | |
275 | return -1; | |
276 | } | |
277 | ||
7e74c392 AK |
278 | static inline int hpte_actual_psize(struct hash_pte *hptep, int psize) |
279 | { | |
280 | /* Look at the 8 bit LP value */ | |
281 | unsigned int lp = (hptep->r >> LP_SHIFT) & ((1 << LP_BITS) - 1); | |
282 | ||
283 | if (!(hptep->v & HPTE_V_VALID)) | |
284 | return -1; | |
285 | ||
286 | /* First check if it is large page */ | |
287 | if (!(hptep->v & HPTE_V_LARGE)) | |
288 | return MMU_PAGE_4K; | |
289 | ||
290 | return __hpte_actual_psize(lp, psize); | |
291 | } | |
292 | ||
3c726f8d | 293 | static long native_hpte_updatepp(unsigned long slot, unsigned long newpp, |
5524a27d | 294 | unsigned long vpn, int psize, int ssize, |
1189be65 | 295 | int local) |
1da177e4 | 296 | { |
8e561e7e | 297 | struct hash_pte *hptep = htab_address + slot; |
3c726f8d BH |
298 | unsigned long hpte_v, want_v; |
299 | int ret = 0; | |
b1022fbd | 300 | int actual_psize; |
3c726f8d | 301 | |
74f227b2 | 302 | want_v = hpte_encode_avpn(vpn, psize, ssize); |
3c726f8d | 303 | |
5524a27d AK |
304 | DBG_LOW(" update(vpn=%016lx, avpnv=%016lx, group=%lx, newpp=%lx)", |
305 | vpn, want_v & HPTE_V_AVPN, slot, newpp); | |
3c726f8d BH |
306 | |
307 | native_lock_hpte(hptep); | |
308 | ||
309 | hpte_v = hptep->v; | |
b1022fbd AK |
310 | actual_psize = hpte_actual_psize(hptep, psize); |
311 | if (actual_psize < 0) { | |
312 | native_unlock_hpte(hptep); | |
313 | return -1; | |
314 | } | |
3c726f8d | 315 | /* Even if we miss, we need to invalidate the TLB */ |
b1022fbd | 316 | if (!HPTE_V_COMPARE(hpte_v, want_v)) { |
3c726f8d | 317 | DBG_LOW(" -> miss\n"); |
3c726f8d BH |
318 | ret = -1; |
319 | } else { | |
320 | DBG_LOW(" -> hit\n"); | |
321 | /* Update the HPTE */ | |
322 | hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) | | |
c5cf0e30 | 323 | (newpp & (HPTE_R_PP | HPTE_R_N | HPTE_R_C)); |
3c726f8d | 324 | } |
3f1df7a2 | 325 | native_unlock_hpte(hptep); |
3c726f8d BH |
326 | |
327 | /* Ensure it is out of the tlb too. */ | |
b1022fbd | 328 | tlbie(vpn, psize, actual_psize, ssize, local); |
3c726f8d BH |
329 | |
330 | return ret; | |
1da177e4 LT |
331 | } |
332 | ||
5524a27d | 333 | static long native_hpte_find(unsigned long vpn, int psize, int ssize) |
1da177e4 | 334 | { |
8e561e7e | 335 | struct hash_pte *hptep; |
1da177e4 | 336 | unsigned long hash; |
1189be65 | 337 | unsigned long i; |
1da177e4 | 338 | long slot; |
3c726f8d | 339 | unsigned long want_v, hpte_v; |
1da177e4 | 340 | |
5524a27d | 341 | hash = hpt_hash(vpn, mmu_psize_defs[psize].shift, ssize); |
74f227b2 | 342 | want_v = hpte_encode_avpn(vpn, psize, ssize); |
1da177e4 | 343 | |
1189be65 PM |
344 | /* Bolted mappings are only ever in the primary group */ |
345 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
346 | for (i = 0; i < HPTES_PER_GROUP; i++) { | |
347 | hptep = htab_address + slot; | |
348 | hpte_v = hptep->v; | |
1da177e4 | 349 | |
1189be65 PM |
350 | if (HPTE_V_COMPARE(hpte_v, want_v) && (hpte_v & HPTE_V_VALID)) |
351 | /* HPTE matches */ | |
352 | return slot; | |
353 | ++slot; | |
1da177e4 LT |
354 | } |
355 | ||
356 | return -1; | |
357 | } | |
358 | ||
1da177e4 LT |
359 | /* |
360 | * Update the page protection bits. Intended to be used to create | |
361 | * guard pages for kernel data structures on pages which are bolted | |
362 | * in the HPT. Assumes pages being operated on will not be stolen. | |
1da177e4 LT |
363 | * |
364 | * No need to lock here because we should be the only user. | |
365 | */ | |
3c726f8d | 366 | static void native_hpte_updateboltedpp(unsigned long newpp, unsigned long ea, |
1189be65 | 367 | int psize, int ssize) |
1da177e4 | 368 | { |
b1022fbd | 369 | int actual_psize; |
5524a27d AK |
370 | unsigned long vpn; |
371 | unsigned long vsid; | |
1da177e4 | 372 | long slot; |
8e561e7e | 373 | struct hash_pte *hptep; |
1da177e4 | 374 | |
1189be65 | 375 | vsid = get_kernel_vsid(ea, ssize); |
5524a27d | 376 | vpn = hpt_vpn(ea, vsid, ssize); |
1da177e4 | 377 | |
5524a27d | 378 | slot = native_hpte_find(vpn, psize, ssize); |
1da177e4 LT |
379 | if (slot == -1) |
380 | panic("could not find page to bolt\n"); | |
381 | hptep = htab_address + slot; | |
b1022fbd AK |
382 | actual_psize = hpte_actual_psize(hptep, psize); |
383 | if (actual_psize < 0) | |
384 | return; | |
1da177e4 | 385 | |
3c726f8d BH |
386 | /* Update the HPTE */ |
387 | hptep->r = (hptep->r & ~(HPTE_R_PP | HPTE_R_N)) | | |
388 | (newpp & (HPTE_R_PP | HPTE_R_N)); | |
1da177e4 | 389 | |
3c726f8d | 390 | /* Ensure it is out of the tlb too. */ |
b1022fbd | 391 | tlbie(vpn, psize, actual_psize, ssize, 0); |
1da177e4 LT |
392 | } |
393 | ||
5524a27d | 394 | static void native_hpte_invalidate(unsigned long slot, unsigned long vpn, |
1189be65 | 395 | int psize, int ssize, int local) |
1da177e4 | 396 | { |
8e561e7e | 397 | struct hash_pte *hptep = htab_address + slot; |
96e28449 | 398 | unsigned long hpte_v; |
3c726f8d | 399 | unsigned long want_v; |
1da177e4 | 400 | unsigned long flags; |
b1022fbd | 401 | int actual_psize; |
1da177e4 LT |
402 | |
403 | local_irq_save(flags); | |
1da177e4 | 404 | |
5524a27d | 405 | DBG_LOW(" invalidate(vpn=%016lx, hash: %lx)\n", vpn, slot); |
3c726f8d | 406 | |
74f227b2 | 407 | want_v = hpte_encode_avpn(vpn, psize, ssize); |
3c726f8d | 408 | native_lock_hpte(hptep); |
96e28449 | 409 | hpte_v = hptep->v; |
1da177e4 | 410 | |
b1022fbd AK |
411 | actual_psize = hpte_actual_psize(hptep, psize); |
412 | if (actual_psize < 0) { | |
413 | native_unlock_hpte(hptep); | |
414 | local_irq_restore(flags); | |
415 | return; | |
416 | } | |
1da177e4 | 417 | /* Even if we miss, we need to invalidate the TLB */ |
b1022fbd | 418 | if (!HPTE_V_COMPARE(hpte_v, want_v)) |
1da177e4 | 419 | native_unlock_hpte(hptep); |
3c726f8d | 420 | else |
1da177e4 | 421 | /* Invalidate the hpte. NOTE: this also unlocks it */ |
96e28449 | 422 | hptep->v = 0; |
1da177e4 | 423 | |
3c726f8d | 424 | /* Invalidate the TLB */ |
b1022fbd | 425 | tlbie(vpn, psize, actual_psize, ssize, local); |
3c726f8d | 426 | |
1da177e4 LT |
427 | local_irq_restore(flags); |
428 | } | |
429 | ||
8e561e7e | 430 | static void hpte_decode(struct hash_pte *hpte, unsigned long slot, |
b1022fbd | 431 | int *psize, int *apsize, int *ssize, unsigned long *vpn) |
71bf08b6 | 432 | { |
dcda287a | 433 | unsigned long avpn, pteg, vpi; |
71bf08b6 | 434 | unsigned long hpte_v = hpte->v; |
dcda287a | 435 | unsigned long vsid, seg_off; |
7e74c392 AK |
436 | int size, a_size, shift; |
437 | /* Look at the 8 bit LP value */ | |
438 | unsigned int lp = (hpte->r >> LP_SHIFT) & ((1 << LP_BITS) - 1); | |
71bf08b6 | 439 | |
b1022fbd AK |
440 | if (!(hpte_v & HPTE_V_LARGE)) { |
441 | size = MMU_PAGE_4K; | |
442 | a_size = MMU_PAGE_4K; | |
443 | } else { | |
71bf08b6 | 444 | for (size = 0; size < MMU_PAGE_COUNT; size++) { |
3c726f8d | 445 | |
71bf08b6 LB |
446 | /* valid entries have a shift value */ |
447 | if (!mmu_psize_defs[size].shift) | |
448 | continue; | |
b1022fbd | 449 | |
7e74c392 AK |
450 | a_size = __hpte_actual_psize(lp, size); |
451 | if (a_size != -1) | |
452 | break; | |
71bf08b6 LB |
453 | } |
454 | } | |
2454c7e9 | 455 | /* This works for all page sizes, and for 256M and 1T segments */ |
dcda287a | 456 | *ssize = hpte_v >> HPTE_V_SSIZE_SHIFT; |
71bf08b6 | 457 | shift = mmu_psize_defs[size].shift; |
71bf08b6 | 458 | |
dcda287a AK |
459 | avpn = (HPTE_V_AVPN_VAL(hpte_v) & ~mmu_psize_defs[size].avpnm); |
460 | pteg = slot / HPTES_PER_GROUP; | |
461 | if (hpte_v & HPTE_V_SECONDARY) | |
462 | pteg = ~pteg; | |
463 | ||
464 | switch (*ssize) { | |
465 | case MMU_SEGSIZE_256M: | |
466 | /* We only have 28 - 23 bits of seg_off in avpn */ | |
467 | seg_off = (avpn & 0x1f) << 23; | |
468 | vsid = avpn >> 5; | |
469 | /* We can find more bits from the pteg value */ | |
470 | if (shift < 23) { | |
471 | vpi = (vsid ^ pteg) & htab_hash_mask; | |
472 | seg_off |= vpi << shift; | |
473 | } | |
5524a27d | 474 | *vpn = vsid << (SID_SHIFT - VPN_SHIFT) | seg_off >> VPN_SHIFT; |
dcda287a AK |
475 | case MMU_SEGSIZE_1T: |
476 | /* We only have 40 - 23 bits of seg_off in avpn */ | |
477 | seg_off = (avpn & 0x1ffff) << 23; | |
478 | vsid = avpn >> 17; | |
479 | if (shift < 23) { | |
2454c7e9 | 480 | vpi = (vsid ^ (vsid << 25) ^ pteg) & htab_hash_mask; |
dcda287a | 481 | seg_off |= vpi << shift; |
71bf08b6 | 482 | } |
5524a27d | 483 | *vpn = vsid << (SID_SHIFT_1T - VPN_SHIFT) | seg_off >> VPN_SHIFT; |
dcda287a | 484 | default: |
5524a27d | 485 | *vpn = size = 0; |
3c726f8d | 486 | } |
b1022fbd AK |
487 | *psize = size; |
488 | *apsize = a_size; | |
3c726f8d BH |
489 | } |
490 | ||
f4c82d51 S |
491 | /* |
492 | * clear all mappings on kexec. All cpus are in real mode (or they will | |
493 | * be when they isi), and we are the only one left. We rely on our kernel | |
494 | * mapping being 0xC0's and the hardware ignoring those two real bits. | |
495 | * | |
496 | * TODO: add batching support when enabled. remember, no dynamic memory here, | |
497 | * athough there is the control page available... | |
498 | */ | |
499 | static void native_hpte_clear(void) | |
500 | { | |
5524a27d | 501 | unsigned long vpn = 0; |
f4c82d51 | 502 | unsigned long slot, slots, flags; |
8e561e7e | 503 | struct hash_pte *hptep = htab_address; |
5524a27d | 504 | unsigned long hpte_v; |
f4c82d51 | 505 | unsigned long pteg_count; |
b1022fbd | 506 | int psize, apsize, ssize; |
f4c82d51 S |
507 | |
508 | pteg_count = htab_hash_mask + 1; | |
509 | ||
510 | local_irq_save(flags); | |
511 | ||
512 | /* we take the tlbie lock and hold it. Some hardware will | |
513 | * deadlock if we try to tlbie from two processors at once. | |
514 | */ | |
6b9c9b8a | 515 | raw_spin_lock(&native_tlbie_lock); |
f4c82d51 S |
516 | |
517 | slots = pteg_count * HPTES_PER_GROUP; | |
518 | ||
519 | for (slot = 0; slot < slots; slot++, hptep++) { | |
520 | /* | |
521 | * we could lock the pte here, but we are the only cpu | |
522 | * running, right? and for crash dump, we probably | |
523 | * don't want to wait for a maybe bad cpu. | |
524 | */ | |
96e28449 | 525 | hpte_v = hptep->v; |
f4c82d51 | 526 | |
47f78a49 S |
527 | /* |
528 | * Call __tlbie() here rather than tlbie() since we | |
529 | * already hold the native_tlbie_lock. | |
530 | */ | |
96e28449 | 531 | if (hpte_v & HPTE_V_VALID) { |
b1022fbd | 532 | hpte_decode(hptep, slot, &psize, &apsize, &ssize, &vpn); |
96e28449 | 533 | hptep->v = 0; |
b1022fbd | 534 | __tlbie(vpn, psize, apsize, ssize); |
f4c82d51 S |
535 | } |
536 | } | |
537 | ||
47f78a49 | 538 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
6b9c9b8a | 539 | raw_spin_unlock(&native_tlbie_lock); |
f4c82d51 S |
540 | local_irq_restore(flags); |
541 | } | |
542 | ||
3c726f8d BH |
543 | /* |
544 | * Batched hash table flush, we batch the tlbie's to avoid taking/releasing | |
545 | * the lock all the time | |
546 | */ | |
61b1a942 | 547 | static void native_flush_hash_range(unsigned long number, int local) |
1da177e4 | 548 | { |
5524a27d AK |
549 | unsigned long vpn; |
550 | unsigned long hash, index, hidx, shift, slot; | |
8e561e7e | 551 | struct hash_pte *hptep; |
96e28449 | 552 | unsigned long hpte_v; |
3c726f8d BH |
553 | unsigned long want_v; |
554 | unsigned long flags; | |
555 | real_pte_t pte; | |
1da177e4 | 556 | struct ppc64_tlb_batch *batch = &__get_cpu_var(ppc64_tlb_batch); |
3c726f8d | 557 | unsigned long psize = batch->psize; |
1189be65 | 558 | int ssize = batch->ssize; |
3c726f8d | 559 | int i; |
1da177e4 LT |
560 | |
561 | local_irq_save(flags); | |
562 | ||
1da177e4 | 563 | for (i = 0; i < number; i++) { |
5524a27d | 564 | vpn = batch->vpn[i]; |
3c726f8d BH |
565 | pte = batch->pte[i]; |
566 | ||
5524a27d AK |
567 | pte_iterate_hashed_subpages(pte, psize, vpn, index, shift) { |
568 | hash = hpt_hash(vpn, shift, ssize); | |
3c726f8d BH |
569 | hidx = __rpte_to_hidx(pte, index); |
570 | if (hidx & _PTEIDX_SECONDARY) | |
571 | hash = ~hash; | |
572 | slot = (hash & htab_hash_mask) * HPTES_PER_GROUP; | |
573 | slot += hidx & _PTEIDX_GROUP_IX; | |
574 | hptep = htab_address + slot; | |
74f227b2 | 575 | want_v = hpte_encode_avpn(vpn, psize, ssize); |
3c726f8d BH |
576 | native_lock_hpte(hptep); |
577 | hpte_v = hptep->v; | |
578 | if (!HPTE_V_COMPARE(hpte_v, want_v) || | |
579 | !(hpte_v & HPTE_V_VALID)) | |
580 | native_unlock_hpte(hptep); | |
581 | else | |
582 | hptep->v = 0; | |
583 | } pte_iterate_hashed_end(); | |
1da177e4 LT |
584 | } |
585 | ||
44ae3ab3 | 586 | if (mmu_has_feature(MMU_FTR_TLBIEL) && |
3c726f8d | 587 | mmu_psize_defs[psize].tlbiel && local) { |
1da177e4 | 588 | asm volatile("ptesync":::"memory"); |
3c726f8d | 589 | for (i = 0; i < number; i++) { |
5524a27d | 590 | vpn = batch->vpn[i]; |
3c726f8d BH |
591 | pte = batch->pte[i]; |
592 | ||
5524a27d AK |
593 | pte_iterate_hashed_subpages(pte, psize, |
594 | vpn, index, shift) { | |
b1022fbd | 595 | __tlbiel(vpn, psize, psize, ssize); |
3c726f8d BH |
596 | } pte_iterate_hashed_end(); |
597 | } | |
1da177e4 LT |
598 | asm volatile("ptesync":::"memory"); |
599 | } else { | |
44ae3ab3 | 600 | int lock_tlbie = !mmu_has_feature(MMU_FTR_LOCKLESS_TLBIE); |
1da177e4 LT |
601 | |
602 | if (lock_tlbie) | |
6b9c9b8a | 603 | raw_spin_lock(&native_tlbie_lock); |
1da177e4 LT |
604 | |
605 | asm volatile("ptesync":::"memory"); | |
3c726f8d | 606 | for (i = 0; i < number; i++) { |
5524a27d | 607 | vpn = batch->vpn[i]; |
3c726f8d BH |
608 | pte = batch->pte[i]; |
609 | ||
5524a27d AK |
610 | pte_iterate_hashed_subpages(pte, psize, |
611 | vpn, index, shift) { | |
b1022fbd | 612 | __tlbie(vpn, psize, psize, ssize); |
3c726f8d BH |
613 | } pte_iterate_hashed_end(); |
614 | } | |
1da177e4 LT |
615 | asm volatile("eieio; tlbsync; ptesync":::"memory"); |
616 | ||
617 | if (lock_tlbie) | |
6b9c9b8a | 618 | raw_spin_unlock(&native_tlbie_lock); |
1da177e4 LT |
619 | } |
620 | ||
621 | local_irq_restore(flags); | |
622 | } | |
623 | ||
7d0daae4 | 624 | void __init hpte_init_native(void) |
1da177e4 LT |
625 | { |
626 | ppc_md.hpte_invalidate = native_hpte_invalidate; | |
627 | ppc_md.hpte_updatepp = native_hpte_updatepp; | |
628 | ppc_md.hpte_updateboltedpp = native_hpte_updateboltedpp; | |
629 | ppc_md.hpte_insert = native_hpte_insert; | |
f4c82d51 S |
630 | ppc_md.hpte_remove = native_hpte_remove; |
631 | ppc_md.hpte_clear_all = native_hpte_clear; | |
8e166991 | 632 | ppc_md.flush_hash_range = native_flush_hash_range; |
1da177e4 | 633 | } |