Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* |
2 | * ppc64 MMU hashtable management routines | |
3 | * | |
3c726f8d | 4 | * (c) Copyright IBM Corp. 2003, 2005 |
1da177e4 LT |
5 | * |
6 | * Maintained by: Benjamin Herrenschmidt | |
7 | * <benh@kernel.crashing.org> | |
8 | * | |
9 | * This file is covered by the GNU Public Licence v2 as | |
10 | * described in the kernel's COPYING file. | |
11 | */ | |
12 | ||
3c726f8d | 13 | #include <linux/config.h> |
ab1f9dac | 14 | #include <asm/reg.h> |
1da177e4 LT |
15 | #include <asm/pgtable.h> |
16 | #include <asm/mmu.h> | |
17 | #include <asm/page.h> | |
18 | #include <asm/types.h> | |
19 | #include <asm/ppc_asm.h> | |
0013a854 | 20 | #include <asm/asm-offsets.h> |
1da177e4 LT |
21 | #include <asm/cputable.h> |
22 | ||
23 | .text | |
24 | ||
25 | /* | |
26 | * Stackframe: | |
27 | * | |
28 | * +-> Back chain (SP + 256) | |
29 | * | General register save area (SP + 112) | |
30 | * | Parameter save area (SP + 48) | |
31 | * | TOC save area (SP + 40) | |
32 | * | link editor doubleword (SP + 32) | |
33 | * | compiler doubleword (SP + 24) | |
34 | * | LR save area (SP + 16) | |
35 | * | CR save area (SP + 8) | |
36 | * SP ---> +-- Back chain (SP + 0) | |
37 | */ | |
38 | #define STACKFRAMESIZE 256 | |
39 | ||
40 | /* Save parameters offsets */ | |
41 | #define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8) | |
42 | ||
43 | /* Save non-volatile offsets */ | |
44 | #define STK_REG(i) (112 + ((i)-14)*8) | |
45 | ||
3c726f8d BH |
46 | |
47 | #ifndef CONFIG_PPC_64K_PAGES | |
48 | ||
49 | /***************************************************************************** | |
50 | * * | |
51 | * 4K SW & 4K HW pages implementation * | |
52 | * * | |
53 | *****************************************************************************/ | |
54 | ||
55 | ||
1da177e4 | 56 | /* |
3c726f8d BH |
57 | * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, |
58 | * pte_t *ptep, unsigned long trap, int local) | |
1da177e4 | 59 | * |
3c726f8d | 60 | * Adds a 4K page to the hash table in a segment of 4K pages only |
1da177e4 LT |
61 | */ |
62 | ||
3c726f8d | 63 | _GLOBAL(__hash_page_4K) |
1da177e4 LT |
64 | mflr r0 |
65 | std r0,16(r1) | |
66 | stdu r1,-STACKFRAMESIZE(r1) | |
67 | /* Save all params that we need after a function call */ | |
68 | std r6,STK_PARM(r6)(r1) | |
69 | std r8,STK_PARM(r8)(r1) | |
70 | ||
71 | /* Add _PAGE_PRESENT to access */ | |
72 | ori r4,r4,_PAGE_PRESENT | |
73 | ||
74 | /* Save non-volatile registers. | |
75 | * r31 will hold "old PTE" | |
76 | * r30 is "new PTE" | |
77 | * r29 is "va" | |
78 | * r28 is a hash value | |
79 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
80 | */ | |
81 | std r27,STK_REG(r27)(r1) | |
82 | std r28,STK_REG(r28)(r1) | |
83 | std r29,STK_REG(r29)(r1) | |
84 | std r30,STK_REG(r30)(r1) | |
85 | std r31,STK_REG(r31)(r1) | |
86 | ||
87 | /* Step 1: | |
88 | * | |
89 | * Check permissions, atomically mark the linux PTE busy | |
90 | * and hashed. | |
91 | */ | |
92 | 1: | |
93 | ldarx r31,0,r6 | |
94 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
95 | andc. r0,r4,r31 | |
96 | bne- htab_wrong_access | |
97 | /* Check if PTE is busy */ | |
98 | andi. r0,r31,_PAGE_BUSY | |
d03853d5 OJ |
99 | /* If so, just bail out and refault if needed. Someone else |
100 | * is changing this PTE anyway and might hash it. | |
101 | */ | |
3c726f8d BH |
102 | bne- htab_bail_ok |
103 | ||
1da177e4 LT |
104 | /* Prepare new PTE value (turn access RW into DIRTY, then |
105 | * add BUSY,HASHPTE and ACCESSED) | |
106 | */ | |
107 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
108 | or r30,r30,r31 | |
109 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | |
110 | /* Write the linux PTE atomically (setting busy) */ | |
111 | stdcx. r30,0,r6 | |
112 | bne- 1b | |
113 | isync | |
114 | ||
115 | /* Step 2: | |
116 | * | |
117 | * Insert/Update the HPTE in the hash table. At this point, | |
118 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
119 | */ | |
120 | ||
121 | /* Calc va and put it in r29 */ | |
122 | rldicr r29,r5,28,63-28 | |
123 | rldicl r3,r3,0,36 | |
124 | or r29,r3,r29 | |
125 | ||
126 | /* Calculate hash value for primary slot and store it in r28 */ | |
127 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
128 | rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ | |
129 | xor r28,r5,r0 | |
130 | ||
131 | /* Convert linux PTE bits into HW equivalents */ | |
132 | andi. r3,r30,0x1fe /* Get basic set of flags */ | |
3c726f8d | 133 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ |
1da177e4 LT |
134 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ |
135 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
3c726f8d | 136 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ |
1da177e4 LT |
137 | andc r0,r30,r0 /* r0 = pte & ~r0 */ |
138 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
c5cf0e30 | 139 | ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ |
1da177e4 LT |
140 | |
141 | /* We eventually do the icache sync here (maybe inline that | |
142 | * code rather than call a C function...) | |
143 | */ | |
1da177e4 LT |
144 | BEGIN_FTR_SECTION |
145 | mr r4,r30 | |
146 | mr r5,r7 | |
147 | bl .hash_page_do_lazy_icache | |
8913ca1c | 148 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) |
1da177e4 LT |
149 | |
150 | /* At this point, r3 contains new PP bits, save them in | |
151 | * place of "access" in the param area (sic) | |
152 | */ | |
153 | std r3,STK_PARM(r4)(r1) | |
154 | ||
155 | /* Get htab_hash_mask */ | |
156 | ld r4,htab_hash_mask@got(2) | |
157 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
158 | ||
159 | /* Check if we may already be in the hashtable, in this case, we | |
160 | * go to out-of-line code to try to modify the HPTE | |
161 | */ | |
162 | andi. r0,r31,_PAGE_HASHPTE | |
163 | bne htab_modify_pte | |
164 | ||
165 | htab_insert_pte: | |
166 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | |
167 | * add _PAGE_HASHPTE | |
168 | */ | |
169 | lis r0,_PAGE_HPTEFLAGS@h | |
170 | ori r0,r0,_PAGE_HPTEFLAGS@l | |
171 | andc r30,r30,r0 | |
172 | ori r30,r30,_PAGE_HASHPTE | |
173 | ||
3c726f8d BH |
174 | /* physical address r5 */ |
175 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
176 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
177 | |
178 | /* Calculate primary group hash */ | |
179 | and r0,r28,r27 | |
3c726f8d | 180 | rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */ |
1da177e4 LT |
181 | |
182 | /* Call ppc_md.hpte_insert */ | |
3c726f8d | 183 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ |
1da177e4 | 184 | mr r4,r29 /* Retreive va */ |
3c726f8d BH |
185 | li r7,0 /* !bolted, !secondary */ |
186 | li r8,MMU_PAGE_4K /* page size */ | |
1da177e4 | 187 | _GLOBAL(htab_call_hpte_insert1) |
3c726f8d | 188 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
189 | cmpdi 0,r3,0 |
190 | bge htab_pte_insert_ok /* Insertion successful */ | |
191 | cmpdi 0,r3,-2 /* Critical failure */ | |
192 | beq- htab_pte_insert_failure | |
193 | ||
194 | /* Now try secondary slot */ | |
195 | ||
3c726f8d BH |
196 | /* physical address r5 */ |
197 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
198 | sldi r5,r5,PAGE_SHIFT | |
1da177e4 LT |
199 | |
200 | /* Calculate secondary group hash */ | |
201 | andc r0,r27,r28 | |
202 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
203 | ||
204 | /* Call ppc_md.hpte_insert */ | |
3c726f8d | 205 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ |
1da177e4 | 206 | mr r4,r29 /* Retreive va */ |
3c726f8d BH |
207 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ |
208 | li r8,MMU_PAGE_4K /* page size */ | |
1da177e4 | 209 | _GLOBAL(htab_call_hpte_insert2) |
3c726f8d | 210 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
211 | cmpdi 0,r3,0 |
212 | bge+ htab_pte_insert_ok /* Insertion successful */ | |
213 | cmpdi 0,r3,-2 /* Critical failure */ | |
214 | beq- htab_pte_insert_failure | |
215 | ||
216 | /* Both are full, we need to evict something */ | |
217 | mftb r0 | |
218 | /* Pick a random group based on TB */ | |
219 | andi. r0,r0,1 | |
220 | mr r5,r28 | |
221 | bne 2f | |
222 | not r5,r5 | |
223 | 2: and r0,r5,r27 | |
224 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
225 | /* Call ppc_md.hpte_remove */ | |
226 | _GLOBAL(htab_call_hpte_remove) | |
3c726f8d | 227 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
228 | |
229 | /* Try all again */ | |
230 | b htab_insert_pte | |
231 | ||
3c726f8d | 232 | htab_bail_ok: |
d03853d5 | 233 | li r3,0 |
3c726f8d | 234 | b htab_bail |
d03853d5 | 235 | |
1da177e4 LT |
236 | htab_pte_insert_ok: |
237 | /* Insert slot number & secondary bit in PTE */ | |
238 | rldimi r30,r3,12,63-15 | |
239 | ||
240 | /* Write out the PTE with a normal write | |
241 | * (maybe add eieio may be good still ?) | |
242 | */ | |
243 | htab_write_out_pte: | |
244 | ld r6,STK_PARM(r6)(r1) | |
245 | std r30,0(r6) | |
246 | li r3, 0 | |
3c726f8d | 247 | htab_bail: |
1da177e4 LT |
248 | ld r27,STK_REG(r27)(r1) |
249 | ld r28,STK_REG(r28)(r1) | |
250 | ld r29,STK_REG(r29)(r1) | |
251 | ld r30,STK_REG(r30)(r1) | |
252 | ld r31,STK_REG(r31)(r1) | |
253 | addi r1,r1,STACKFRAMESIZE | |
254 | ld r0,16(r1) | |
255 | mtlr r0 | |
256 | blr | |
257 | ||
258 | htab_modify_pte: | |
259 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
260 | mr r4,r3 | |
261 | rlwinm r3,r31,32-12,29,31 | |
262 | ||
263 | /* Secondary group ? if yes, get a inverted hash value */ | |
264 | mr r5,r28 | |
265 | andi. r0,r31,_PAGE_SECONDARY | |
266 | beq 1f | |
267 | not r5,r5 | |
268 | 1: | |
269 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
270 | and r0,r5,r27 | |
271 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
272 | add r3,r0,r3 /* add slot idx */ | |
273 | ||
274 | /* Call ppc_md.hpte_updatepp */ | |
275 | mr r5,r29 /* va */ | |
3c726f8d | 276 | li r6,MMU_PAGE_4K /* page size */ |
1da177e4 LT |
277 | ld r7,STK_PARM(r8)(r1) /* get "local" param */ |
278 | _GLOBAL(htab_call_hpte_updatepp) | |
3c726f8d | 279 | bl . /* Patched by htab_finish_init() */ |
1da177e4 LT |
280 | |
281 | /* if we failed because typically the HPTE wasn't really here | |
282 | * we try an insertion. | |
283 | */ | |
284 | cmpdi 0,r3,-1 | |
285 | beq- htab_insert_pte | |
286 | ||
287 | /* Clear the BUSY bit and Write out the PTE */ | |
288 | li r0,_PAGE_BUSY | |
289 | andc r30,r30,r0 | |
290 | b htab_write_out_pte | |
291 | ||
292 | htab_wrong_access: | |
293 | /* Bail out clearing reservation */ | |
294 | stdcx. r31,0,r6 | |
295 | li r3,1 | |
3c726f8d BH |
296 | b htab_bail |
297 | ||
298 | htab_pte_insert_failure: | |
299 | /* Bail out restoring old PTE */ | |
300 | ld r6,STK_PARM(r6)(r1) | |
301 | std r31,0(r6) | |
302 | li r3,-1 | |
303 | b htab_bail | |
304 | ||
305 | ||
306 | #else /* CONFIG_PPC_64K_PAGES */ | |
307 | ||
308 | ||
309 | /***************************************************************************** | |
310 | * * | |
311 | * 64K SW & 4K or 64K HW in a 4K segment pages implementation * | |
312 | * * | |
313 | *****************************************************************************/ | |
314 | ||
315 | /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid, | |
316 | * pte_t *ptep, unsigned long trap, int local) | |
317 | */ | |
318 | ||
319 | /* | |
320 | * For now, we do NOT implement Admixed pages | |
321 | */ | |
322 | _GLOBAL(__hash_page_4K) | |
323 | mflr r0 | |
324 | std r0,16(r1) | |
325 | stdu r1,-STACKFRAMESIZE(r1) | |
326 | /* Save all params that we need after a function call */ | |
327 | std r6,STK_PARM(r6)(r1) | |
328 | std r8,STK_PARM(r8)(r1) | |
329 | ||
330 | /* Add _PAGE_PRESENT to access */ | |
331 | ori r4,r4,_PAGE_PRESENT | |
332 | ||
333 | /* Save non-volatile registers. | |
334 | * r31 will hold "old PTE" | |
335 | * r30 is "new PTE" | |
336 | * r29 is "va" | |
337 | * r28 is a hash value | |
338 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
339 | * r26 is the hidx mask | |
340 | * r25 is the index in combo page | |
341 | */ | |
342 | std r25,STK_REG(r25)(r1) | |
343 | std r26,STK_REG(r26)(r1) | |
344 | std r27,STK_REG(r27)(r1) | |
345 | std r28,STK_REG(r28)(r1) | |
346 | std r29,STK_REG(r29)(r1) | |
347 | std r30,STK_REG(r30)(r1) | |
348 | std r31,STK_REG(r31)(r1) | |
349 | ||
350 | /* Step 1: | |
351 | * | |
352 | * Check permissions, atomically mark the linux PTE busy | |
353 | * and hashed. | |
354 | */ | |
355 | 1: | |
356 | ldarx r31,0,r6 | |
357 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
358 | andc. r0,r4,r31 | |
359 | bne- htab_wrong_access | |
360 | /* Check if PTE is busy */ | |
361 | andi. r0,r31,_PAGE_BUSY | |
362 | /* If so, just bail out and refault if needed. Someone else | |
363 | * is changing this PTE anyway and might hash it. | |
364 | */ | |
365 | bne- htab_bail_ok | |
366 | /* Prepare new PTE value (turn access RW into DIRTY, then | |
367 | * add BUSY and ACCESSED) | |
368 | */ | |
369 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
370 | or r30,r30,r31 | |
371 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | |
bf72aeba | 372 | oris r30,r30,_PAGE_COMBO@h |
3c726f8d BH |
373 | /* Write the linux PTE atomically (setting busy) */ |
374 | stdcx. r30,0,r6 | |
375 | bne- 1b | |
376 | isync | |
377 | ||
378 | /* Step 2: | |
379 | * | |
380 | * Insert/Update the HPTE in the hash table. At this point, | |
381 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
382 | */ | |
383 | ||
384 | /* Load the hidx index */ | |
385 | rldicl r25,r3,64-12,60 | |
386 | ||
387 | /* Calc va and put it in r29 */ | |
388 | rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */ | |
389 | rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */ | |
390 | or r29,r3,r29 /* r29 = va | |
391 | ||
392 | /* Calculate hash value for primary slot and store it in r28 */ | |
393 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
394 | rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */ | |
395 | xor r28,r5,r0 | |
396 | ||
397 | /* Convert linux PTE bits into HW equivalents */ | |
398 | andi. r3,r30,0x1fe /* Get basic set of flags */ | |
399 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ | |
400 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ | |
401 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
402 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ | |
403 | andc r0,r30,r0 /* r0 = pte & ~r0 */ | |
404 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
c5cf0e30 | 405 | ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ |
3c726f8d BH |
406 | |
407 | /* We eventually do the icache sync here (maybe inline that | |
408 | * code rather than call a C function...) | |
409 | */ | |
410 | BEGIN_FTR_SECTION | |
411 | mr r4,r30 | |
412 | mr r5,r7 | |
413 | bl .hash_page_do_lazy_icache | |
414 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) | |
415 | ||
416 | /* At this point, r3 contains new PP bits, save them in | |
417 | * place of "access" in the param area (sic) | |
418 | */ | |
419 | std r3,STK_PARM(r4)(r1) | |
420 | ||
421 | /* Get htab_hash_mask */ | |
422 | ld r4,htab_hash_mask@got(2) | |
423 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
424 | ||
425 | /* Check if we may already be in the hashtable, in this case, we | |
426 | * go to out-of-line code to try to modify the HPTE. We look for | |
427 | * the bit at (1 >> (index + 32)) | |
428 | */ | |
429 | andi. r0,r31,_PAGE_HASHPTE | |
430 | li r26,0 /* Default hidx */ | |
431 | beq htab_insert_pte | |
bf72aeba PM |
432 | |
433 | /* | |
434 | * Check if the pte was already inserted into the hash table | |
435 | * as a 64k HW page, and invalidate the 64k HPTE if so. | |
436 | */ | |
437 | andis. r0,r31,_PAGE_COMBO@h | |
438 | beq htab_inval_old_hpte | |
439 | ||
3c726f8d BH |
440 | ld r6,STK_PARM(r6)(r1) |
441 | ori r26,r6,0x8000 /* Load the hidx mask */ | |
442 | ld r26,0(r26) | |
443 | addi r5,r25,36 /* Check actual HPTE_SUB bit, this */ | |
444 | rldcr. r0,r31,r5,0 /* must match pgtable.h definition */ | |
445 | bne htab_modify_pte | |
446 | ||
447 | htab_insert_pte: | |
448 | /* real page number in r5, PTE RPN value + index */ | |
449 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
450 | sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT | |
451 | add r5,r5,r25 | |
452 | sldi r5,r5,HW_PAGE_SHIFT | |
453 | ||
454 | /* Calculate primary group hash */ | |
455 | and r0,r28,r27 | |
456 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
457 | ||
458 | /* Call ppc_md.hpte_insert */ | |
459 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
460 | mr r4,r29 /* Retreive va */ | |
461 | li r7,0 /* !bolted, !secondary */ | |
462 | li r8,MMU_PAGE_4K /* page size */ | |
463 | _GLOBAL(htab_call_hpte_insert1) | |
464 | bl . /* patched by htab_finish_init() */ | |
465 | cmpdi 0,r3,0 | |
466 | bge htab_pte_insert_ok /* Insertion successful */ | |
467 | cmpdi 0,r3,-2 /* Critical failure */ | |
468 | beq- htab_pte_insert_failure | |
469 | ||
470 | /* Now try secondary slot */ | |
471 | ||
472 | /* real page number in r5, PTE RPN value + index */ | |
473 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
474 | sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT | |
475 | add r5,r5,r25 | |
476 | sldi r5,r5,HW_PAGE_SHIFT | |
477 | ||
478 | /* Calculate secondary group hash */ | |
479 | andc r0,r27,r28 | |
480 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
481 | ||
482 | /* Call ppc_md.hpte_insert */ | |
483 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
484 | mr r4,r29 /* Retreive va */ | |
485 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ | |
486 | li r8,MMU_PAGE_4K /* page size */ | |
487 | _GLOBAL(htab_call_hpte_insert2) | |
488 | bl . /* patched by htab_finish_init() */ | |
489 | cmpdi 0,r3,0 | |
490 | bge+ htab_pte_insert_ok /* Insertion successful */ | |
491 | cmpdi 0,r3,-2 /* Critical failure */ | |
492 | beq- htab_pte_insert_failure | |
493 | ||
494 | /* Both are full, we need to evict something */ | |
495 | mftb r0 | |
496 | /* Pick a random group based on TB */ | |
497 | andi. r0,r0,1 | |
498 | mr r5,r28 | |
499 | bne 2f | |
500 | not r5,r5 | |
501 | 2: and r0,r5,r27 | |
502 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
503 | /* Call ppc_md.hpte_remove */ | |
504 | _GLOBAL(htab_call_hpte_remove) | |
505 | bl . /* patched by htab_finish_init() */ | |
506 | ||
507 | /* Try all again */ | |
508 | b htab_insert_pte | |
509 | ||
bf72aeba PM |
510 | /* |
511 | * Call out to C code to invalidate an 64k HW HPTE that is | |
512 | * useless now that the segment has been switched to 4k pages. | |
513 | */ | |
514 | htab_inval_old_hpte: | |
515 | mr r3,r29 /* virtual addr */ | |
516 | mr r4,r31 /* PTE.pte */ | |
517 | li r5,0 /* PTE.hidx */ | |
518 | li r6,MMU_PAGE_64K /* psize */ | |
519 | ld r7,STK_PARM(r8)(r1) /* local */ | |
520 | bl .flush_hash_page | |
521 | b htab_insert_pte | |
522 | ||
3c726f8d BH |
523 | htab_bail_ok: |
524 | li r3,0 | |
525 | b htab_bail | |
526 | ||
527 | htab_pte_insert_ok: | |
528 | /* Insert slot number & secondary bit in PTE second half, | |
529 | * clear _PAGE_BUSY and set approriate HPTE slot bit | |
530 | */ | |
531 | ld r6,STK_PARM(r6)(r1) | |
532 | li r0,_PAGE_BUSY | |
533 | andc r30,r30,r0 | |
534 | /* HPTE SUB bit */ | |
535 | li r0,1 | |
536 | subfic r5,r25,27 /* Must match bit position in */ | |
537 | sld r0,r0,r5 /* pgtable.h */ | |
538 | or r30,r30,r0 | |
539 | /* hindx */ | |
540 | sldi r5,r25,2 | |
541 | sld r3,r3,r5 | |
542 | li r4,0xf | |
543 | sld r4,r4,r5 | |
544 | andc r26,r26,r4 | |
545 | or r26,r26,r3 | |
546 | ori r5,r6,0x8000 | |
547 | std r26,0(r5) | |
548 | lwsync | |
549 | std r30,0(r6) | |
550 | li r3, 0 | |
551 | htab_bail: | |
552 | ld r25,STK_REG(r25)(r1) | |
553 | ld r26,STK_REG(r26)(r1) | |
554 | ld r27,STK_REG(r27)(r1) | |
555 | ld r28,STK_REG(r28)(r1) | |
556 | ld r29,STK_REG(r29)(r1) | |
557 | ld r30,STK_REG(r30)(r1) | |
558 | ld r31,STK_REG(r31)(r1) | |
559 | addi r1,r1,STACKFRAMESIZE | |
560 | ld r0,16(r1) | |
561 | mtlr r0 | |
562 | blr | |
563 | ||
564 | htab_modify_pte: | |
565 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
566 | mr r4,r3 | |
567 | sldi r5,r25,2 | |
568 | srd r3,r26,r5 | |
569 | ||
570 | /* Secondary group ? if yes, get a inverted hash value */ | |
571 | mr r5,r28 | |
572 | andi. r0,r3,0x8 /* page secondary ? */ | |
573 | beq 1f | |
574 | not r5,r5 | |
575 | 1: andi. r3,r3,0x7 /* extract idx alone */ | |
576 | ||
577 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
578 | and r0,r5,r27 | |
579 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
580 | add r3,r0,r3 /* add slot idx */ | |
581 | ||
582 | /* Call ppc_md.hpte_updatepp */ | |
583 | mr r5,r29 /* va */ | |
584 | li r6,MMU_PAGE_4K /* page size */ | |
585 | ld r7,STK_PARM(r8)(r1) /* get "local" param */ | |
586 | _GLOBAL(htab_call_hpte_updatepp) | |
587 | bl . /* patched by htab_finish_init() */ | |
588 | ||
589 | /* if we failed because typically the HPTE wasn't really here | |
590 | * we try an insertion. | |
591 | */ | |
592 | cmpdi 0,r3,-1 | |
593 | beq- htab_insert_pte | |
594 | ||
595 | /* Clear the BUSY bit and Write out the PTE */ | |
596 | li r0,_PAGE_BUSY | |
597 | andc r30,r30,r0 | |
598 | ld r6,STK_PARM(r6)(r1) | |
599 | std r30,0(r6) | |
600 | li r3,0 | |
601 | b htab_bail | |
602 | ||
603 | htab_wrong_access: | |
604 | /* Bail out clearing reservation */ | |
605 | stdcx. r31,0,r6 | |
606 | li r3,1 | |
607 | b htab_bail | |
1da177e4 LT |
608 | |
609 | htab_pte_insert_failure: | |
610 | /* Bail out restoring old PTE */ | |
611 | ld r6,STK_PARM(r6)(r1) | |
612 | std r31,0(r6) | |
613 | li r3,-1 | |
3c726f8d BH |
614 | b htab_bail |
615 | ||
616 | ||
617 | /***************************************************************************** | |
618 | * * | |
619 | * 64K SW & 64K HW in a 64K segment pages implementation * | |
620 | * * | |
621 | *****************************************************************************/ | |
622 | ||
623 | _GLOBAL(__hash_page_64K) | |
624 | mflr r0 | |
625 | std r0,16(r1) | |
626 | stdu r1,-STACKFRAMESIZE(r1) | |
627 | /* Save all params that we need after a function call */ | |
628 | std r6,STK_PARM(r6)(r1) | |
629 | std r8,STK_PARM(r8)(r1) | |
630 | ||
631 | /* Add _PAGE_PRESENT to access */ | |
632 | ori r4,r4,_PAGE_PRESENT | |
633 | ||
634 | /* Save non-volatile registers. | |
635 | * r31 will hold "old PTE" | |
636 | * r30 is "new PTE" | |
637 | * r29 is "va" | |
638 | * r28 is a hash value | |
639 | * r27 is hashtab mask (maybe dynamic patched instead ?) | |
640 | */ | |
641 | std r27,STK_REG(r27)(r1) | |
642 | std r28,STK_REG(r28)(r1) | |
643 | std r29,STK_REG(r29)(r1) | |
644 | std r30,STK_REG(r30)(r1) | |
645 | std r31,STK_REG(r31)(r1) | |
646 | ||
647 | /* Step 1: | |
648 | * | |
649 | * Check permissions, atomically mark the linux PTE busy | |
650 | * and hashed. | |
651 | */ | |
652 | 1: | |
653 | ldarx r31,0,r6 | |
654 | /* Check access rights (access & ~(pte_val(*ptep))) */ | |
655 | andc. r0,r4,r31 | |
656 | bne- ht64_wrong_access | |
657 | /* Check if PTE is busy */ | |
658 | andi. r0,r31,_PAGE_BUSY | |
659 | /* If so, just bail out and refault if needed. Someone else | |
660 | * is changing this PTE anyway and might hash it. | |
661 | */ | |
662 | bne- ht64_bail_ok | |
bf72aeba PM |
663 | BEGIN_FTR_SECTION |
664 | /* Check if PTE has the cache-inhibit bit set */ | |
665 | andi. r0,r31,_PAGE_NO_CACHE | |
666 | /* If so, bail out and refault as a 4k page */ | |
667 | bne- ht64_bail_ok | |
668 | END_FTR_SECTION_IFCLR(CPU_FTR_CI_LARGE_PAGE) | |
3c726f8d BH |
669 | /* Prepare new PTE value (turn access RW into DIRTY, then |
670 | * add BUSY,HASHPTE and ACCESSED) | |
671 | */ | |
672 | rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */ | |
673 | or r30,r30,r31 | |
674 | ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE | |
675 | /* Write the linux PTE atomically (setting busy) */ | |
676 | stdcx. r30,0,r6 | |
677 | bne- 1b | |
678 | isync | |
679 | ||
680 | /* Step 2: | |
681 | * | |
682 | * Insert/Update the HPTE in the hash table. At this point, | |
683 | * r4 (access) is re-useable, we use it for the new HPTE flags | |
684 | */ | |
685 | ||
686 | /* Calc va and put it in r29 */ | |
687 | rldicr r29,r5,28,63-28 | |
688 | rldicl r3,r3,0,36 | |
689 | or r29,r3,r29 | |
690 | ||
691 | /* Calculate hash value for primary slot and store it in r28 */ | |
692 | rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */ | |
693 | rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */ | |
694 | xor r28,r5,r0 | |
695 | ||
696 | /* Convert linux PTE bits into HW equivalents */ | |
697 | andi. r3,r30,0x1fe /* Get basic set of flags */ | |
698 | xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */ | |
699 | rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */ | |
700 | rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */ | |
701 | and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/ | |
702 | andc r0,r30,r0 /* r0 = pte & ~r0 */ | |
703 | rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */ | |
c5cf0e30 | 704 | ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */ |
3c726f8d BH |
705 | |
706 | /* We eventually do the icache sync here (maybe inline that | |
707 | * code rather than call a C function...) | |
708 | */ | |
709 | BEGIN_FTR_SECTION | |
710 | mr r4,r30 | |
711 | mr r5,r7 | |
712 | bl .hash_page_do_lazy_icache | |
713 | END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE) | |
714 | ||
715 | /* At this point, r3 contains new PP bits, save them in | |
716 | * place of "access" in the param area (sic) | |
717 | */ | |
718 | std r3,STK_PARM(r4)(r1) | |
719 | ||
720 | /* Get htab_hash_mask */ | |
721 | ld r4,htab_hash_mask@got(2) | |
722 | ld r27,0(r4) /* htab_hash_mask -> r27 */ | |
723 | ||
724 | /* Check if we may already be in the hashtable, in this case, we | |
725 | * go to out-of-line code to try to modify the HPTE | |
726 | */ | |
727 | andi. r0,r31,_PAGE_HASHPTE | |
728 | bne ht64_modify_pte | |
729 | ||
730 | ht64_insert_pte: | |
731 | /* Clear hpte bits in new pte (we also clear BUSY btw) and | |
732 | * add _PAGE_HASHPTE | |
733 | */ | |
734 | lis r0,_PAGE_HPTEFLAGS@h | |
735 | ori r0,r0,_PAGE_HPTEFLAGS@l | |
736 | andc r30,r30,r0 | |
737 | ori r30,r30,_PAGE_HASHPTE | |
738 | ||
739 | /* Phyical address in r5 */ | |
740 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
741 | sldi r5,r5,PAGE_SHIFT | |
742 | ||
743 | /* Calculate primary group hash */ | |
744 | and r0,r28,r27 | |
745 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
746 | ||
747 | /* Call ppc_md.hpte_insert */ | |
748 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
749 | mr r4,r29 /* Retreive va */ | |
750 | li r7,0 /* !bolted, !secondary */ | |
751 | li r8,MMU_PAGE_64K | |
752 | _GLOBAL(ht64_call_hpte_insert1) | |
753 | bl . /* patched by htab_finish_init() */ | |
754 | cmpdi 0,r3,0 | |
755 | bge ht64_pte_insert_ok /* Insertion successful */ | |
756 | cmpdi 0,r3,-2 /* Critical failure */ | |
757 | beq- ht64_pte_insert_failure | |
758 | ||
759 | /* Now try secondary slot */ | |
760 | ||
761 | /* Phyical address in r5 */ | |
762 | rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT | |
763 | sldi r5,r5,PAGE_SHIFT | |
764 | ||
765 | /* Calculate secondary group hash */ | |
766 | andc r0,r27,r28 | |
767 | rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */ | |
768 | ||
769 | /* Call ppc_md.hpte_insert */ | |
770 | ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */ | |
771 | mr r4,r29 /* Retreive va */ | |
772 | li r7,HPTE_V_SECONDARY /* !bolted, secondary */ | |
773 | li r8,MMU_PAGE_64K | |
774 | _GLOBAL(ht64_call_hpte_insert2) | |
775 | bl . /* patched by htab_finish_init() */ | |
776 | cmpdi 0,r3,0 | |
777 | bge+ ht64_pte_insert_ok /* Insertion successful */ | |
778 | cmpdi 0,r3,-2 /* Critical failure */ | |
779 | beq- ht64_pte_insert_failure | |
780 | ||
781 | /* Both are full, we need to evict something */ | |
782 | mftb r0 | |
783 | /* Pick a random group based on TB */ | |
784 | andi. r0,r0,1 | |
785 | mr r5,r28 | |
786 | bne 2f | |
787 | not r5,r5 | |
788 | 2: and r0,r5,r27 | |
789 | rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
790 | /* Call ppc_md.hpte_remove */ | |
791 | _GLOBAL(ht64_call_hpte_remove) | |
792 | bl . /* patched by htab_finish_init() */ | |
793 | ||
794 | /* Try all again */ | |
795 | b ht64_insert_pte | |
796 | ||
797 | ht64_bail_ok: | |
798 | li r3,0 | |
799 | b ht64_bail | |
800 | ||
801 | ht64_pte_insert_ok: | |
802 | /* Insert slot number & secondary bit in PTE */ | |
803 | rldimi r30,r3,12,63-15 | |
804 | ||
805 | /* Write out the PTE with a normal write | |
806 | * (maybe add eieio may be good still ?) | |
807 | */ | |
808 | ht64_write_out_pte: | |
809 | ld r6,STK_PARM(r6)(r1) | |
810 | std r30,0(r6) | |
811 | li r3, 0 | |
812 | ht64_bail: | |
813 | ld r27,STK_REG(r27)(r1) | |
814 | ld r28,STK_REG(r28)(r1) | |
815 | ld r29,STK_REG(r29)(r1) | |
816 | ld r30,STK_REG(r30)(r1) | |
817 | ld r31,STK_REG(r31)(r1) | |
818 | addi r1,r1,STACKFRAMESIZE | |
819 | ld r0,16(r1) | |
820 | mtlr r0 | |
821 | blr | |
822 | ||
823 | ht64_modify_pte: | |
824 | /* Keep PP bits in r4 and slot idx from the PTE around in r3 */ | |
825 | mr r4,r3 | |
826 | rlwinm r3,r31,32-12,29,31 | |
827 | ||
828 | /* Secondary group ? if yes, get a inverted hash value */ | |
829 | mr r5,r28 | |
830 | andi. r0,r31,_PAGE_F_SECOND | |
831 | beq 1f | |
832 | not r5,r5 | |
833 | 1: | |
834 | /* Calculate proper slot value for ppc_md.hpte_updatepp */ | |
835 | and r0,r5,r27 | |
836 | rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */ | |
837 | add r3,r0,r3 /* add slot idx */ | |
838 | ||
839 | /* Call ppc_md.hpte_updatepp */ | |
840 | mr r5,r29 /* va */ | |
841 | li r6,MMU_PAGE_64K | |
842 | ld r7,STK_PARM(r8)(r1) /* get "local" param */ | |
843 | _GLOBAL(ht64_call_hpte_updatepp) | |
844 | bl . /* patched by htab_finish_init() */ | |
845 | ||
846 | /* if we failed because typically the HPTE wasn't really here | |
847 | * we try an insertion. | |
848 | */ | |
849 | cmpdi 0,r3,-1 | |
850 | beq- ht64_insert_pte | |
851 | ||
852 | /* Clear the BUSY bit and Write out the PTE */ | |
853 | li r0,_PAGE_BUSY | |
854 | andc r30,r30,r0 | |
855 | b ht64_write_out_pte | |
856 | ||
857 | ht64_wrong_access: | |
858 | /* Bail out clearing reservation */ | |
859 | stdcx. r31,0,r6 | |
860 | li r3,1 | |
861 | b ht64_bail | |
862 | ||
863 | ht64_pte_insert_failure: | |
864 | /* Bail out restoring old PTE */ | |
865 | ld r6,STK_PARM(r6)(r1) | |
866 | std r31,0(r6) | |
867 | li r3,-1 | |
868 | b ht64_bail | |
869 | ||
870 | ||
871 | #endif /* CONFIG_PPC_64K_PAGES */ | |
1da177e4 LT |
872 | |
873 | ||
3c726f8d BH |
874 | /***************************************************************************** |
875 | * * | |
876 | * Huge pages implementation is in hugetlbpage.c * | |
877 | * * | |
878 | *****************************************************************************/ |