projects / deliverable / linux.git / blob
? search:
summary | shortlog | log | commit | commitdiff | tree
blame | history | raw | HEAD
Merge branch 'merge'
[deliverable/linux.git] / arch / powerpc / mm / hash_low_64.S
1 /*
2 * ppc64 MMU hashtable management routines
3 *
4 * (c) Copyright IBM Corp. 2003, 2005
5 *
6 * Maintained by: Benjamin Herrenschmidt
7 * <benh@kernel.crashing.org>
8 *
9 * This file is covered by the GNU Public Licence v2 as
10 * described in the kernel's COPYING file.
11 */
12
13 #include <linux/config.h>
14 #include <asm/reg.h>
15 #include <asm/pgtable.h>
16 #include <asm/mmu.h>
17 #include <asm/page.h>
18 #include <asm/types.h>
19 #include <asm/ppc_asm.h>
20 #include <asm/asm-offsets.h>
21 #include <asm/cputable.h>
22
23 .text
24
25 /*
26 * Stackframe:
27 *
28 * +-> Back chain (SP + 256)
29 * | General register save area (SP + 112)
30 * | Parameter save area (SP + 48)
31 * | TOC save area (SP + 40)
32 * | link editor doubleword (SP + 32)
33 * | compiler doubleword (SP + 24)
34 * | LR save area (SP + 16)
35 * | CR save area (SP + 8)
36 * SP ---> +-- Back chain (SP + 0)
37 */
38 #define STACKFRAMESIZE 256
39
40 /* Save parameters offsets */
41 #define STK_PARM(i) (STACKFRAMESIZE + 48 + ((i)-3)*8)
42
43 /* Save non-volatile offsets */
44 #define STK_REG(i) (112 + ((i)-14)*8)
45
46
47 #ifndef CONFIG_PPC_64K_PAGES
48
49 /*****************************************************************************
50 * *
51 * 4K SW & 4K HW pages implementation *
52 * *
53 *****************************************************************************/
54
55
56 /*
57 * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
58 * pte_t *ptep, unsigned long trap, int local)
59 *
60 * Adds a 4K page to the hash table in a segment of 4K pages only
61 */
62
63 _GLOBAL(__hash_page_4K)
64 mflr r0
65 std r0,16(r1)
66 stdu r1,-STACKFRAMESIZE(r1)
67 /* Save all params that we need after a function call */
68 std r6,STK_PARM(r6)(r1)
69 std r8,STK_PARM(r8)(r1)
70
71 /* Add _PAGE_PRESENT to access */
72 ori r4,r4,_PAGE_PRESENT
73
74 /* Save non-volatile registers.
75 * r31 will hold "old PTE"
76 * r30 is "new PTE"
77 * r29 is "va"
78 * r28 is a hash value
79 * r27 is hashtab mask (maybe dynamic patched instead ?)
80 */
81 std r27,STK_REG(r27)(r1)
82 std r28,STK_REG(r28)(r1)
83 std r29,STK_REG(r29)(r1)
84 std r30,STK_REG(r30)(r1)
85 std r31,STK_REG(r31)(r1)
86
87 /* Step 1:
88 *
89 * Check permissions, atomically mark the linux PTE busy
90 * and hashed.
91 */
92 1:
93 ldarx r31,0,r6
94 /* Check access rights (access & ~(pte_val(*ptep))) */
95 andc. r0,r4,r31
96 bne- htab_wrong_access
97 /* Check if PTE is busy */
98 andi. r0,r31,_PAGE_BUSY
99 /* If so, just bail out and refault if needed. Someone else
100 * is changing this PTE anyway and might hash it.
101 */
102 bne- htab_bail_ok
103
104 /* Prepare new PTE value (turn access RW into DIRTY, then
105 * add BUSY,HASHPTE and ACCESSED)
106 */
107 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
108 or r30,r30,r31
109 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
110 /* Write the linux PTE atomically (setting busy) */
111 stdcx. r30,0,r6
112 bne- 1b
113 isync
114
115 /* Step 2:
116 *
117 * Insert/Update the HPTE in the hash table. At this point,
118 * r4 (access) is re-useable, we use it for the new HPTE flags
119 */
120
121 /* Calc va and put it in r29 */
122 rldicr r29,r5,28,63-28
123 rldicl r3,r3,0,36
124 or r29,r3,r29
125
126 /* Calculate hash value for primary slot and store it in r28 */
127 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
128 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
129 xor r28,r5,r0
130
131 /* Convert linux PTE bits into HW equivalents */
132 andi. r3,r30,0x1fe /* Get basic set of flags */
133 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
134 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
135 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
136 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
137 andc r0,r30,r0 /* r0 = pte & ~r0 */
138 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
139 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
140
141 /* We eventually do the icache sync here (maybe inline that
142 * code rather than call a C function...)
143 */
144 BEGIN_FTR_SECTION
145 mr r4,r30
146 mr r5,r7
147 bl .hash_page_do_lazy_icache
148 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
149
150 /* At this point, r3 contains new PP bits, save them in
151 * place of "access" in the param area (sic)
152 */
153 std r3,STK_PARM(r4)(r1)
154
155 /* Get htab_hash_mask */
156 ld r4,htab_hash_mask@got(2)
157 ld r27,0(r4) /* htab_hash_mask -> r27 */
158
159 /* Check if we may already be in the hashtable, in this case, we
160 * go to out-of-line code to try to modify the HPTE
161 */
162 andi. r0,r31,_PAGE_HASHPTE
163 bne htab_modify_pte
164
165 htab_insert_pte:
166 /* Clear hpte bits in new pte (we also clear BUSY btw) and
167 * add _PAGE_HASHPTE
168 */
169 lis r0,_PAGE_HPTEFLAGS@h
170 ori r0,r0,_PAGE_HPTEFLAGS@l
171 andc r30,r30,r0
172 ori r30,r30,_PAGE_HASHPTE
173
174 /* physical address r5 */
175 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
176 sldi r5,r5,PAGE_SHIFT
177
178 /* Calculate primary group hash */
179 and r0,r28,r27
180 rldicr r3,r0,3,63-3 /* r3 = (hash & mask) << 3 */
181
182 /* Call ppc_md.hpte_insert */
183 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
184 mr r4,r29 /* Retreive va */
185 li r7,0 /* !bolted, !secondary */
186 li r8,MMU_PAGE_4K /* page size */
187 _GLOBAL(htab_call_hpte_insert1)
188 bl . /* Patched by htab_finish_init() */
189 cmpdi 0,r3,0
190 bge htab_pte_insert_ok /* Insertion successful */
191 cmpdi 0,r3,-2 /* Critical failure */
192 beq- htab_pte_insert_failure
193
194 /* Now try secondary slot */
195
196 /* physical address r5 */
197 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
198 sldi r5,r5,PAGE_SHIFT
199
200 /* Calculate secondary group hash */
201 andc r0,r27,r28
202 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
203
204 /* Call ppc_md.hpte_insert */
205 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
206 mr r4,r29 /* Retreive va */
207 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
208 li r8,MMU_PAGE_4K /* page size */
209 _GLOBAL(htab_call_hpte_insert2)
210 bl . /* Patched by htab_finish_init() */
211 cmpdi 0,r3,0
212 bge+ htab_pte_insert_ok /* Insertion successful */
213 cmpdi 0,r3,-2 /* Critical failure */
214 beq- htab_pte_insert_failure
215
216 /* Both are full, we need to evict something */
217 mftb r0
218 /* Pick a random group based on TB */
219 andi. r0,r0,1
220 mr r5,r28
221 bne 2f
222 not r5,r5
223 2: and r0,r5,r27
224 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
225 /* Call ppc_md.hpte_remove */
226 _GLOBAL(htab_call_hpte_remove)
227 bl . /* Patched by htab_finish_init() */
228
229 /* Try all again */
230 b htab_insert_pte
231
232 htab_bail_ok:
233 li r3,0
234 b htab_bail
235
236 htab_pte_insert_ok:
237 /* Insert slot number & secondary bit in PTE */
238 rldimi r30,r3,12,63-15
239
240 /* Write out the PTE with a normal write
241 * (maybe add eieio may be good still ?)
242 */
243 htab_write_out_pte:
244 ld r6,STK_PARM(r6)(r1)
245 std r30,0(r6)
246 li r3, 0
247 htab_bail:
248 ld r27,STK_REG(r27)(r1)
249 ld r28,STK_REG(r28)(r1)
250 ld r29,STK_REG(r29)(r1)
251 ld r30,STK_REG(r30)(r1)
252 ld r31,STK_REG(r31)(r1)
253 addi r1,r1,STACKFRAMESIZE
254 ld r0,16(r1)
255 mtlr r0
256 blr
257
258 htab_modify_pte:
259 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
260 mr r4,r3
261 rlwinm r3,r31,32-12,29,31
262
263 /* Secondary group ? if yes, get a inverted hash value */
264 mr r5,r28
265 andi. r0,r31,_PAGE_SECONDARY
266 beq 1f
267 not r5,r5
268 1:
269 /* Calculate proper slot value for ppc_md.hpte_updatepp */
270 and r0,r5,r27
271 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
272 add r3,r0,r3 /* add slot idx */
273
274 /* Call ppc_md.hpte_updatepp */
275 mr r5,r29 /* va */
276 li r6,MMU_PAGE_4K /* page size */
277 ld r7,STK_PARM(r8)(r1) /* get "local" param */
278 _GLOBAL(htab_call_hpte_updatepp)
279 bl . /* Patched by htab_finish_init() */
280
281 /* if we failed because typically the HPTE wasn't really here
282 * we try an insertion.
283 */
284 cmpdi 0,r3,-1
285 beq- htab_insert_pte
286
287 /* Clear the BUSY bit and Write out the PTE */
288 li r0,_PAGE_BUSY
289 andc r30,r30,r0
290 b htab_write_out_pte
291
292 htab_wrong_access:
293 /* Bail out clearing reservation */
294 stdcx. r31,0,r6
295 li r3,1
296 b htab_bail
297
298 htab_pte_insert_failure:
299 /* Bail out restoring old PTE */
300 ld r6,STK_PARM(r6)(r1)
301 std r31,0(r6)
302 li r3,-1
303 b htab_bail
304
305
306 #else /* CONFIG_PPC_64K_PAGES */
307
308
309 /*****************************************************************************
310 * *
311 * 64K SW & 4K or 64K HW in a 4K segment pages implementation *
312 * *
313 *****************************************************************************/
314
315 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
316 * pte_t *ptep, unsigned long trap, int local)
317 */
318
319 /*
320 * For now, we do NOT implement Admixed pages
321 */
322 _GLOBAL(__hash_page_4K)
323 mflr r0
324 std r0,16(r1)
325 stdu r1,-STACKFRAMESIZE(r1)
326 /* Save all params that we need after a function call */
327 std r6,STK_PARM(r6)(r1)
328 std r8,STK_PARM(r8)(r1)
329
330 /* Add _PAGE_PRESENT to access */
331 ori r4,r4,_PAGE_PRESENT
332
333 /* Save non-volatile registers.
334 * r31 will hold "old PTE"
335 * r30 is "new PTE"
336 * r29 is "va"
337 * r28 is a hash value
338 * r27 is hashtab mask (maybe dynamic patched instead ?)
339 * r26 is the hidx mask
340 * r25 is the index in combo page
341 */
342 std r25,STK_REG(r25)(r1)
343 std r26,STK_REG(r26)(r1)
344 std r27,STK_REG(r27)(r1)
345 std r28,STK_REG(r28)(r1)
346 std r29,STK_REG(r29)(r1)
347 std r30,STK_REG(r30)(r1)
348 std r31,STK_REG(r31)(r1)
349
350 /* Step 1:
351 *
352 * Check permissions, atomically mark the linux PTE busy
353 * and hashed.
354 */
355 1:
356 ldarx r31,0,r6
357 /* Check access rights (access & ~(pte_val(*ptep))) */
358 andc. r0,r4,r31
359 bne- htab_wrong_access
360 /* Check if PTE is busy */
361 andi. r0,r31,_PAGE_BUSY
362 /* If so, just bail out and refault if needed. Someone else
363 * is changing this PTE anyway and might hash it.
364 */
365 bne- htab_bail_ok
366 /* Prepare new PTE value (turn access RW into DIRTY, then
367 * add BUSY and ACCESSED)
368 */
369 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
370 or r30,r30,r31
371 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
372 /* Write the linux PTE atomically (setting busy) */
373 stdcx. r30,0,r6
374 bne- 1b
375 isync
376
377 /* Step 2:
378 *
379 * Insert/Update the HPTE in the hash table. At this point,
380 * r4 (access) is re-useable, we use it for the new HPTE flags
381 */
382
383 /* Load the hidx index */
384 rldicl r25,r3,64-12,60
385
386 /* Calc va and put it in r29 */
387 rldicr r29,r5,28,63-28 /* r29 = (vsid << 28) */
388 rldicl r3,r3,0,36 /* r3 = (ea & 0x0fffffff) */
389 or r29,r3,r29 /* r29 = va
390
391 /* Calculate hash value for primary slot and store it in r28 */
392 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
393 rldicl r0,r3,64-12,48 /* (ea >> 12) & 0xffff */
394 xor r28,r5,r0
395
396 /* Convert linux PTE bits into HW equivalents */
397 andi. r3,r30,0x1fe /* Get basic set of flags */
398 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
399 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
400 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
401 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
402 andc r0,r30,r0 /* r0 = pte & ~r0 */
403 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
404 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
405
406 /* We eventually do the icache sync here (maybe inline that
407 * code rather than call a C function...)
408 */
409 BEGIN_FTR_SECTION
410 mr r4,r30
411 mr r5,r7
412 bl .hash_page_do_lazy_icache
413 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
414
415 /* At this point, r3 contains new PP bits, save them in
416 * place of "access" in the param area (sic)
417 */
418 std r3,STK_PARM(r4)(r1)
419
420 /* Get htab_hash_mask */
421 ld r4,htab_hash_mask@got(2)
422 ld r27,0(r4) /* htab_hash_mask -> r27 */
423
424 /* Check if we may already be in the hashtable, in this case, we
425 * go to out-of-line code to try to modify the HPTE. We look for
426 * the bit at (1 >> (index + 32))
427 */
428 andi. r0,r31,_PAGE_HASHPTE
429 li r26,0 /* Default hidx */
430 beq htab_insert_pte
431 ld r6,STK_PARM(r6)(r1)
432 ori r26,r6,0x8000 /* Load the hidx mask */
433 ld r26,0(r26)
434 addi r5,r25,36 /* Check actual HPTE_SUB bit, this */
435 rldcr. r0,r31,r5,0 /* must match pgtable.h definition */
436 bne htab_modify_pte
437
438 htab_insert_pte:
439 /* real page number in r5, PTE RPN value + index */
440 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
441 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
442 add r5,r5,r25
443 sldi r5,r5,HW_PAGE_SHIFT
444
445 /* Calculate primary group hash */
446 and r0,r28,r27
447 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
448
449 /* Call ppc_md.hpte_insert */
450 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
451 mr r4,r29 /* Retreive va */
452 li r7,0 /* !bolted, !secondary */
453 li r8,MMU_PAGE_4K /* page size */
454 _GLOBAL(htab_call_hpte_insert1)
455 bl . /* patched by htab_finish_init() */
456 cmpdi 0,r3,0
457 bge htab_pte_insert_ok /* Insertion successful */
458 cmpdi 0,r3,-2 /* Critical failure */
459 beq- htab_pte_insert_failure
460
461 /* Now try secondary slot */
462
463 /* real page number in r5, PTE RPN value + index */
464 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
465 sldi r5,r5,PAGE_SHIFT-HW_PAGE_SHIFT
466 add r5,r5,r25
467 sldi r5,r5,HW_PAGE_SHIFT
468
469 /* Calculate secondary group hash */
470 andc r0,r27,r28
471 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
472
473 /* Call ppc_md.hpte_insert */
474 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
475 mr r4,r29 /* Retreive va */
476 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
477 li r8,MMU_PAGE_4K /* page size */
478 _GLOBAL(htab_call_hpte_insert2)
479 bl . /* patched by htab_finish_init() */
480 cmpdi 0,r3,0
481 bge+ htab_pte_insert_ok /* Insertion successful */
482 cmpdi 0,r3,-2 /* Critical failure */
483 beq- htab_pte_insert_failure
484
485 /* Both are full, we need to evict something */
486 mftb r0
487 /* Pick a random group based on TB */
488 andi. r0,r0,1
489 mr r5,r28
490 bne 2f
491 not r5,r5
492 2: and r0,r5,r27
493 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
494 /* Call ppc_md.hpte_remove */
495 _GLOBAL(htab_call_hpte_remove)
496 bl . /* patched by htab_finish_init() */
497
498 /* Try all again */
499 b htab_insert_pte
500
501 htab_bail_ok:
502 li r3,0
503 b htab_bail
504
505 htab_pte_insert_ok:
506 /* Insert slot number & secondary bit in PTE second half,
507 * clear _PAGE_BUSY and set approriate HPTE slot bit
508 */
509 ld r6,STK_PARM(r6)(r1)
510 li r0,_PAGE_BUSY
511 andc r30,r30,r0
512 /* HPTE SUB bit */
513 li r0,1
514 subfic r5,r25,27 /* Must match bit position in */
515 sld r0,r0,r5 /* pgtable.h */
516 or r30,r30,r0
517 /* hindx */
518 sldi r5,r25,2
519 sld r3,r3,r5
520 li r4,0xf
521 sld r4,r4,r5
522 andc r26,r26,r4
523 or r26,r26,r3
524 ori r5,r6,0x8000
525 std r26,0(r5)
526 lwsync
527 std r30,0(r6)
528 li r3, 0
529 htab_bail:
530 ld r25,STK_REG(r25)(r1)
531 ld r26,STK_REG(r26)(r1)
532 ld r27,STK_REG(r27)(r1)
533 ld r28,STK_REG(r28)(r1)
534 ld r29,STK_REG(r29)(r1)
535 ld r30,STK_REG(r30)(r1)
536 ld r31,STK_REG(r31)(r1)
537 addi r1,r1,STACKFRAMESIZE
538 ld r0,16(r1)
539 mtlr r0
540 blr
541
542 htab_modify_pte:
543 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
544 mr r4,r3
545 sldi r5,r25,2
546 srd r3,r26,r5
547
548 /* Secondary group ? if yes, get a inverted hash value */
549 mr r5,r28
550 andi. r0,r3,0x8 /* page secondary ? */
551 beq 1f
552 not r5,r5
553 1: andi. r3,r3,0x7 /* extract idx alone */
554
555 /* Calculate proper slot value for ppc_md.hpte_updatepp */
556 and r0,r5,r27
557 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
558 add r3,r0,r3 /* add slot idx */
559
560 /* Call ppc_md.hpte_updatepp */
561 mr r5,r29 /* va */
562 li r6,MMU_PAGE_4K /* page size */
563 ld r7,STK_PARM(r8)(r1) /* get "local" param */
564 _GLOBAL(htab_call_hpte_updatepp)
565 bl . /* patched by htab_finish_init() */
566
567 /* if we failed because typically the HPTE wasn't really here
568 * we try an insertion.
569 */
570 cmpdi 0,r3,-1
571 beq- htab_insert_pte
572
573 /* Clear the BUSY bit and Write out the PTE */
574 li r0,_PAGE_BUSY
575 andc r30,r30,r0
576 ld r6,STK_PARM(r6)(r1)
577 std r30,0(r6)
578 li r3,0
579 b htab_bail
580
581 htab_wrong_access:
582 /* Bail out clearing reservation */
583 stdcx. r31,0,r6
584 li r3,1
585 b htab_bail
586
587 htab_pte_insert_failure:
588 /* Bail out restoring old PTE */
589 ld r6,STK_PARM(r6)(r1)
590 std r31,0(r6)
591 li r3,-1
592 b htab_bail
593
594
595 /*****************************************************************************
596 * *
597 * 64K SW & 64K HW in a 64K segment pages implementation *
598 * *
599 *****************************************************************************/
600
601 _GLOBAL(__hash_page_64K)
602 mflr r0
603 std r0,16(r1)
604 stdu r1,-STACKFRAMESIZE(r1)
605 /* Save all params that we need after a function call */
606 std r6,STK_PARM(r6)(r1)
607 std r8,STK_PARM(r8)(r1)
608
609 /* Add _PAGE_PRESENT to access */
610 ori r4,r4,_PAGE_PRESENT
611
612 /* Save non-volatile registers.
613 * r31 will hold "old PTE"
614 * r30 is "new PTE"
615 * r29 is "va"
616 * r28 is a hash value
617 * r27 is hashtab mask (maybe dynamic patched instead ?)
618 */
619 std r27,STK_REG(r27)(r1)
620 std r28,STK_REG(r28)(r1)
621 std r29,STK_REG(r29)(r1)
622 std r30,STK_REG(r30)(r1)
623 std r31,STK_REG(r31)(r1)
624
625 /* Step 1:
626 *
627 * Check permissions, atomically mark the linux PTE busy
628 * and hashed.
629 */
630 1:
631 ldarx r31,0,r6
632 /* Check access rights (access & ~(pte_val(*ptep))) */
633 andc. r0,r4,r31
634 bne- ht64_wrong_access
635 /* Check if PTE is busy */
636 andi. r0,r31,_PAGE_BUSY
637 /* If so, just bail out and refault if needed. Someone else
638 * is changing this PTE anyway and might hash it.
639 */
640 bne- ht64_bail_ok
641 /* Prepare new PTE value (turn access RW into DIRTY, then
642 * add BUSY,HASHPTE and ACCESSED)
643 */
644 rlwinm r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
645 or r30,r30,r31
646 ori r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
647 /* Write the linux PTE atomically (setting busy) */
648 stdcx. r30,0,r6
649 bne- 1b
650 isync
651
652 /* Step 2:
653 *
654 * Insert/Update the HPTE in the hash table. At this point,
655 * r4 (access) is re-useable, we use it for the new HPTE flags
656 */
657
658 /* Calc va and put it in r29 */
659 rldicr r29,r5,28,63-28
660 rldicl r3,r3,0,36
661 or r29,r3,r29
662
663 /* Calculate hash value for primary slot and store it in r28 */
664 rldicl r5,r5,0,25 /* vsid & 0x0000007fffffffff */
665 rldicl r0,r3,64-16,52 /* (ea >> 16) & 0xfff */
666 xor r28,r5,r0
667
668 /* Convert linux PTE bits into HW equivalents */
669 andi. r3,r30,0x1fe /* Get basic set of flags */
670 xori r3,r3,HPTE_R_N /* _PAGE_EXEC -> NOEXEC */
671 rlwinm r0,r30,32-9+1,30,30 /* _PAGE_RW -> _PAGE_USER (r0) */
672 rlwinm r4,r30,32-7+1,30,30 /* _PAGE_DIRTY -> _PAGE_USER (r4) */
673 and r0,r0,r4 /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
674 andc r0,r30,r0 /* r0 = pte & ~r0 */
675 rlwimi r3,r0,32-1,31,31 /* Insert result into PP lsb */
676 ori r3,r3,HPTE_R_C /* Always add "C" bit for perf. */
677
678 /* We eventually do the icache sync here (maybe inline that
679 * code rather than call a C function...)
680 */
681 BEGIN_FTR_SECTION
682 mr r4,r30
683 mr r5,r7
684 bl .hash_page_do_lazy_icache
685 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
686
687 /* At this point, r3 contains new PP bits, save them in
688 * place of "access" in the param area (sic)
689 */
690 std r3,STK_PARM(r4)(r1)
691
692 /* Get htab_hash_mask */
693 ld r4,htab_hash_mask@got(2)
694 ld r27,0(r4) /* htab_hash_mask -> r27 */
695
696 /* Check if we may already be in the hashtable, in this case, we
697 * go to out-of-line code to try to modify the HPTE
698 */
699 andi. r0,r31,_PAGE_HASHPTE
700 bne ht64_modify_pte
701
702 ht64_insert_pte:
703 /* Clear hpte bits in new pte (we also clear BUSY btw) and
704 * add _PAGE_HASHPTE
705 */
706 lis r0,_PAGE_HPTEFLAGS@h
707 ori r0,r0,_PAGE_HPTEFLAGS@l
708 andc r30,r30,r0
709 ori r30,r30,_PAGE_HASHPTE
710
711 /* Phyical address in r5 */
712 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
713 sldi r5,r5,PAGE_SHIFT
714
715 /* Calculate primary group hash */
716 and r0,r28,r27
717 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
718
719 /* Call ppc_md.hpte_insert */
720 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
721 mr r4,r29 /* Retreive va */
722 li r7,0 /* !bolted, !secondary */
723 li r8,MMU_PAGE_64K
724 _GLOBAL(ht64_call_hpte_insert1)
725 bl . /* patched by htab_finish_init() */
726 cmpdi 0,r3,0
727 bge ht64_pte_insert_ok /* Insertion successful */
728 cmpdi 0,r3,-2 /* Critical failure */
729 beq- ht64_pte_insert_failure
730
731 /* Now try secondary slot */
732
733 /* Phyical address in r5 */
734 rldicl r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
735 sldi r5,r5,PAGE_SHIFT
736
737 /* Calculate secondary group hash */
738 andc r0,r27,r28
739 rldicr r3,r0,3,63-3 /* r0 = (~hash & mask) << 3 */
740
741 /* Call ppc_md.hpte_insert */
742 ld r6,STK_PARM(r4)(r1) /* Retreive new pp bits */
743 mr r4,r29 /* Retreive va */
744 li r7,HPTE_V_SECONDARY /* !bolted, secondary */
745 li r8,MMU_PAGE_64K
746 _GLOBAL(ht64_call_hpte_insert2)
747 bl . /* patched by htab_finish_init() */
748 cmpdi 0,r3,0
749 bge+ ht64_pte_insert_ok /* Insertion successful */
750 cmpdi 0,r3,-2 /* Critical failure */
751 beq- ht64_pte_insert_failure
752
753 /* Both are full, we need to evict something */
754 mftb r0
755 /* Pick a random group based on TB */
756 andi. r0,r0,1
757 mr r5,r28
758 bne 2f
759 not r5,r5
760 2: and r0,r5,r27
761 rldicr r3,r0,3,63-3 /* r0 = (hash & mask) << 3 */
762 /* Call ppc_md.hpte_remove */
763 _GLOBAL(ht64_call_hpte_remove)
764 bl . /* patched by htab_finish_init() */
765
766 /* Try all again */
767 b ht64_insert_pte
768
769 ht64_bail_ok:
770 li r3,0
771 b ht64_bail
772
773 ht64_pte_insert_ok:
774 /* Insert slot number & secondary bit in PTE */
775 rldimi r30,r3,12,63-15
776
777 /* Write out the PTE with a normal write
778 * (maybe add eieio may be good still ?)
779 */
780 ht64_write_out_pte:
781 ld r6,STK_PARM(r6)(r1)
782 std r30,0(r6)
783 li r3, 0
784 ht64_bail:
785 ld r27,STK_REG(r27)(r1)
786 ld r28,STK_REG(r28)(r1)
787 ld r29,STK_REG(r29)(r1)
788 ld r30,STK_REG(r30)(r1)
789 ld r31,STK_REG(r31)(r1)
790 addi r1,r1,STACKFRAMESIZE
791 ld r0,16(r1)
792 mtlr r0
793 blr
794
795 ht64_modify_pte:
796 /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
797 mr r4,r3
798 rlwinm r3,r31,32-12,29,31
799
800 /* Secondary group ? if yes, get a inverted hash value */
801 mr r5,r28
802 andi. r0,r31,_PAGE_F_SECOND
803 beq 1f
804 not r5,r5
805 1:
806 /* Calculate proper slot value for ppc_md.hpte_updatepp */
807 and r0,r5,r27
808 rldicr r0,r0,3,63-3 /* r0 = (hash & mask) << 3 */
809 add r3,r0,r3 /* add slot idx */
810
811 /* Call ppc_md.hpte_updatepp */
812 mr r5,r29 /* va */
813 li r6,MMU_PAGE_64K
814 ld r7,STK_PARM(r8)(r1) /* get "local" param */
815 _GLOBAL(ht64_call_hpte_updatepp)
816 bl . /* patched by htab_finish_init() */
817
818 /* if we failed because typically the HPTE wasn't really here
819 * we try an insertion.
820 */
821 cmpdi 0,r3,-1
822 beq- ht64_insert_pte
823
824 /* Clear the BUSY bit and Write out the PTE */
825 li r0,_PAGE_BUSY
826 andc r30,r30,r0
827 b ht64_write_out_pte
828
829 ht64_wrong_access:
830 /* Bail out clearing reservation */
831 stdcx. r31,0,r6
832 li r3,1
833 b ht64_bail
834
835 ht64_pte_insert_failure:
836 /* Bail out restoring old PTE */
837 ld r6,STK_PARM(r6)(r1)
838 std r31,0(r6)
839 li r3,-1
840 b ht64_bail
841
842
843 #endif /* CONFIG_PPC_64K_PAGES */
844
845
846 /*****************************************************************************
847 * *
848 * Huge pages implementation is in hugetlbpage.c *
849 * *
850 *****************************************************************************/
Linux kernel - Deliverables
This page took 0.068611 seconds and 5 git commands to generate.