arch/powerpc/mm/hash_low_64.S

   1 /*
   2  * ppc64 MMU hashtable management routines
   3  *
   4  * (c) Copyright IBM Corp. 2003, 2005
   5  *
   6  * Maintained by: Benjamin Herrenschmidt
   7  *                <benh@kernel.crashing.org>
   8  *
   9  * This file is covered by the GNU Public Licence v2 as
  10  * described in the kernel's COPYING file.
  11  */
  12
  13 #include <asm/reg.h>
  14 #include <asm/pgtable.h>
  15 #include <asm/mmu.h>
  16 #include <asm/page.h>
  17 #include <asm/types.h>
  18 #include <asm/ppc_asm.h>
  19 #include <asm/asm-offsets.h>
  20 #include <asm/cputable.h>
  21
  22         .text
  23
  24 /*
  25  * Stackframe:
  26  *
  27  *         +-> Back chain                       (SP + 256)
  28  *         |   General register save area       (SP + 112)
  29  *         |   Parameter save area              (SP + 48)
  30  *         |   TOC save area                    (SP + 40)
  31  *         |   link editor doubleword           (SP + 32)
  32  *         |   compiler doubleword              (SP + 24)
  33  *         |   LR save area                     (SP + 16)
  34  *         |   CR save area                     (SP + 8)
  35  * SP ---> +-- Back chain                       (SP + 0)
  36  */
  37
  38 #ifndef CONFIG_PPC_64K_PAGES
  39
  40 /*****************************************************************************
  41  *                                                                           *
  42  *           4K SW & 4K HW pages implementation                              *
  43  *                                                                           *
  44  *****************************************************************************/
  45
  46
  47 /*
  48  * _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
  49  *               pte_t *ptep, unsigned long trap, unsigned long flags,
  50  *               int ssize)
  51  *
  52  * Adds a 4K page to the hash table in a segment of 4K pages only
  53  */
  54
  55 _GLOBAL(__hash_page_4K)
  56         mflr    r0
  57         std     r0,16(r1)
  58         stdu    r1,-STACKFRAMESIZE(r1)
  59         /* Save all params that we need after a function call */
  60         std     r6,STK_PARAM(R6)(r1)
  61         std     r8,STK_PARAM(R8)(r1)
  62         std     r9,STK_PARAM(R9)(r1)
  63
  64         /* Save non-volatile registers.
  65          * r31 will hold "old PTE"
  66          * r30 is "new PTE"
  67          * r29 is vpn
  68          * r28 is a hash value
  69          * r27 is hashtab mask (maybe dynamic patched instead ?)
  70          */
  71         std     r27,STK_REG(R27)(r1)
  72         std     r28,STK_REG(R28)(r1)
  73         std     r29,STK_REG(R29)(r1)
  74         std     r30,STK_REG(R30)(r1)
  75         std     r31,STK_REG(R31)(r1)
  76
  77         /* Step 1:
  78          *
  79          * Check permissions, atomically mark the linux PTE busy
  80          * and hashed.
  81          */
  82 1:
  83         ldarx   r31,0,r6
  84         /* Check access rights (access & ~(pte_val(*ptep))) */
  85         andc.   r0,r4,r31
  86         bne-    htab_wrong_access
  87         /* Check if PTE is busy */
  88         andi.   r0,r31,_PAGE_BUSY
  89         /* If so, just bail out and refault if needed. Someone else
  90          * is changing this PTE anyway and might hash it.
  91          */
  92         bne-    htab_bail_ok
  93
  94         /* Prepare new PTE value (turn access RW into DIRTY, then
  95          * add BUSY,HASHPTE and ACCESSED)
  96          */
  97         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
  98         or      r30,r30,r31
  99         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED | _PAGE_HASHPTE
 100         /* Write the linux PTE atomically (setting busy) */
 101         stdcx.  r30,0,r6
 102         bne-    1b
 103         isync
 104
 105         /* Step 2:
 106          *
 107          * Insert/Update the HPTE in the hash table. At this point,
 108          * r4 (access) is re-useable, we use it for the new HPTE flags
 109          */
 110
 111 BEGIN_FTR_SECTION
 112         cmpdi   r9,0                    /* check segment size */
 113         bne     3f
 114 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 115         /* Calc vpn and put it in r29 */
 116         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 117         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 118         or      r29,r28,r29
 119         /*
 120          * Calculate hash value for primary slot and store it in r28
 121          * r3 = va, r5 = vsid
 122          * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
 123          */
 124         rldicl  r0,r3,64-12,48
 125         xor     r28,r5,r0               /* hash */
 126         b       4f
 127
 128 3:      /* Calc vpn and put it in r29 */
 129         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 130         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 131         or      r29,r28,r29
 132
 133         /*
 134          * calculate hash value for primary slot and
 135          * store it in r28 for 1T segment
 136          * r3 = va, r5 = vsid
 137          */
 138         sldi    r28,r5,25               /* vsid << 25 */
 139         /* r0 =  (va >> 12) & ((1ul << (40 - 12)) -1) */
 140         rldicl  r0,r3,64-12,36
 141         xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
 142         xor     r28,r28,r0              /* hash */
 143
 144         /* Convert linux PTE bits into HW equivalents */
 145 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 146         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 147         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 148         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 149         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 150         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 151         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 152         /*
 153          * Always add "C" bit for perf. Memory coherence is always enabled
 154          */
 155         ori     r3,r3,HPTE_R_C | HPTE_R_M
 156
 157         /* We eventually do the icache sync here (maybe inline that
 158          * code rather than call a C function...)
 159          */
 160 BEGIN_FTR_SECTION
 161         mr      r4,r30
 162         mr      r5,r7
 163         bl      hash_page_do_lazy_icache
 164 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 165
 166         /* At this point, r3 contains new PP bits, save them in
 167          * place of "access" in the param area (sic)
 168          */
 169         std     r3,STK_PARAM(R4)(r1)
 170
 171         /* Get htab_hash_mask */
 172         ld      r4,htab_hash_mask@got(2)
 173         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 174
 175         /* Check if we may already be in the hashtable, in this case, we
 176          * go to out-of-line code to try to modify the HPTE
 177          */
 178         andi.   r0,r31,_PAGE_HASHPTE
 179         bne     htab_modify_pte
 180
 181 htab_insert_pte:
 182         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 183          * add _PAGE_HASHPTE
 184          */
 185         lis     r0,_PAGE_HPTEFLAGS@h
 186         ori     r0,r0,_PAGE_HPTEFLAGS@l
 187         andc    r30,r30,r0
 188         ori     r30,r30,_PAGE_HASHPTE
 189
 190         /* physical address r5 */
 191         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 192         sldi    r5,r5,PAGE_SHIFT
 193
 194         /* Calculate primary group hash */
 195         and     r0,r28,r27
 196         rldicr  r3,r0,3,63-3            /* r3 = (hash & mask) << 3 */
 197
 198         /* Call ppc_md.hpte_insert */
 199         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 200         mr      r4,r29                  /* Retrieve vpn */
 201         li      r7,0                    /* !bolted, !secondary */
 202         li      r8,MMU_PAGE_4K          /* page size */
 203         li      r9,MMU_PAGE_4K          /* actual page size */
 204         ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 205 .globl htab_call_hpte_insert1
 206 htab_call_hpte_insert1:
 207         bl      .                       /* Patched by htab_finish_init() */
 208         cmpdi   0,r3,0
 209         bge     htab_pte_insert_ok      /* Insertion successful */
 210         cmpdi   0,r3,-2                 /* Critical failure */
 211         beq-    htab_pte_insert_failure
 212
 213         /* Now try secondary slot */
 214
 215         /* physical address r5 */
 216         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 217         sldi    r5,r5,PAGE_SHIFT
 218
 219         /* Calculate secondary group hash */
 220         andc    r0,r27,r28
 221         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 222
 223         /* Call ppc_md.hpte_insert */
 224         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 225         mr      r4,r29                  /* Retrieve vpn */
 226         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 227         li      r8,MMU_PAGE_4K          /* page size */
 228         li      r9,MMU_PAGE_4K          /* actual page size */
 229         ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 230 .globl htab_call_hpte_insert2
 231 htab_call_hpte_insert2:
 232         bl      .                       /* Patched by htab_finish_init() */
 233         cmpdi   0,r3,0
 234         bge+    htab_pte_insert_ok      /* Insertion successful */
 235         cmpdi   0,r3,-2                 /* Critical failure */
 236         beq-    htab_pte_insert_failure
 237
 238         /* Both are full, we need to evict something */
 239         mftb    r0
 240         /* Pick a random group based on TB */
 241         andi.   r0,r0,1
 242         mr      r5,r28
 243         bne     2f
 244         not     r5,r5
 245 2:      and     r0,r5,r27
 246         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 247         /* Call ppc_md.hpte_remove */
 248 .globl htab_call_hpte_remove
 249 htab_call_hpte_remove:
 250         bl      .                       /* Patched by htab_finish_init() */
 251
 252         /* Try all again */
 253         b       htab_insert_pte
 254
 255 htab_bail_ok:
 256         li      r3,0
 257         b       htab_bail
 258
 259 htab_pte_insert_ok:
 260         /* Insert slot number & secondary bit in PTE */
 261         rldimi  r30,r3,12,63-15
 262
 263         /* Write out the PTE with a normal write
 264          * (maybe add eieio may be good still ?)
 265          */
 266 htab_write_out_pte:
 267         ld      r6,STK_PARAM(R6)(r1)
 268         std     r30,0(r6)
 269         li      r3, 0
 270 htab_bail:
 271         ld      r27,STK_REG(R27)(r1)
 272         ld      r28,STK_REG(R28)(r1)
 273         ld      r29,STK_REG(R29)(r1)
 274         ld      r30,STK_REG(R30)(r1)
 275         ld      r31,STK_REG(R31)(r1)
 276         addi    r1,r1,STACKFRAMESIZE
 277         ld      r0,16(r1)
 278         mtlr    r0
 279         blr
 280
 281 htab_modify_pte:
 282         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 283         mr      r4,r3
 284         rlwinm  r3,r31,32-12,29,31
 285
 286         /* Secondary group ? if yes, get a inverted hash value */
 287         mr      r5,r28
 288         andi.   r0,r31,_PAGE_SECONDARY
 289         beq     1f
 290         not     r5,r5
 291 1:
 292         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 293         and     r0,r5,r27
 294         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 295         add     r3,r0,r3        /* add slot idx */
 296
 297         /* Call ppc_md.hpte_updatepp */
 298         mr      r5,r29                  /* vpn */
 299         li      r6,MMU_PAGE_4K          /* base page size */
 300         li      r7,MMU_PAGE_4K          /* actual page size */
 301         ld      r8,STK_PARAM(R9)(r1)    /* segment size */
 302         ld      r9,STK_PARAM(R8)(r1)    /* get "flags" param */
 303 .globl htab_call_hpte_updatepp
 304 htab_call_hpte_updatepp:
 305         bl      .                       /* Patched by htab_finish_init() */
 306
 307         /* if we failed because typically the HPTE wasn't really here
 308          * we try an insertion.
 309          */
 310         cmpdi   0,r3,-1
 311         beq-    htab_insert_pte
 312
 313         /* Clear the BUSY bit and Write out the PTE */
 314         li      r0,_PAGE_BUSY
 315         andc    r30,r30,r0
 316         b       htab_write_out_pte
 317
 318 htab_wrong_access:
 319         /* Bail out clearing reservation */
 320         stdcx.  r31,0,r6
 321         li      r3,1
 322         b       htab_bail
 323
 324 htab_pte_insert_failure:
 325         /* Bail out restoring old PTE */
 326         ld      r6,STK_PARAM(R6)(r1)
 327         std     r31,0(r6)
 328         li      r3,-1
 329         b       htab_bail
 330
 331
 332 #else /* CONFIG_PPC_64K_PAGES */
 333
 334
 335 /*****************************************************************************
 336  *                                                                           *
 337  *           64K SW & 4K or 64K HW in a 4K segment pages implementation      *
 338  *                                                                           *
 339  *****************************************************************************/
 340
 341 /* _hash_page_4K(unsigned long ea, unsigned long access, unsigned long vsid,
 342  *               pte_t *ptep, unsigned long trap, unsigned local flags,
 343  *               int ssize, int subpg_prot)
 344  */
 345
 346 /*
 347  * For now, we do NOT implement Admixed pages
 348  */
 349 _GLOBAL(__hash_page_4K)
 350         mflr    r0
 351         std     r0,16(r1)
 352         stdu    r1,-STACKFRAMESIZE(r1)
 353         /* Save all params that we need after a function call */
 354         std     r6,STK_PARAM(R6)(r1)
 355         std     r8,STK_PARAM(R8)(r1)
 356         std     r9,STK_PARAM(R9)(r1)
 357
 358         /* Save non-volatile registers.
 359          * r31 will hold "old PTE"
 360          * r30 is "new PTE"
 361          * r29 is vpn
 362          * r28 is a hash value
 363          * r27 is hashtab mask (maybe dynamic patched instead ?)
 364          * r26 is the hidx mask
 365          * r25 is the index in combo page
 366          */
 367         std     r25,STK_REG(R25)(r1)
 368         std     r26,STK_REG(R26)(r1)
 369         std     r27,STK_REG(R27)(r1)
 370         std     r28,STK_REG(R28)(r1)
 371         std     r29,STK_REG(R29)(r1)
 372         std     r30,STK_REG(R30)(r1)
 373         std     r31,STK_REG(R31)(r1)
 374
 375         /* Step 1:
 376          *
 377          * Check permissions, atomically mark the linux PTE busy
 378          * and hashed.
 379          */
 380 1:
 381         ldarx   r31,0,r6
 382         /* Check access rights (access & ~(pte_val(*ptep))) */
 383         andc.   r0,r4,r31
 384         bne-    htab_wrong_access
 385         /* Check if PTE is busy */
 386         andi.   r0,r31,_PAGE_BUSY
 387         /* If so, just bail out and refault if needed. Someone else
 388          * is changing this PTE anyway and might hash it.
 389          */
 390         bne-    htab_bail_ok
 391         /* Prepare new PTE value (turn access RW into DIRTY, then
 392          * add BUSY and ACCESSED)
 393          */
 394         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 395         or      r30,r30,r31
 396         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 397         oris    r30,r30,_PAGE_COMBO@h
 398         /* Write the linux PTE atomically (setting busy) */
 399         stdcx.  r30,0,r6
 400         bne-    1b
 401         isync
 402
 403         /* Step 2:
 404          *
 405          * Insert/Update the HPTE in the hash table. At this point,
 406          * r4 (access) is re-useable, we use it for the new HPTE flags
 407          */
 408
 409         /* Load the hidx index */
 410         rldicl  r25,r3,64-12,60
 411
 412 BEGIN_FTR_SECTION
 413         cmpdi   r9,0                    /* check segment size */
 414         bne     3f
 415 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 416         /* Calc vpn and put it in r29 */
 417         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 418         /*
 419          * clrldi r3,r3,64 - SID_SHIFT -->  ea & 0xfffffff
 420          * srdi  r28,r3,VPN_SHIFT
 421          */
 422         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 423         or      r29,r28,r29
 424         /*
 425          * Calculate hash value for primary slot and store it in r28
 426          * r3 = va, r5 = vsid
 427          * r0 = (va >> 12) & ((1ul << (28 - 12)) -1)
 428          */
 429         rldicl  r0,r3,64-12,48
 430         xor     r28,r5,r0               /* hash */
 431         b       4f
 432
 433 3:      /* Calc vpn and put it in r29 */
 434         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 435         /*
 436          * clrldi r3,r3,64 - SID_SHIFT_1T -->  ea & 0xffffffffff
 437          * srdi r28,r3,VPN_SHIFT
 438          */
 439         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 440         or      r29,r28,r29
 441
 442         /*
 443          * Calculate hash value for primary slot and
 444          * store it in r28  for 1T segment
 445          * r3 = va, r5 = vsid
 446          */
 447         sldi    r28,r5,25               /* vsid << 25 */
 448         /* r0 = (va >> 12) & ((1ul << (40 - 12)) -1) */
 449         rldicl  r0,r3,64-12,36
 450         xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
 451         xor     r28,r28,r0              /* hash */
 452
 453         /* Convert linux PTE bits into HW equivalents */
 454 4:
 455 #ifdef CONFIG_PPC_SUBPAGE_PROT
 456         andc    r10,r30,r10
 457         andi.   r3,r10,0x1fe            /* Get basic set of flags */
 458         rlwinm  r0,r10,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 459 #else
 460         andi.   r3,r30,0x1fe            /* Get basic set of flags */
 461         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 462 #endif
 463         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 464         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 465         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 466         andc    r0,r3,r0                /* r0 = pte & ~r0 */
 467         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 468         /*
 469          * Always add "C" bit for perf. Memory coherence is always enabled
 470          */
 471         ori     r3,r3,HPTE_R_C | HPTE_R_M
 472
 473         /* We eventually do the icache sync here (maybe inline that
 474          * code rather than call a C function...)
 475          */
 476 BEGIN_FTR_SECTION
 477         mr      r4,r30
 478         mr      r5,r7
 479         bl      hash_page_do_lazy_icache
 480 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 481
 482         /* At this point, r3 contains new PP bits, save them in
 483          * place of "access" in the param area (sic)
 484          */
 485         std     r3,STK_PARAM(R4)(r1)
 486
 487         /* Get htab_hash_mask */
 488         ld      r4,htab_hash_mask@got(2)
 489         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 490
 491         /* Check if we may already be in the hashtable, in this case, we
 492          * go to out-of-line code to try to modify the HPTE. We look for
 493          * the bit at (1 >> (index + 32))
 494          */
 495         rldicl. r0,r31,64-12,48
 496         li      r26,0                   /* Default hidx */
 497         beq     htab_insert_pte
 498
 499         /*
 500          * Check if the pte was already inserted into the hash table
 501          * as a 64k HW page, and invalidate the 64k HPTE if so.
 502          */
 503         andis.  r0,r31,_PAGE_COMBO@h
 504         beq     htab_inval_old_hpte
 505
 506         ld      r6,STK_PARAM(R6)(r1)
 507         ori     r26,r6,PTE_PAGE_HIDX_OFFSET /* Load the hidx mask. */
 508         ld      r26,0(r26)
 509         addi    r5,r25,36               /* Check actual HPTE_SUB bit, this */
 510         rldcr.  r0,r31,r5,0             /* must match pgtable.h definition */
 511         bne     htab_modify_pte
 512
 513 htab_insert_pte:
 514         /* real page number in r5, PTE RPN value + index */
 515         andis.  r0,r31,_PAGE_4K_PFN@h
 516         srdi    r5,r31,PTE_RPN_SHIFT
 517         bne-    htab_special_pfn
 518         sldi    r5,r5,PAGE_FACTOR
 519         add     r5,r5,r25
 520 htab_special_pfn:
 521         sldi    r5,r5,HW_PAGE_SHIFT
 522
 523         /* Calculate primary group hash */
 524         and     r0,r28,r27
 525         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 526
 527         /* Call ppc_md.hpte_insert */
 528         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 529         mr      r4,r29                  /* Retrieve vpn */
 530         li      r7,0                    /* !bolted, !secondary */
 531         li      r8,MMU_PAGE_4K          /* page size */
 532         li      r9,MMU_PAGE_4K          /* actual page size */
 533         ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 534 .globl htab_call_hpte_insert1
 535 htab_call_hpte_insert1:
 536         bl      .                       /* patched by htab_finish_init() */
 537         cmpdi   0,r3,0
 538         bge     htab_pte_insert_ok      /* Insertion successful */
 539         cmpdi   0,r3,-2                 /* Critical failure */
 540         beq-    htab_pte_insert_failure
 541
 542         /* Now try secondary slot */
 543
 544         /* real page number in r5, PTE RPN value + index */
 545         andis.  r0,r31,_PAGE_4K_PFN@h
 546         srdi    r5,r31,PTE_RPN_SHIFT
 547         bne-    3f
 548         sldi    r5,r5,PAGE_FACTOR
 549         add     r5,r5,r25
 550 3:      sldi    r5,r5,HW_PAGE_SHIFT
 551
 552         /* Calculate secondary group hash */
 553         andc    r0,r27,r28
 554         rldicr  r3,r0,3,63-3            /* r0 = (~hash & mask) << 3 */
 555
 556         /* Call ppc_md.hpte_insert */
 557         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 558         mr      r4,r29                  /* Retrieve vpn */
 559         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 560         li      r8,MMU_PAGE_4K          /* page size */
 561         li      r9,MMU_PAGE_4K          /* actual page size */
 562         ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 563 .globl htab_call_hpte_insert2
 564 htab_call_hpte_insert2:
 565         bl      .                       /* patched by htab_finish_init() */
 566         cmpdi   0,r3,0
 567         bge+    htab_pte_insert_ok      /* Insertion successful */
 568         cmpdi   0,r3,-2                 /* Critical failure */
 569         beq-    htab_pte_insert_failure
 570
 571         /* Both are full, we need to evict something */
 572         mftb    r0
 573         /* Pick a random group based on TB */
 574         andi.   r0,r0,1
 575         mr      r5,r28
 576         bne     2f
 577         not     r5,r5
 578 2:      and     r0,r5,r27
 579         rldicr  r3,r0,3,63-3            /* r0 = (hash & mask) << 3 */
 580         /* Call ppc_md.hpte_remove */
 581 .globl htab_call_hpte_remove
 582 htab_call_hpte_remove:
 583         bl      .                       /* patched by htab_finish_init() */
 584
 585         /* Try all again */
 586         b       htab_insert_pte
 587
 588         /*
 589          * Call out to C code to invalidate an 64k HW HPTE that is
 590          * useless now that the segment has been switched to 4k pages.
 591          */
 592 htab_inval_old_hpte:
 593         mr      r3,r29                  /* vpn */
 594         mr      r4,r31                  /* PTE.pte */
 595         li      r5,0                    /* PTE.hidx */
 596         li      r6,MMU_PAGE_64K         /* psize */
 597         ld      r7,STK_PARAM(R9)(r1)    /* ssize */
 598         ld      r8,STK_PARAM(R8)(r1)    /* flags */
 599         bl      flush_hash_page
 600         /* Clear out _PAGE_HPTE_SUB bits in the new linux PTE */
 601         lis     r0,_PAGE_HPTE_SUB@h
 602         ori     r0,r0,_PAGE_HPTE_SUB@l
 603         andc    r30,r30,r0
 604         b       htab_insert_pte
 605
 606 htab_bail_ok:
 607         li      r3,0
 608         b       htab_bail
 609
 610 htab_pte_insert_ok:
 611         /* Insert slot number & secondary bit in PTE second half,
 612          * clear _PAGE_BUSY and set approriate HPTE slot bit
 613          */
 614         ld      r6,STK_PARAM(R6)(r1)
 615         li      r0,_PAGE_BUSY
 616         andc    r30,r30,r0
 617         /* HPTE SUB bit */
 618         li      r0,1
 619         subfic  r5,r25,27               /* Must match bit position in */
 620         sld     r0,r0,r5                /* pgtable.h */
 621         or      r30,r30,r0
 622         /* hindx */
 623         sldi    r5,r25,2
 624         sld     r3,r3,r5
 625         li      r4,0xf
 626         sld     r4,r4,r5
 627         andc    r26,r26,r4
 628         or      r26,r26,r3
 629         ori     r5,r6,PTE_PAGE_HIDX_OFFSET
 630         std     r26,0(r5)
 631         lwsync
 632         std     r30,0(r6)
 633         li      r3, 0
 634 htab_bail:
 635         ld      r25,STK_REG(R25)(r1)
 636         ld      r26,STK_REG(R26)(r1)
 637         ld      r27,STK_REG(R27)(r1)
 638         ld      r28,STK_REG(R28)(r1)
 639         ld      r29,STK_REG(R29)(r1)
 640         ld      r30,STK_REG(R30)(r1)
 641         ld      r31,STK_REG(R31)(r1)
 642         addi    r1,r1,STACKFRAMESIZE
 643         ld      r0,16(r1)
 644         mtlr    r0
 645         blr
 646
 647 htab_modify_pte:
 648         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 649         mr      r4,r3
 650         sldi    r5,r25,2
 651         srd     r3,r26,r5
 652
 653         /* Secondary group ? if yes, get a inverted hash value */
 654         mr      r5,r28
 655         andi.   r0,r3,0x8 /* page secondary ? */
 656         beq     1f
 657         not     r5,r5
 658 1:      andi.   r3,r3,0x7 /* extract idx alone */
 659
 660         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 661         and     r0,r5,r27
 662         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 663         add     r3,r0,r3        /* add slot idx */
 664
 665         /* Call ppc_md.hpte_updatepp */
 666         mr      r5,r29                  /* vpn */
 667         li      r6,MMU_PAGE_4K          /* base page size */
 668         li      r7,MMU_PAGE_4K          /* actual page size */
 669         ld      r8,STK_PARAM(R9)(r1)    /* segment size */
 670         ld      r9,STK_PARAM(R8)(r1)    /* get "flags" param */
 671 .globl htab_call_hpte_updatepp
 672 htab_call_hpte_updatepp:
 673         bl      .                       /* patched by htab_finish_init() */
 674
 675         /* if we failed because typically the HPTE wasn't really here
 676          * we try an insertion.
 677          */
 678         cmpdi   0,r3,-1
 679         beq-    htab_insert_pte
 680
 681         /* Clear the BUSY bit and Write out the PTE */
 682         li      r0,_PAGE_BUSY
 683         andc    r30,r30,r0
 684         ld      r6,STK_PARAM(R6)(r1)
 685         std     r30,0(r6)
 686         li      r3,0
 687         b       htab_bail
 688
 689 htab_wrong_access:
 690         /* Bail out clearing reservation */
 691         stdcx.  r31,0,r6
 692         li      r3,1
 693         b       htab_bail
 694
 695 htab_pte_insert_failure:
 696         /* Bail out restoring old PTE */
 697         ld      r6,STK_PARAM(R6)(r1)
 698         std     r31,0(r6)
 699         li      r3,-1
 700         b       htab_bail
 701
 702 #endif /* CONFIG_PPC_64K_PAGES */
 703
 704 #ifdef CONFIG_PPC_HAS_HASH_64K
 705
 706 /*****************************************************************************
 707  *                                                                           *
 708  *           64K SW & 64K HW in a 64K segment pages implementation           *
 709  *                                                                           *
 710  *****************************************************************************/
 711
 712 _GLOBAL(__hash_page_64K)
 713         mflr    r0
 714         std     r0,16(r1)
 715         stdu    r1,-STACKFRAMESIZE(r1)
 716         /* Save all params that we need after a function call */
 717         std     r6,STK_PARAM(R6)(r1)
 718         std     r8,STK_PARAM(R8)(r1)
 719         std     r9,STK_PARAM(R9)(r1)
 720
 721         /* Save non-volatile registers.
 722          * r31 will hold "old PTE"
 723          * r30 is "new PTE"
 724          * r29 is vpn
 725          * r28 is a hash value
 726          * r27 is hashtab mask (maybe dynamic patched instead ?)
 727          */
 728         std     r27,STK_REG(R27)(r1)
 729         std     r28,STK_REG(R28)(r1)
 730         std     r29,STK_REG(R29)(r1)
 731         std     r30,STK_REG(R30)(r1)
 732         std     r31,STK_REG(R31)(r1)
 733
 734         /* Step 1:
 735          *
 736          * Check permissions, atomically mark the linux PTE busy
 737          * and hashed.
 738          */
 739 1:
 740         ldarx   r31,0,r6
 741         /* Check access rights (access & ~(pte_val(*ptep))) */
 742         andc.   r0,r4,r31
 743         bne-    ht64_wrong_access
 744         /* Check if PTE is busy */
 745         andi.   r0,r31,_PAGE_BUSY
 746         /* If so, just bail out and refault if needed. Someone else
 747          * is changing this PTE anyway and might hash it.
 748          */
 749         bne-    ht64_bail_ok
 750 BEGIN_FTR_SECTION
 751         /* Check if PTE has the cache-inhibit bit set */
 752         andi.   r0,r31,_PAGE_NO_CACHE
 753         /* If so, bail out and refault as a 4k page */
 754         bne-    ht64_bail_ok
 755 END_MMU_FTR_SECTION_IFCLR(MMU_FTR_CI_LARGE_PAGE)
 756         /* Prepare new PTE value (turn access RW into DIRTY, then
 757          * add BUSY and ACCESSED)
 758          */
 759         rlwinm  r30,r4,32-9+7,31-7,31-7 /* _PAGE_RW -> _PAGE_DIRTY */
 760         or      r30,r30,r31
 761         ori     r30,r30,_PAGE_BUSY | _PAGE_ACCESSED
 762         /* Write the linux PTE atomically (setting busy) */
 763         stdcx.  r30,0,r6
 764         bne-    1b
 765         isync
 766
 767         /* Step 2:
 768          *
 769          * Insert/Update the HPTE in the hash table. At this point,
 770          * r4 (access) is re-useable, we use it for the new HPTE flags
 771          */
 772
 773 BEGIN_FTR_SECTION
 774         cmpdi   r9,0                    /* check segment size */
 775         bne     3f
 776 END_MMU_FTR_SECTION_IFSET(MMU_FTR_1T_SEGMENT)
 777         /* Calc vpn and put it in r29 */
 778         sldi    r29,r5,SID_SHIFT - VPN_SHIFT
 779         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT - VPN_SHIFT)
 780         or      r29,r28,r29
 781
 782         /* Calculate hash value for primary slot and store it in r28
 783          * r3 = va, r5 = vsid
 784          * r0 = (va >> 16) & ((1ul << (28 - 16)) -1)
 785          */
 786         rldicl  r0,r3,64-16,52
 787         xor     r28,r5,r0               /* hash */
 788         b       4f
 789
 790 3:      /* Calc vpn and put it in r29 */
 791         sldi    r29,r5,SID_SHIFT_1T - VPN_SHIFT
 792         rldicl  r28,r3,64 - VPN_SHIFT,64 - (SID_SHIFT_1T - VPN_SHIFT)
 793         or      r29,r28,r29
 794         /*
 795          * calculate hash value for primary slot and
 796          * store it in r28 for 1T segment
 797          * r3 = va, r5 = vsid
 798          */
 799         sldi    r28,r5,25               /* vsid << 25 */
 800         /* r0 = (va >> 16) & ((1ul << (40 - 16)) -1) */
 801         rldicl  r0,r3,64-16,40
 802         xor     r28,r28,r5              /* vsid ^ ( vsid << 25) */
 803         xor     r28,r28,r0              /* hash */
 804
 805         /* Convert linux PTE bits into HW equivalents */
 806 4:      andi.   r3,r30,0x1fe            /* Get basic set of flags */
 807         xori    r3,r3,HPTE_R_N          /* _PAGE_EXEC -> NOEXEC */
 808         rlwinm  r0,r30,32-9+1,30,30     /* _PAGE_RW -> _PAGE_USER (r0) */
 809         rlwinm  r4,r30,32-7+1,30,30     /* _PAGE_DIRTY -> _PAGE_USER (r4) */
 810         and     r0,r0,r4                /* _PAGE_RW & _PAGE_DIRTY ->r0 bit 30*/
 811         andc    r0,r30,r0               /* r0 = pte & ~r0 */
 812         rlwimi  r3,r0,32-1,31,31        /* Insert result into PP lsb */
 813         /*
 814          * Always add "C" bit for perf. Memory coherence is always enabled
 815          */
 816         ori     r3,r3,HPTE_R_C | HPTE_R_M
 817
 818         /* We eventually do the icache sync here (maybe inline that
 819          * code rather than call a C function...)
 820          */
 821 BEGIN_FTR_SECTION
 822         mr      r4,r30
 823         mr      r5,r7
 824         bl      hash_page_do_lazy_icache
 825 END_FTR_SECTION(CPU_FTR_NOEXECUTE|CPU_FTR_COHERENT_ICACHE, CPU_FTR_NOEXECUTE)
 826
 827         /* At this point, r3 contains new PP bits, save them in
 828          * place of "access" in the param area (sic)
 829          */
 830         std     r3,STK_PARAM(R4)(r1)
 831
 832         /* Get htab_hash_mask */
 833         ld      r4,htab_hash_mask@got(2)
 834         ld      r27,0(r4)       /* htab_hash_mask -> r27 */
 835
 836         /* Check if we may already be in the hashtable, in this case, we
 837          * go to out-of-line code to try to modify the HPTE
 838          */
 839         rldicl. r0,r31,64-12,48
 840         bne     ht64_modify_pte
 841
 842 ht64_insert_pte:
 843         /* Clear hpte bits in new pte (we also clear BUSY btw) and
 844          * add _PAGE_HPTE_SUB0
 845          */
 846         lis     r0,_PAGE_HPTEFLAGS@h
 847         ori     r0,r0,_PAGE_HPTEFLAGS@l
 848         andc    r30,r30,r0
 849 #ifdef CONFIG_PPC_64K_PAGES
 850         oris    r30,r30,_PAGE_HPTE_SUB0@h
 851 #else
 852         ori     r30,r30,_PAGE_HASHPTE
 853 #endif
 854         /* Phyical address in r5 */
 855         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 856         sldi    r5,r5,PAGE_SHIFT
 857
 858         /* Calculate primary group hash */
 859         and     r0,r28,r27
 860         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 861
 862         /* Call ppc_md.hpte_insert */
 863         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 864         mr      r4,r29                  /* Retrieve vpn */
 865         li      r7,0                    /* !bolted, !secondary */
 866         li      r8,MMU_PAGE_64K
 867         li      r9,MMU_PAGE_64K         /* actual page size */
 868         ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 869 .globl ht64_call_hpte_insert1
 870 ht64_call_hpte_insert1:
 871         bl      .                       /* patched by htab_finish_init() */
 872         cmpdi   0,r3,0
 873         bge     ht64_pte_insert_ok      /* Insertion successful */
 874         cmpdi   0,r3,-2                 /* Critical failure */
 875         beq-    ht64_pte_insert_failure
 876
 877         /* Now try secondary slot */
 878
 879         /* Phyical address in r5 */
 880         rldicl  r5,r31,64-PTE_RPN_SHIFT,PTE_RPN_SHIFT
 881         sldi    r5,r5,PAGE_SHIFT
 882
 883         /* Calculate secondary group hash */
 884         andc    r0,r27,r28
 885         rldicr  r3,r0,3,63-3    /* r0 = (~hash & mask) << 3 */
 886
 887         /* Call ppc_md.hpte_insert */
 888         ld      r6,STK_PARAM(R4)(r1)    /* Retrieve new pp bits */
 889         mr      r4,r29                  /* Retrieve vpn */
 890         li      r7,HPTE_V_SECONDARY     /* !bolted, secondary */
 891         li      r8,MMU_PAGE_64K
 892         li      r9,MMU_PAGE_64K         /* actual page size */
 893         ld      r10,STK_PARAM(R9)(r1)   /* segment size */
 894 .globl ht64_call_hpte_insert2
 895 ht64_call_hpte_insert2:
 896         bl      .                       /* patched by htab_finish_init() */
 897         cmpdi   0,r3,0
 898         bge+    ht64_pte_insert_ok      /* Insertion successful */
 899         cmpdi   0,r3,-2                 /* Critical failure */
 900         beq-    ht64_pte_insert_failure
 901
 902         /* Both are full, we need to evict something */
 903         mftb    r0
 904         /* Pick a random group based on TB */
 905         andi.   r0,r0,1
 906         mr      r5,r28
 907         bne     2f
 908         not     r5,r5
 909 2:      and     r0,r5,r27
 910         rldicr  r3,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 911         /* Call ppc_md.hpte_remove */
 912 .globl ht64_call_hpte_remove
 913 ht64_call_hpte_remove:
 914         bl      .                       /* patched by htab_finish_init() */
 915
 916         /* Try all again */
 917         b       ht64_insert_pte
 918
 919 ht64_bail_ok:
 920         li      r3,0
 921         b       ht64_bail
 922
 923 ht64_pte_insert_ok:
 924         /* Insert slot number & secondary bit in PTE */
 925         rldimi  r30,r3,12,63-15
 926
 927         /* Write out the PTE with a normal write
 928          * (maybe add eieio may be good still ?)
 929          */
 930 ht64_write_out_pte:
 931         ld      r6,STK_PARAM(R6)(r1)
 932         std     r30,0(r6)
 933         li      r3, 0
 934 ht64_bail:
 935         ld      r27,STK_REG(R27)(r1)
 936         ld      r28,STK_REG(R28)(r1)
 937         ld      r29,STK_REG(R29)(r1)
 938         ld      r30,STK_REG(R30)(r1)
 939         ld      r31,STK_REG(R31)(r1)
 940         addi    r1,r1,STACKFRAMESIZE
 941         ld      r0,16(r1)
 942         mtlr    r0
 943         blr
 944
 945 ht64_modify_pte:
 946         /* Keep PP bits in r4 and slot idx from the PTE around in r3 */
 947         mr      r4,r3
 948         rlwinm  r3,r31,32-12,29,31
 949
 950         /* Secondary group ? if yes, get a inverted hash value */
 951         mr      r5,r28
 952         andi.   r0,r31,_PAGE_F_SECOND
 953         beq     1f
 954         not     r5,r5
 955 1:
 956         /* Calculate proper slot value for ppc_md.hpte_updatepp */
 957         and     r0,r5,r27
 958         rldicr  r0,r0,3,63-3    /* r0 = (hash & mask) << 3 */
 959         add     r3,r0,r3        /* add slot idx */
 960
 961         /* Call ppc_md.hpte_updatepp */
 962         mr      r5,r29                  /* vpn */
 963         li      r6,MMU_PAGE_64K         /* base page size */
 964         li      r7,MMU_PAGE_64K         /* actual page size */
 965         ld      r8,STK_PARAM(R9)(r1)    /* segment size */
 966         ld      r9,STK_PARAM(R8)(r1)    /* get "flags" param */
 967 .globl ht64_call_hpte_updatepp
 968 ht64_call_hpte_updatepp:
 969         bl      .                       /* patched by htab_finish_init() */
 970
 971         /* if we failed because typically the HPTE wasn't really here
 972          * we try an insertion.
 973          */
 974         cmpdi   0,r3,-1
 975         beq-    ht64_insert_pte
 976
 977         /* Clear the BUSY bit and Write out the PTE */
 978         li      r0,_PAGE_BUSY
 979         andc    r30,r30,r0
 980         b       ht64_write_out_pte
 981
 982 ht64_wrong_access:
 983         /* Bail out clearing reservation */
 984         stdcx.  r31,0,r6
 985         li      r3,1
 986         b       ht64_bail
 987
 988 ht64_pte_insert_failure:
 989         /* Bail out restoring old PTE */
 990         ld      r6,STK_PARAM(R6)(r1)
 991         std     r31,0(r6)
 992         li      r3,-1
 993         b       ht64_bail
 994
 995
 996 #endif /* CONFIG_PPC_HAS_HASH_64K */
 997
 998
 999 /*****************************************************************************
1000  *                                                                           *
1001  *           Huge pages implementation is in hugetlbpage.c                   *
1002  *                                                                           *
1003  *****************************************************************************/