arch/powerpc/kernel/head_64.S

   1 /*
   2  *  PowerPC version
   3  *    Copyright (C) 1995-1996 Gary Thomas (gdt@linuxppc.org)
   4  *
   5  *  Rewritten by Cort Dougan (cort@cs.nmt.edu) for PReP
   6  *    Copyright (C) 1996 Cort Dougan <cort@cs.nmt.edu>
   7  *  Adapted for Power Macintosh by Paul Mackerras.
   8  *  Low-level exception handlers and MMU support
   9  *  rewritten by Paul Mackerras.
  10  *    Copyright (C) 1996 Paul Mackerras.
  11  *
  12  *  Adapted for 64bit PowerPC by Dave Engebretsen, Peter Bergner, and
  13  *    Mike Corrigan {engebret|bergner|mikejc}@us.ibm.com
  14  *
  15  *  This file contains the entry point for the 64-bit kernel along
  16  *  with some early initialization code common to all 64-bit powerpc
  17  *  variants.
  18  *
  19  *  This program is free software; you can redistribute it and/or
  20  *  modify it under the terms of the GNU General Public License
  21  *  as published by the Free Software Foundation; either version
  22  *  2 of the License, or (at your option) any later version.
  23  */
  24
  25 #include <linux/threads.h>
  26 #include <linux/init.h>
  27 #include <asm/reg.h>
  28 #include <asm/page.h>
  29 #include <asm/mmu.h>
  30 #include <asm/ppc_asm.h>
  31 #include <asm/asm-offsets.h>
  32 #include <asm/bug.h>
  33 #include <asm/cputable.h>
  34 #include <asm/setup.h>
  35 #include <asm/hvcall.h>
  36 #include <asm/thread_info.h>
  37 #include <asm/firmware.h>
  38 #include <asm/page_64.h>
  39 #include <asm/irqflags.h>
  40 #include <asm/kvm_book3s_asm.h>
  41 #include <asm/ptrace.h>
  42 #include <asm/hw_irq.h>
  43
  44 /* The physical memory is laid out such that the secondary processor
  45  * spin code sits at 0x0000...0x00ff. On server, the vectors follow
  46  * using the layout described in exceptions-64s.S
  47  */
  48
  49 /*
  50  * Entering into this code we make the following assumptions:
  51  *
  52  *  For pSeries or server processors:
  53  *   1. The MMU is off & open firmware is running in real mode.
  54  *   2. The kernel is entered at __start
  55  * -or- For OPAL entry:
  56  *   1. The MMU is off, processor in HV mode, primary CPU enters at 0
  57  *      with device-tree in gpr3. We also get OPAL base in r8 and
  58  *      entry in r9 for debugging purposes
  59  *   2. Secondary processors enter at 0x60 with PIR in gpr3
  60  *
  61  *  For Book3E processors:
  62  *   1. The MMU is on running in AS0 in a state defined in ePAPR
  63  *   2. The kernel is entered at __start
  64  */
  65
  66         .text
  67         .globl  _stext
  68 _stext:
  69 _GLOBAL(__start)
  70         /* NOP this out unconditionally */
  71 BEGIN_FTR_SECTION
  72         FIXUP_ENDIAN
  73         b       __start_initialization_multiplatform
  74 END_FTR_SECTION(0, 1)
  75
  76         /* Catch branch to 0 in real mode */
  77         trap
  78
  79         /* Secondary processors spin on this value until it becomes non-zero.
  80          * When non-zero, it contains the real address of the function the cpu
  81          * should jump to.
  82          */
  83         .balign 8
  84         .globl  __secondary_hold_spinloop
  85 __secondary_hold_spinloop:
  86         .llong  0x0
  87
  88         /* Secondary processors write this value with their cpu # */
  89         /* after they enter the spin loop immediately below.      */
  90         .globl  __secondary_hold_acknowledge
  91 __secondary_hold_acknowledge:
  92         .llong  0x0
  93
  94 #ifdef CONFIG_RELOCATABLE
  95         /* This flag is set to 1 by a loader if the kernel should run
  96          * at the loaded address instead of the linked address.  This
  97          * is used by kexec-tools to keep the the kdump kernel in the
  98          * crash_kernel region.  The loader is responsible for
  99          * observing the alignment requirement.
 100          */
 101         /* Do not move this variable as kexec-tools knows about it. */
 102         . = 0x5c
 103         .globl  __run_at_load
 104 __run_at_load:
 105         .long   0x72756e30      /* "run0" -- relocate to 0 by default */
 106 #endif
 107
 108         . = 0x60
 109 /*
 110  * The following code is used to hold secondary processors
 111  * in a spin loop after they have entered the kernel, but
 112  * before the bulk of the kernel has been relocated.  This code
 113  * is relocated to physical address 0x60 before prom_init is run.
 114  * All of it must fit below the first exception vector at 0x100.
 115  * Use .globl here not _GLOBAL because we want __secondary_hold
 116  * to be the actual text address, not a descriptor.
 117  */
 118         .globl  __secondary_hold
 119 __secondary_hold:
 120         FIXUP_ENDIAN
 121 #ifndef CONFIG_PPC_BOOK3E
 122         mfmsr   r24
 123         ori     r24,r24,MSR_RI
 124         mtmsrd  r24                     /* RI on */
 125 #endif
 126         /* Grab our physical cpu number */
 127         mr      r24,r3
 128         /* stash r4 for book3e */
 129         mr      r25,r4
 130
 131         /* Tell the master cpu we're here */
 132         /* Relocation is off & we are located at an address less */
 133         /* than 0x100, so only need to grab low order offset.    */
 134         std     r24,__secondary_hold_acknowledge-_stext(0)
 135         sync
 136
 137         li      r26,0
 138 #ifdef CONFIG_PPC_BOOK3E
 139         tovirt(r26,r26)
 140 #endif
 141         /* All secondary cpus wait here until told to start. */
 142 100:    ld      r12,__secondary_hold_spinloop-_stext(r26)
 143         cmpdi   0,r12,0
 144         beq     100b
 145
 146 #if defined(CONFIG_SMP) || defined(CONFIG_KEXEC)
 147 #ifdef CONFIG_PPC_BOOK3E
 148         tovirt(r12,r12)
 149 #endif
 150         mtctr   r12
 151         mr      r3,r24
 152         /*
 153          * it may be the case that other platforms have r4 right to
 154          * begin with, this gives us some safety in case it is not
 155          */
 156 #ifdef CONFIG_PPC_BOOK3E
 157         mr      r4,r25
 158 #else
 159         li      r4,0
 160 #endif
 161         /* Make sure that patched code is visible */
 162         isync
 163         bctr
 164 #else
 165         BUG_OPCODE
 166 #endif
 167
 168 /* This value is used to mark exception frames on the stack. */
 169         .section ".toc","aw"
 170 exception_marker:
 171         .tc     ID_72656773_68657265[TC],0x7265677368657265
 172         .text
 173
 174 /*
 175  * On server, we include the exception vectors code here as it
 176  * relies on absolute addressing which is only possible within
 177  * this compilation unit
 178  */
 179 #ifdef CONFIG_PPC_BOOK3S
 180 #include "exceptions-64s.S"
 181 #endif
 182
 183 #ifdef CONFIG_PPC_BOOK3E
 184 _GLOBAL(fsl_secondary_thread_init)
 185         /* Enable branch prediction */
 186         lis     r3,BUCSR_INIT@h
 187         ori     r3,r3,BUCSR_INIT@l
 188         mtspr   SPRN_BUCSR,r3
 189         isync
 190
 191         /*
 192          * Fix PIR to match the linear numbering in the device tree.
 193          *
 194          * On e6500, the reset value of PIR uses the low three bits for
 195          * the thread within a core, and the upper bits for the core
 196          * number.  There are two threads per core, so shift everything
 197          * but the low bit right by two bits so that the cpu numbering is
 198          * continuous.
 199          */
 200         mfspr   r3, SPRN_PIR
 201         rlwimi  r3, r3, 30, 2, 30
 202         mtspr   SPRN_PIR, r3
 203 #endif
 204
 205 _GLOBAL(generic_secondary_thread_init)
 206         mr      r24,r3
 207
 208         /* turn on 64-bit mode */
 209         bl      enable_64b_mode
 210
 211         /* get a valid TOC pointer, wherever we're mapped at */
 212         bl      relative_toc
 213         tovirt(r2,r2)
 214
 215 #ifdef CONFIG_PPC_BOOK3E
 216         /* Book3E initialization */
 217         mr      r3,r24
 218         bl      book3e_secondary_thread_init
 219 #endif
 220         b       generic_secondary_common_init
 221
 222 /*
 223  * On pSeries and most other platforms, secondary processors spin
 224  * in the following code.
 225  * At entry, r3 = this processor's number (physical cpu id)
 226  *
 227  * On Book3E, r4 = 1 to indicate that the initial TLB entry for
 228  * this core already exists (setup via some other mechanism such
 229  * as SCOM before entry).
 230  */
 231 _GLOBAL(generic_secondary_smp_init)
 232         FIXUP_ENDIAN
 233         mr      r24,r3
 234         mr      r25,r4
 235
 236         /* turn on 64-bit mode */
 237         bl      enable_64b_mode
 238
 239         /* get a valid TOC pointer, wherever we're mapped at */
 240         bl      relative_toc
 241         tovirt(r2,r2)
 242
 243 #ifdef CONFIG_PPC_BOOK3E
 244         /* Book3E initialization */
 245         mr      r3,r24
 246         mr      r4,r25
 247         bl      book3e_secondary_core_init
 248 #endif
 249
 250 generic_secondary_common_init:
 251         /* Set up a paca value for this processor. Since we have the
 252          * physical cpu id in r24, we need to search the pacas to find
 253          * which logical id maps to our physical one.
 254          */
 255         LOAD_REG_ADDR(r13, paca)        /* Load paca pointer             */
 256         ld      r13,0(r13)              /* Get base vaddr of paca array  */
 257 #ifndef CONFIG_SMP
 258         addi    r13,r13,PACA_SIZE       /* know r13 if used accidentally */
 259         b       kexec_wait              /* wait for next kernel if !SMP  */
 260 #else
 261         LOAD_REG_ADDR(r7, nr_cpu_ids)   /* Load nr_cpu_ids address       */
 262         lwz     r7,0(r7)                /* also the max paca allocated   */
 263         li      r5,0                    /* logical cpu id                */
 264 1:      lhz     r6,PACAHWCPUID(r13)     /* Load HW procid from paca      */
 265         cmpw    r6,r24                  /* Compare to our id             */
 266         beq     2f
 267         addi    r13,r13,PACA_SIZE       /* Loop to next PACA on miss     */
 268         addi    r5,r5,1
 269         cmpw    r5,r7                   /* Check if more pacas exist     */
 270         blt     1b
 271
 272         mr      r3,r24                  /* not found, copy phys to r3    */
 273         b       kexec_wait              /* next kernel might do better   */
 274
 275 2:      SET_PACA(r13)
 276 #ifdef CONFIG_PPC_BOOK3E
 277         addi    r12,r13,PACA_EXTLB      /* and TLB exc frame in another  */
 278         mtspr   SPRN_SPRG_TLB_EXFRAME,r12
 279 #endif
 280
 281         /* From now on, r24 is expected to be logical cpuid */
 282         mr      r24,r5
 283
 284         /* See if we need to call a cpu state restore handler */
 285         LOAD_REG_ADDR(r23, cur_cpu_spec)
 286         ld      r23,0(r23)
 287         ld      r12,CPU_SPEC_RESTORE(r23)
 288         cmpdi   0,r12,0
 289         beq     3f
 290 #if !defined(_CALL_ELF) || _CALL_ELF != 2
 291         ld      r12,0(r12)
 292 #endif
 293         mtctr   r12
 294         bctrl
 295
 296 3:      LOAD_REG_ADDR(r3, spinning_secondaries) /* Decrement spinning_secondaries */
 297         lwarx   r4,0,r3
 298         subi    r4,r4,1
 299         stwcx.  r4,0,r3
 300         bne     3b
 301         isync
 302
 303 4:      HMT_LOW
 304         lbz     r23,PACAPROCSTART(r13)  /* Test if this processor should */
 305                                         /* start.                        */
 306         cmpwi   0,r23,0
 307         beq     4b                      /* Loop until told to go         */
 308
 309         sync                            /* order paca.run and cur_cpu_spec */
 310         isync                           /* In case code patching happened */
 311
 312         /* Create a temp kernel stack for use before relocation is on.  */
 313         ld      r1,PACAEMERGSP(r13)
 314         subi    r1,r1,STACK_FRAME_OVERHEAD
 315
 316         b       __secondary_start
 317 #endif /* SMP */
 318
 319 /*
 320  * Turn the MMU off.
 321  * Assumes we're mapped EA == RA if the MMU is on.
 322  */
 323 #ifdef CONFIG_PPC_BOOK3S
 324 __mmu_off:
 325         mfmsr   r3
 326         andi.   r0,r3,MSR_IR|MSR_DR
 327         beqlr
 328         mflr    r4
 329         andc    r3,r3,r0
 330         mtspr   SPRN_SRR0,r4
 331         mtspr   SPRN_SRR1,r3
 332         sync
 333         rfid
 334         b       .       /* prevent speculative execution */
 335 #endif
 336
 337
 338 /*
 339  * Here is our main kernel entry point. We support currently 2 kind of entries
 340  * depending on the value of r5.
 341  *
 342  *   r5 != NULL -> OF entry, we go to prom_init, "legacy" parameter content
 343  *                 in r3...r7
 344  *
 345  *   r5 == NULL -> kexec style entry. r3 is a physical pointer to the
 346  *                 DT block, r4 is a physical pointer to the kernel itself
 347  *
 348  */
 349 __start_initialization_multiplatform:
 350         /* Make sure we are running in 64 bits mode */
 351         bl      enable_64b_mode
 352
 353         /* Get TOC pointer (current runtime address) */
 354         bl      relative_toc
 355
 356         /* find out where we are now */
 357         bcl     20,31,$+4
 358 0:      mflr    r26                     /* r26 = runtime addr here */
 359         addis   r26,r26,(_stext - 0b)@ha
 360         addi    r26,r26,(_stext - 0b)@l /* current runtime base addr */
 361
 362         /*
 363          * Are we booted from a PROM Of-type client-interface ?
 364          */
 365         cmpldi  cr0,r5,0
 366         beq     1f
 367         b       __boot_from_prom                /* yes -> prom */
 368 1:
 369         /* Save parameters */
 370         mr      r31,r3
 371         mr      r30,r4
 372 #ifdef CONFIG_PPC_EARLY_DEBUG_OPAL
 373         /* Save OPAL entry */
 374         mr      r28,r8
 375         mr      r29,r9
 376 #endif
 377
 378 #ifdef CONFIG_PPC_BOOK3E
 379         bl      start_initialization_book3e
 380         b       __after_prom_start
 381 #else
 382         /* Setup some critical 970 SPRs before switching MMU off */
 383         mfspr   r0,SPRN_PVR
 384         srwi    r0,r0,16
 385         cmpwi   r0,0x39         /* 970 */
 386         beq     1f
 387         cmpwi   r0,0x3c         /* 970FX */
 388         beq     1f
 389         cmpwi   r0,0x44         /* 970MP */
 390         beq     1f
 391         cmpwi   r0,0x45         /* 970GX */
 392         bne     2f
 393 1:      bl      __cpu_preinit_ppc970
 394 2:
 395
 396         /* Switch off MMU if not already off */
 397         bl      __mmu_off
 398         b       __after_prom_start
 399 #endif /* CONFIG_PPC_BOOK3E */
 400
 401 __boot_from_prom:
 402 #ifdef CONFIG_PPC_OF_BOOT_TRAMPOLINE
 403         /* Save parameters */
 404         mr      r31,r3
 405         mr      r30,r4
 406         mr      r29,r5
 407         mr      r28,r6
 408         mr      r27,r7
 409
 410         /*
 411          * Align the stack to 16-byte boundary
 412          * Depending on the size and layout of the ELF sections in the initial
 413          * boot binary, the stack pointer may be unaligned on PowerMac
 414          */
 415         rldicr  r1,r1,0,59
 416
 417 #ifdef CONFIG_RELOCATABLE
 418         /* Relocate code for where we are now */
 419         mr      r3,r26
 420         bl      relocate
 421 #endif
 422
 423         /* Restore parameters */
 424         mr      r3,r31
 425         mr      r4,r30
 426         mr      r5,r29
 427         mr      r6,r28
 428         mr      r7,r27
 429
 430         /* Do all of the interaction with OF client interface */
 431         mr      r8,r26
 432         bl      prom_init
 433 #endif /* #CONFIG_PPC_OF_BOOT_TRAMPOLINE */
 434
 435         /* We never return. We also hit that trap if trying to boot
 436          * from OF while CONFIG_PPC_OF_BOOT_TRAMPOLINE isn't selected */
 437         trap
 438
 439 __after_prom_start:
 440 #ifdef CONFIG_RELOCATABLE
 441         /* process relocations for the final address of the kernel */
 442         lis     r25,PAGE_OFFSET@highest /* compute virtual base of kernel */
 443         sldi    r25,r25,32
 444         lwz     r7,__run_at_load-_stext(r26)
 445         cmplwi  cr0,r7,1        /* flagged to stay where we are ? */
 446         bne     1f
 447         add     r25,r25,r26
 448 1:      mr      r3,r25
 449         bl      relocate
 450 #endif
 451
 452 /*
 453  * We need to run with _stext at physical address PHYSICAL_START.
 454  * This will leave some code in the first 256B of
 455  * real memory, which are reserved for software use.
 456  *
 457  * Note: This process overwrites the OF exception vectors.
 458  */
 459         li      r3,0                    /* target addr */
 460 #ifdef CONFIG_PPC_BOOK3E
 461         tovirt(r3,r3)                   /* on booke, we already run at PAGE_OFFSET */
 462 #endif
 463         mr.     r4,r26                  /* In some cases the loader may  */
 464         beq     9f                      /* have already put us at zero */
 465         li      r6,0x100                /* Start offset, the first 0x100 */
 466                                         /* bytes were copied earlier.    */
 467 #ifdef CONFIG_PPC_BOOK3E
 468         tovirt(r6,r6)                   /* on booke, we already run at PAGE_OFFSET */
 469 #endif
 470
 471 #ifdef CONFIG_RELOCATABLE
 472 /*
 473  * Check if the kernel has to be running as relocatable kernel based on the
 474  * variable __run_at_load, if it is set the kernel is treated as relocatable
 475  * kernel, otherwise it will be moved to PHYSICAL_START
 476  */
 477         lwz     r7,__run_at_load-_stext(r26)
 478         cmplwi  cr0,r7,1
 479         bne     3f
 480
 481         /* just copy interrupts */
 482         LOAD_REG_IMMEDIATE(r5, __end_interrupts - _stext)
 483         b       5f
 484 3:
 485 #endif
 486         lis     r5,(copy_to_here - _stext)@ha
 487         addi    r5,r5,(copy_to_here - _stext)@l /* # bytes of memory to copy */
 488
 489         bl      copy_and_flush          /* copy the first n bytes        */
 490                                         /* this includes the code being  */
 491                                         /* executed here.                */
 492         addis   r8,r3,(4f - _stext)@ha  /* Jump to the copy of this code */
 493         addi    r12,r8,(4f - _stext)@l  /* that we just made */
 494         mtctr   r12
 495         bctr
 496
 497 .balign 8
 498 p_end:  .llong  _end - _stext
 499
 500 4:      /* Now copy the rest of the kernel up to _end */
 501         addis   r5,r26,(p_end - _stext)@ha
 502         ld      r5,(p_end - _stext)@l(r5)       /* get _end */
 503 5:      bl      copy_and_flush          /* copy the rest */
 504
 505 9:      b       start_here_multiplatform
 506
 507 /*
 508  * Copy routine used to copy the kernel to start at physical address 0
 509  * and flush and invalidate the caches as needed.
 510  * r3 = dest addr, r4 = source addr, r5 = copy limit, r6 = start offset
 511  * on exit, r3, r4, r5 are unchanged, r6 is updated to be >= r5.
 512  *
 513  * Note: this routine *only* clobbers r0, r6 and lr
 514  */
 515 _GLOBAL(copy_and_flush)
 516         addi    r5,r5,-8
 517         addi    r6,r6,-8
 518 4:      li      r0,8                    /* Use the smallest common      */
 519                                         /* denominator cache line       */
 520                                         /* size.  This results in       */
 521                                         /* extra cache line flushes     */
 522                                         /* but operation is correct.    */
 523                                         /* Can't get cache line size    */
 524                                         /* from NACA as it is being     */
 525                                         /* moved too.                   */
 526
 527         mtctr   r0                      /* put # words/line in ctr      */
 528 3:      addi    r6,r6,8                 /* copy a cache line            */
 529         ldx     r0,r6,r4
 530         stdx    r0,r6,r3
 531         bdnz    3b
 532         dcbst   r6,r3                   /* write it to memory           */
 533         sync
 534         icbi    r6,r3                   /* flush the icache line        */
 535         cmpld   0,r6,r5
 536         blt     4b
 537         sync
 538         addi    r5,r5,8
 539         addi    r6,r6,8
 540         isync
 541         blr
 542
 543 .align 8
 544 copy_to_here:
 545
 546 #ifdef CONFIG_SMP
 547 #ifdef CONFIG_PPC_PMAC
 548 /*
 549  * On PowerMac, secondary processors starts from the reset vector, which
 550  * is temporarily turned into a call to one of the functions below.
 551  */
 552         .section ".text";
 553         .align 2 ;
 554
 555         .globl  __secondary_start_pmac_0
 556 __secondary_start_pmac_0:
 557         /* NB the entries for cpus 0, 1, 2 must each occupy 8 bytes. */
 558         li      r24,0
 559         b       1f
 560         li      r24,1
 561         b       1f
 562         li      r24,2
 563         b       1f
 564         li      r24,3
 565 1:
 566
 567 _GLOBAL(pmac_secondary_start)
 568         /* turn on 64-bit mode */
 569         bl      enable_64b_mode
 570
 571         li      r0,0
 572         mfspr   r3,SPRN_HID4
 573         rldimi  r3,r0,40,23     /* clear bit 23 (rm_ci) */
 574         sync
 575         mtspr   SPRN_HID4,r3
 576         isync
 577         sync
 578         slbia
 579
 580         /* get TOC pointer (real address) */
 581         bl      relative_toc
 582         tovirt(r2,r2)
 583
 584         /* Copy some CPU settings from CPU 0 */
 585         bl      __restore_cpu_ppc970
 586
 587         /* pSeries do that early though I don't think we really need it */
 588         mfmsr   r3
 589         ori     r3,r3,MSR_RI
 590         mtmsrd  r3                      /* RI on */
 591
 592         /* Set up a paca value for this processor. */
 593         LOAD_REG_ADDR(r4,paca)          /* Load paca pointer            */
 594         ld      r4,0(r4)                /* Get base vaddr of paca array */
 595         mulli   r13,r24,PACA_SIZE       /* Calculate vaddr of right paca */
 596         add     r13,r13,r4              /* for this processor.          */
 597         SET_PACA(r13)                   /* Save vaddr of paca in an SPRG*/
 598
 599         /* Mark interrupts soft and hard disabled (they might be enabled
 600          * in the PACA when doing hotplug)
 601          */
 602         li      r0,0
 603         stb     r0,PACASOFTIRQEN(r13)
 604         li      r0,PACA_IRQ_HARD_DIS
 605         stb     r0,PACAIRQHAPPENED(r13)
 606
 607         /* Create a temp kernel stack for use before relocation is on.  */
 608         ld      r1,PACAEMERGSP(r13)
 609         subi    r1,r1,STACK_FRAME_OVERHEAD
 610
 611         b       __secondary_start
 612
 613 #endif /* CONFIG_PPC_PMAC */
 614
 615 /*
 616  * This function is called after the master CPU has released the
 617  * secondary processors.  The execution environment is relocation off.
 618  * The paca for this processor has the following fields initialized at
 619  * this point:
 620  *   1. Processor number
 621  *   2. Segment table pointer (virtual address)
 622  * On entry the following are set:
 623  *   r1        = stack pointer (real addr of temp stack)
 624  *   r24       = cpu# (in Linux terms)
 625  *   r13       = paca virtual address
 626  *   SPRG_PACA = paca virtual address
 627  */
 628         .section ".text";
 629         .align 2 ;
 630
 631         .globl  __secondary_start
 632 __secondary_start:
 633         /* Set thread priority to MEDIUM */
 634         HMT_MEDIUM
 635
 636         /* Initialize the kernel stack */
 637         LOAD_REG_ADDR(r3, current_set)
 638         sldi    r28,r24,3               /* get current_set[cpu#]         */
 639         ldx     r14,r3,r28
 640         addi    r14,r14,THREAD_SIZE-STACK_FRAME_OVERHEAD
 641         std     r14,PACAKSAVE(r13)
 642
 643         /* Do early setup for that CPU (SLB and hash table pointer) */
 644         bl      early_setup_secondary
 645
 646         /*
 647          * setup the new stack pointer, but *don't* use this until
 648          * translation is on.
 649          */
 650         mr      r1, r14
 651
 652         /* Clear backchain so we get nice backtraces */
 653         li      r7,0
 654         mtlr    r7
 655
 656         /* Mark interrupts soft and hard disabled (they might be enabled
 657          * in the PACA when doing hotplug)
 658          */
 659         stb     r7,PACASOFTIRQEN(r13)
 660         li      r0,PACA_IRQ_HARD_DIS
 661         stb     r0,PACAIRQHAPPENED(r13)
 662
 663         /* enable MMU and jump to start_secondary */
 664         LOAD_REG_ADDR(r3, start_secondary_prolog)
 665         LOAD_REG_IMMEDIATE(r4, MSR_KERNEL)
 666
 667         mtspr   SPRN_SRR0,r3
 668         mtspr   SPRN_SRR1,r4
 669         RFI
 670         b       .       /* prevent speculative execution */
 671
 672 /*
 673  * Running with relocation on at this point.  All we want to do is
 674  * zero the stack back-chain pointer and get the TOC virtual address
 675  * before going into C code.
 676  */
 677 start_secondary_prolog:
 678         ld      r2,PACATOC(r13)
 679         li      r3,0
 680         std     r3,0(r1)                /* Zero the stack frame pointer */
 681         bl      start_secondary
 682         b       .
 683 /*
 684  * Reset stack pointer and call start_secondary
 685  * to continue with online operation when woken up
 686  * from cede in cpu offline.
 687  */
 688 _GLOBAL(start_secondary_resume)
 689         ld      r1,PACAKSAVE(r13)       /* Reload kernel stack pointer */
 690         li      r3,0
 691         std     r3,0(r1)                /* Zero the stack frame pointer */
 692         bl      start_secondary
 693         b       .
 694 #endif
 695
 696 /*
 697  * This subroutine clobbers r11 and r12
 698  */
 699 enable_64b_mode:
 700         mfmsr   r11                     /* grab the current MSR */
 701 #ifdef CONFIG_PPC_BOOK3E
 702         oris    r11,r11,0x8000          /* CM bit set, we'll set ICM later */
 703         mtmsr   r11
 704 #else /* CONFIG_PPC_BOOK3E */
 705         li      r12,(MSR_64BIT | MSR_ISF)@highest
 706         sldi    r12,r12,48
 707         or      r11,r11,r12
 708         mtmsrd  r11
 709         isync
 710 #endif
 711         blr
 712
 713 /*
 714  * This puts the TOC pointer into r2, offset by 0x8000 (as expected
 715  * by the toolchain).  It computes the correct value for wherever we
 716  * are running at the moment, using position-independent code.
 717  *
 718  * Note: The compiler constructs pointers using offsets from the
 719  * TOC in -mcmodel=medium mode. After we relocate to 0 but before
 720  * the MMU is on we need our TOC to be a virtual address otherwise
 721  * these pointers will be real addresses which may get stored and
 722  * accessed later with the MMU on. We use tovirt() at the call
 723  * sites to handle this.
 724  */
 725 _GLOBAL(relative_toc)
 726         mflr    r0
 727         bcl     20,31,$+4
 728 0:      mflr    r11
 729         ld      r2,(p_toc - 0b)(r11)
 730         add     r2,r2,r11
 731         mtlr    r0
 732         blr
 733
 734 .balign 8
 735 p_toc:  .llong  __toc_start + 0x8000 - 0b
 736
 737 /*
 738  * This is where the main kernel code starts.
 739  */
 740 start_here_multiplatform:
 741         /* set up the TOC */
 742         bl      relative_toc
 743         tovirt(r2,r2)
 744
 745         /* Clear out the BSS. It may have been done in prom_init,
 746          * already but that's irrelevant since prom_init will soon
 747          * be detached from the kernel completely. Besides, we need
 748          * to clear it now for kexec-style entry.
 749          */
 750         LOAD_REG_ADDR(r11,__bss_stop)
 751         LOAD_REG_ADDR(r8,__bss_start)
 752         sub     r11,r11,r8              /* bss size                     */
 753         addi    r11,r11,7               /* round up to an even double word */
 754         srdi.   r11,r11,3               /* shift right by 3             */
 755         beq     4f
 756         addi    r8,r8,-8
 757         li      r0,0
 758         mtctr   r11                     /* zero this many doublewords   */
 759 3:      stdu    r0,8(r8)
 760         bdnz    3b
 761 4:
 762
 763 #ifdef CONFIG_PPC_EARLY_DEBUG_OPAL
 764         /* Setup OPAL entry */
 765         LOAD_REG_ADDR(r11, opal)
 766         std     r28,0(r11);
 767         std     r29,8(r11);
 768 #endif
 769
 770 #ifndef CONFIG_PPC_BOOK3E
 771         mfmsr   r6
 772         ori     r6,r6,MSR_RI
 773         mtmsrd  r6                      /* RI on */
 774 #endif
 775
 776 #ifdef CONFIG_RELOCATABLE
 777         /* Save the physical address we're running at in kernstart_addr */
 778         LOAD_REG_ADDR(r4, kernstart_addr)
 779         clrldi  r0,r25,2
 780         std     r0,0(r4)
 781 #endif
 782
 783         /* The following gets the stack set up with the regs */
 784         /* pointing to the real addr of the kernel stack.  This is   */
 785         /* all done to support the C function call below which sets  */
 786         /* up the htab.  This is done because we have relocated the  */
 787         /* kernel but are still running in real mode. */
 788
 789         LOAD_REG_ADDR(r3,init_thread_union)
 790
 791         /* set up a stack pointer */
 792         addi    r1,r3,THREAD_SIZE
 793         li      r0,0
 794         stdu    r0,-STACK_FRAME_OVERHEAD(r1)
 795
 796         /*
 797          * Do very early kernel initializations, including initial hash table
 798          * and SLB setup before we turn on relocation.
 799          */
 800
 801         /* Restore parameters passed from prom_init/kexec */
 802         mr      r3,r31
 803         bl      early_setup             /* also sets r13 and SPRG_PACA */
 804
 805         LOAD_REG_ADDR(r3, start_here_common)
 806         ld      r4,PACAKMSR(r13)
 807         mtspr   SPRN_SRR0,r3
 808         mtspr   SPRN_SRR1,r4
 809         RFI
 810         b       .       /* prevent speculative execution */
 811
 812         /* This is where all platforms converge execution */
 813
 814 start_here_common:
 815         /* relocation is on at this point */
 816         std     r1,PACAKSAVE(r13)
 817
 818         /* Load the TOC (virtual address) */
 819         ld      r2,PACATOC(r13)
 820
 821         /* Do more system initializations in virtual mode */
 822         bl      setup_system
 823
 824         /* Mark interrupts soft and hard disabled (they might be enabled
 825          * in the PACA when doing hotplug)
 826          */
 827         li      r0,0
 828         stb     r0,PACASOFTIRQEN(r13)
 829         li      r0,PACA_IRQ_HARD_DIS
 830         stb     r0,PACAIRQHAPPENED(r13)
 831
 832         /* Generic kernel entry */
 833         bl      start_kernel
 834
 835         /* Not reached */
 836         BUG_OPCODE
 837
 838 /*
 839  * We put a few things here that have to be page-aligned.
 840  * This stuff goes at the beginning of the bss, which is page-aligned.
 841  */
 842         .section ".bss"
 843
 844         .align  PAGE_SHIFT
 845
 846         .globl  empty_zero_page
 847 empty_zero_page:
 848         .space  PAGE_SIZE
 849
 850         .globl  swapper_pg_dir
 851 swapper_pg_dir:
 852         .space  PGD_TABLE_SIZE