arch/arm64/kernel/setup.c

   1 /*
   2  * Based on arch/arm/kernel/setup.c
   3  *
   4  * Copyright (C) 1995-2001 Russell King
   5  * Copyright (C) 2012 ARM Ltd.
   6  *
   7  * This program is free software; you can redistribute it and/or modify
   8  * it under the terms of the GNU General Public License version 2 as
   9  * published by the Free Software Foundation.
  10  *
  11  * This program is distributed in the hope that it will be useful,
  12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
  13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
  14  * GNU General Public License for more details.
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * along with this program.  If not, see <http://www.gnu.org/licenses/>.
  18  */
  19
  20 #include <linux/acpi.h>
  21 #include <linux/export.h>
  22 #include <linux/kernel.h>
  23 #include <linux/stddef.h>
  24 #include <linux/ioport.h>
  25 #include <linux/delay.h>
  26 #include <linux/utsname.h>
  27 #include <linux/initrd.h>
  28 #include <linux/console.h>
  29 #include <linux/cache.h>
  30 #include <linux/bootmem.h>
  31 #include <linux/seq_file.h>
  32 #include <linux/screen_info.h>
  33 #include <linux/init.h>
  34 #include <linux/kexec.h>
  35 #include <linux/crash_dump.h>
  36 #include <linux/root_dev.h>
  37 #include <linux/clk-provider.h>
  38 #include <linux/cpu.h>
  39 #include <linux/interrupt.h>
  40 #include <linux/smp.h>
  41 #include <linux/fs.h>
  42 #include <linux/proc_fs.h>
  43 #include <linux/memblock.h>
  44 #include <linux/of_iommu.h>
  45 #include <linux/of_fdt.h>
  46 #include <linux/of_platform.h>
  47 #include <linux/efi.h>
  48 #include <linux/personality.h>
  49
  50 #include <asm/acpi.h>
  51 #include <asm/fixmap.h>
  52 #include <asm/cpu.h>
  53 #include <asm/cputype.h>
  54 #include <asm/elf.h>
  55 #include <asm/cpufeature.h>
  56 #include <asm/cpu_ops.h>
  57 #include <asm/sections.h>
  58 #include <asm/setup.h>
  59 #include <asm/smp_plat.h>
  60 #include <asm/cacheflush.h>
  61 #include <asm/tlbflush.h>
  62 #include <asm/traps.h>
  63 #include <asm/memblock.h>
  64 #include <asm/psci.h>
  65 #include <asm/efi.h>
  66 #include <asm/virt.h>
  67 #include <asm/xen/hypervisor.h>
  68
  69 unsigned long elf_hwcap __read_mostly;
  70 EXPORT_SYMBOL_GPL(elf_hwcap);
  71
  72 #ifdef CONFIG_COMPAT
  73 #define COMPAT_ELF_HWCAP_DEFAULT        \
  74                                 (COMPAT_HWCAP_HALF|COMPAT_HWCAP_THUMB|\
  75                                  COMPAT_HWCAP_FAST_MULT|COMPAT_HWCAP_EDSP|\
  76                                  COMPAT_HWCAP_TLS|COMPAT_HWCAP_VFP|\
  77                                  COMPAT_HWCAP_VFPv3|COMPAT_HWCAP_VFPv4|\
  78                                  COMPAT_HWCAP_NEON|COMPAT_HWCAP_IDIV|\
  79                                  COMPAT_HWCAP_LPAE)
  80 unsigned int compat_elf_hwcap __read_mostly = COMPAT_ELF_HWCAP_DEFAULT;
  81 unsigned int compat_elf_hwcap2 __read_mostly;
  82 #endif
  83
  84 DECLARE_BITMAP(cpu_hwcaps, ARM64_NCAPS);
  85
  86 phys_addr_t __fdt_pointer __initdata;
  87
  88 /*
  89  * Standard memory resources
  90  */
  91 static struct resource mem_res[] = {
  92         {
  93                 .name = "Kernel code",
  94                 .start = 0,
  95                 .end = 0,
  96                 .flags = IORESOURCE_MEM
  97         },
  98         {
  99                 .name = "Kernel data",
 100                 .start = 0,
 101                 .end = 0,
 102                 .flags = IORESOURCE_MEM
 103         }
 104 };
 105
 106 #define kernel_code mem_res[0]
 107 #define kernel_data mem_res[1]
 108
 109 /*
 110  * The recorded values of x0 .. x3 upon kernel entry.
 111  */
 112 u64 __cacheline_aligned boot_args[4];
 113
 114 void __init smp_setup_processor_id(void)
 115 {
 116         u64 mpidr = read_cpuid_mpidr() & MPIDR_HWID_BITMASK;
 117         cpu_logical_map(0) = mpidr;
 118
 119         /*
 120          * clear __my_cpu_offset on boot CPU to avoid hang caused by
 121          * using percpu variable early, for example, lockdep will
 122          * access percpu variable inside lock_release
 123          */
 124         set_my_cpu_offset(0);
 125         pr_info("Booting Linux on physical CPU 0x%lx\n", (unsigned long)mpidr);
 126 }
 127
 128 bool arch_match_cpu_phys_id(int cpu, u64 phys_id)
 129 {
 130         return phys_id == cpu_logical_map(cpu);
 131 }
 132
 133 struct mpidr_hash mpidr_hash;
 134 /**
 135  * smp_build_mpidr_hash - Pre-compute shifts required at each affinity
 136  *                        level in order to build a linear index from an
 137  *                        MPIDR value. Resulting algorithm is a collision
 138  *                        free hash carried out through shifting and ORing
 139  */
 140 static void __init smp_build_mpidr_hash(void)
 141 {
 142         u32 i, affinity, fs[4], bits[4], ls;
 143         u64 mask = 0;
 144         /*
 145          * Pre-scan the list of MPIDRS and filter out bits that do
 146          * not contribute to affinity levels, ie they never toggle.
 147          */
 148         for_each_possible_cpu(i)
 149                 mask |= (cpu_logical_map(i) ^ cpu_logical_map(0));
 150         pr_debug("mask of set bits %#llx\n", mask);
 151         /*
 152          * Find and stash the last and first bit set at all affinity levels to
 153          * check how many bits are required to represent them.
 154          */
 155         for (i = 0; i < 4; i++) {
 156                 affinity = MPIDR_AFFINITY_LEVEL(mask, i);
 157                 /*
 158                  * Find the MSB bit and LSB bits position
 159                  * to determine how many bits are required
 160                  * to express the affinity level.
 161                  */
 162                 ls = fls(affinity);
 163                 fs[i] = affinity ? ffs(affinity) - 1 : 0;
 164                 bits[i] = ls - fs[i];
 165         }
 166         /*
 167          * An index can be created from the MPIDR_EL1 by isolating the
 168          * significant bits at each affinity level and by shifting
 169          * them in order to compress the 32 bits values space to a
 170          * compressed set of values. This is equivalent to hashing
 171          * the MPIDR_EL1 through shifting and ORing. It is a collision free
 172          * hash though not minimal since some levels might contain a number
 173          * of CPUs that is not an exact power of 2 and their bit
 174          * representation might contain holes, eg MPIDR_EL1[7:0] = {0x2, 0x80}.
 175          */
 176         mpidr_hash.shift_aff[0] = MPIDR_LEVEL_SHIFT(0) + fs[0];
 177         mpidr_hash.shift_aff[1] = MPIDR_LEVEL_SHIFT(1) + fs[1] - bits[0];
 178         mpidr_hash.shift_aff[2] = MPIDR_LEVEL_SHIFT(2) + fs[2] -
 179                                                 (bits[1] + bits[0]);
 180         mpidr_hash.shift_aff[3] = MPIDR_LEVEL_SHIFT(3) +
 181                                   fs[3] - (bits[2] + bits[1] + bits[0]);
 182         mpidr_hash.mask = mask;
 183         mpidr_hash.bits = bits[3] + bits[2] + bits[1] + bits[0];
 184         pr_debug("MPIDR hash: aff0[%u] aff1[%u] aff2[%u] aff3[%u] mask[%#llx] bits[%u]\n",
 185                 mpidr_hash.shift_aff[0],
 186                 mpidr_hash.shift_aff[1],
 187                 mpidr_hash.shift_aff[2],
 188                 mpidr_hash.shift_aff[3],
 189                 mpidr_hash.mask,
 190                 mpidr_hash.bits);
 191         /*
 192          * 4x is an arbitrary value used to warn on a hash table much bigger
 193          * than expected on most systems.
 194          */
 195         if (mpidr_hash_size() > 4 * num_possible_cpus())
 196                 pr_warn("Large number of MPIDR hash buckets detected\n");
 197         __flush_dcache_area(&mpidr_hash, sizeof(struct mpidr_hash));
 198 }
 199
 200 static void __init hyp_mode_check(void)
 201 {
 202         if (is_hyp_mode_available())
 203                 pr_info("CPU: All CPU(s) started at EL2\n");
 204         else if (is_hyp_mode_mismatched())
 205                 WARN_TAINT(1, TAINT_CPU_OUT_OF_SPEC,
 206                            "CPU: CPUs started in inconsistent modes");
 207         else
 208                 pr_info("CPU: All CPU(s) started at EL1\n");
 209 }
 210
 211 void __init do_post_cpus_up_work(void)
 212 {
 213         hyp_mode_check();
 214         apply_alternatives_all();
 215 }
 216
 217 #ifdef CONFIG_UP_LATE_INIT
 218 void __init up_late_init(void)
 219 {
 220         do_post_cpus_up_work();
 221 }
 222 #endif /* CONFIG_UP_LATE_INIT */
 223
 224 static void __init setup_processor(void)
 225 {
 226         u64 features, block;
 227         u32 cwg;
 228         int cls;
 229
 230         printk("CPU: AArch64 Processor [%08x] revision %d\n",
 231                read_cpuid_id(), read_cpuid_id() & 15);
 232
 233         sprintf(init_utsname()->machine, ELF_PLATFORM);
 234         elf_hwcap = 0;
 235
 236         cpuinfo_store_boot_cpu();
 237
 238         /*
 239          * Check for sane CTR_EL0.CWG value.
 240          */
 241         cwg = cache_type_cwg();
 242         cls = cache_line_size();
 243         if (!cwg)
 244                 pr_warn("No Cache Writeback Granule information, assuming cache line size %d\n",
 245                         cls);
 246         if (L1_CACHE_BYTES < cls)
 247                 pr_warn("L1_CACHE_BYTES smaller than the Cache Writeback Granule (%d < %d)\n",
 248                         L1_CACHE_BYTES, cls);
 249
 250         /*
 251          * ID_AA64ISAR0_EL1 contains 4-bit wide signed feature blocks.
 252          * The blocks we test below represent incremental functionality
 253          * for non-negative values. Negative values are reserved.
 254          */
 255         features = read_cpuid(ID_AA64ISAR0_EL1);
 256         block = (features >> 4) & 0xf;
 257         if (!(block & 0x8)) {
 258                 switch (block) {
 259                 default:
 260                 case 2:
 261                         elf_hwcap |= HWCAP_PMULL;
 262                 case 1:
 263                         elf_hwcap |= HWCAP_AES;
 264                 case 0:
 265                         break;
 266                 }
 267         }
 268
 269         block = (features >> 8) & 0xf;
 270         if (block && !(block & 0x8))
 271                 elf_hwcap |= HWCAP_SHA1;
 272
 273         block = (features >> 12) & 0xf;
 274         if (block && !(block & 0x8))
 275                 elf_hwcap |= HWCAP_SHA2;
 276
 277         block = (features >> 16) & 0xf;
 278         if (block && !(block & 0x8))
 279                 elf_hwcap |= HWCAP_CRC32;
 280
 281         block = (features >> 20) & 0xf;
 282         if (!(block & 0x8)) {
 283                 switch (block) {
 284                 default:
 285                 case 2:
 286                         elf_hwcap |= HWCAP_ATOMICS;
 287                         cpus_set_cap(ARM64_CPU_FEAT_LSE_ATOMICS);
 288                         if (IS_ENABLED(CONFIG_AS_LSE) &&
 289                             IS_ENABLED(CONFIG_ARM64_LSE_ATOMICS))
 290                                 pr_info("LSE atomics supported\n");
 291                 case 1:
 292                         /* RESERVED */
 293                 case 0:
 294                         break;
 295                 }
 296         }
 297
 298 #ifdef CONFIG_COMPAT
 299         /*
 300          * ID_ISAR5_EL1 carries similar information as above, but pertaining to
 301          * the Aarch32 32-bit execution state.
 302          */
 303         features = read_cpuid(ID_ISAR5_EL1);
 304         block = (features >> 4) & 0xf;
 305         if (!(block & 0x8)) {
 306                 switch (block) {
 307                 default:
 308                 case 2:
 309                         compat_elf_hwcap2 |= COMPAT_HWCAP2_PMULL;
 310                 case 1:
 311                         compat_elf_hwcap2 |= COMPAT_HWCAP2_AES;
 312                 case 0:
 313                         break;
 314                 }
 315         }
 316
 317         block = (features >> 8) & 0xf;
 318         if (block && !(block & 0x8))
 319                 compat_elf_hwcap2 |= COMPAT_HWCAP2_SHA1;
 320
 321         block = (features >> 12) & 0xf;
 322         if (block && !(block & 0x8))
 323                 compat_elf_hwcap2 |= COMPAT_HWCAP2_SHA2;
 324
 325         block = (features >> 16) & 0xf;
 326         if (block && !(block & 0x8))
 327                 compat_elf_hwcap2 |= COMPAT_HWCAP2_CRC32;
 328 #endif
 329 }
 330
 331 static void __init setup_machine_fdt(phys_addr_t dt_phys)
 332 {
 333         void *dt_virt = fixmap_remap_fdt(dt_phys);
 334
 335         if (!dt_virt || !early_init_dt_scan(dt_virt)) {
 336                 pr_crit("\n"
 337                         "Error: invalid device tree blob at physical address %pa (virtual address 0x%p)\n"
 338                         "The dtb must be 8-byte aligned and must not exceed 2 MB in size\n"
 339                         "\nPlease check your bootloader.",
 340                         &dt_phys, dt_virt);
 341
 342                 while (true)
 343                         cpu_relax();
 344         }
 345
 346         dump_stack_set_arch_desc("%s (DT)", of_flat_dt_get_machine_name());
 347 }
 348
 349 static void __init request_standard_resources(void)
 350 {
 351         struct memblock_region *region;
 352         struct resource *res;
 353
 354         kernel_code.start   = virt_to_phys(_text);
 355         kernel_code.end     = virt_to_phys(_etext - 1);
 356         kernel_data.start   = virt_to_phys(_sdata);
 357         kernel_data.end     = virt_to_phys(_end - 1);
 358
 359         for_each_memblock(memory, region) {
 360                 res = alloc_bootmem_low(sizeof(*res));
 361                 res->name  = "System RAM";
 362                 res->start = __pfn_to_phys(memblock_region_memory_base_pfn(region));
 363                 res->end = __pfn_to_phys(memblock_region_memory_end_pfn(region)) - 1;
 364                 res->flags = IORESOURCE_MEM | IORESOURCE_BUSY;
 365
 366                 request_resource(&iomem_resource, res);
 367
 368                 if (kernel_code.start >= res->start &&
 369                     kernel_code.end <= res->end)
 370                         request_resource(res, &kernel_code);
 371                 if (kernel_data.start >= res->start &&
 372                     kernel_data.end <= res->end)
 373                         request_resource(res, &kernel_data);
 374         }
 375 }
 376
 377 u64 __cpu_logical_map[NR_CPUS] = { [0 ... NR_CPUS-1] = INVALID_HWID };
 378
 379 void __init setup_arch(char **cmdline_p)
 380 {
 381         setup_processor();
 382
 383         init_mm.start_code = (unsigned long) _text;
 384         init_mm.end_code   = (unsigned long) _etext;
 385         init_mm.end_data   = (unsigned long) _edata;
 386         init_mm.brk        = (unsigned long) _end;
 387
 388         *cmdline_p = boot_command_line;
 389
 390         early_fixmap_init();
 391         early_ioremap_init();
 392
 393         setup_machine_fdt(__fdt_pointer);
 394
 395         parse_early_param();
 396
 397         /*
 398          *  Unmask asynchronous aborts after bringing up possible earlycon.
 399          * (Report possible System Errors once we can report this occurred)
 400          */
 401         local_async_enable();
 402
 403         efi_init();
 404         arm64_memblock_init();
 405
 406         /* Parse the ACPI tables for possible boot-time configuration */
 407         acpi_boot_table_init();
 408
 409         paging_init();
 410         request_standard_resources();
 411
 412         early_ioremap_reset();
 413
 414         if (acpi_disabled) {
 415                 unflatten_device_tree();
 416                 psci_dt_init();
 417         } else {
 418                 psci_acpi_init();
 419         }
 420         xen_early_init();
 421
 422         cpu_read_bootcpu_ops();
 423         smp_init_cpus();
 424         smp_build_mpidr_hash();
 425
 426 #ifdef CONFIG_VT
 427 #if defined(CONFIG_VGA_CONSOLE)
 428         conswitchp = &vga_con;
 429 #elif defined(CONFIG_DUMMY_CONSOLE)
 430         conswitchp = &dummy_con;
 431 #endif
 432 #endif
 433         if (boot_args[1] || boot_args[2] || boot_args[3]) {
 434                 pr_err("WARNING: x1-x3 nonzero in violation of boot protocol:\n"
 435                         "\tx1: %016llx\n\tx2: %016llx\n\tx3: %016llx\n"
 436                         "This indicates a broken bootloader or old kernel\n",
 437                         boot_args[1], boot_args[2], boot_args[3]);
 438         }
 439 }
 440
 441 static int __init arm64_device_init(void)
 442 {
 443         if (of_have_populated_dt()) {
 444                 of_iommu_init();
 445                 of_platform_populate(NULL, of_default_bus_match_table,
 446                                      NULL, NULL);
 447         } else if (acpi_disabled) {
 448                 pr_crit("Device tree not populated\n");
 449         }
 450         return 0;
 451 }
 452 arch_initcall_sync(arm64_device_init);
 453
 454 static int __init topology_init(void)
 455 {
 456         int i;
 457
 458         for_each_possible_cpu(i) {
 459                 struct cpu *cpu = &per_cpu(cpu_data.cpu, i);
 460                 cpu->hotpluggable = 1;
 461                 register_cpu(cpu, i);
 462         }
 463
 464         return 0;
 465 }
 466 subsys_initcall(topology_init);
 467
 468 static const char *hwcap_str[] = {
 469         "fp",
 470         "asimd",
 471         "evtstrm",
 472         "aes",
 473         "pmull",
 474         "sha1",
 475         "sha2",
 476         "crc32",
 477         "atomics",
 478         NULL
 479 };
 480
 481 #ifdef CONFIG_COMPAT
 482 static const char *compat_hwcap_str[] = {
 483         "swp",
 484         "half",
 485         "thumb",
 486         "26bit",
 487         "fastmult",
 488         "fpa",
 489         "vfp",
 490         "edsp",
 491         "java",
 492         "iwmmxt",
 493         "crunch",
 494         "thumbee",
 495         "neon",
 496         "vfpv3",
 497         "vfpv3d16",
 498         "tls",
 499         "vfpv4",
 500         "idiva",
 501         "idivt",
 502         "vfpd32",
 503         "lpae",
 504         "evtstrm"
 505 };
 506
 507 static const char *compat_hwcap2_str[] = {
 508         "aes",
 509         "pmull",
 510         "sha1",
 511         "sha2",
 512         "crc32",
 513         NULL
 514 };
 515 #endif /* CONFIG_COMPAT */
 516
 517 static int c_show(struct seq_file *m, void *v)
 518 {
 519         int i, j;
 520
 521         for_each_online_cpu(i) {
 522                 struct cpuinfo_arm64 *cpuinfo = &per_cpu(cpu_data, i);
 523                 u32 midr = cpuinfo->reg_midr;
 524
 525                 /*
 526                  * glibc reads /proc/cpuinfo to determine the number of
 527                  * online processors, looking for lines beginning with
 528                  * "processor".  Give glibc what it expects.
 529                  */
 530                 seq_printf(m, "processor\t: %d\n", i);
 531
 532                 /*
 533                  * Dump out the common processor features in a single line.
 534                  * Userspace should read the hwcaps with getauxval(AT_HWCAP)
 535                  * rather than attempting to parse this, but there's a body of
 536                  * software which does already (at least for 32-bit).
 537                  */
 538                 seq_puts(m, "Features\t:");
 539                 if (personality(current->personality) == PER_LINUX32) {
 540 #ifdef CONFIG_COMPAT
 541                         for (j = 0; compat_hwcap_str[j]; j++)
 542                                 if (compat_elf_hwcap & (1 << j))
 543                                         seq_printf(m, " %s", compat_hwcap_str[j]);
 544
 545                         for (j = 0; compat_hwcap2_str[j]; j++)
 546                                 if (compat_elf_hwcap2 & (1 << j))
 547                                         seq_printf(m, " %s", compat_hwcap2_str[j]);
 548 #endif /* CONFIG_COMPAT */
 549                 } else {
 550                         for (j = 0; hwcap_str[j]; j++)
 551                                 if (elf_hwcap & (1 << j))
 552                                         seq_printf(m, " %s", hwcap_str[j]);
 553                 }
 554                 seq_puts(m, "\n");
 555
 556                 seq_printf(m, "CPU implementer\t: 0x%02x\n",
 557                            MIDR_IMPLEMENTOR(midr));
 558                 seq_printf(m, "CPU architecture: 8\n");
 559                 seq_printf(m, "CPU variant\t: 0x%x\n", MIDR_VARIANT(midr));
 560                 seq_printf(m, "CPU part\t: 0x%03x\n", MIDR_PARTNUM(midr));
 561                 seq_printf(m, "CPU revision\t: %d\n\n", MIDR_REVISION(midr));
 562         }
 563
 564         return 0;
 565 }
 566
 567 static void *c_start(struct seq_file *m, loff_t *pos)
 568 {
 569         return *pos < 1 ? (void *)1 : NULL;
 570 }
 571
 572 static void *c_next(struct seq_file *m, void *v, loff_t *pos)
 573 {
 574         ++*pos;
 575         return NULL;
 576 }
 577
 578 static void c_stop(struct seq_file *m, void *v)
 579 {
 580 }
 581
 582 const struct seq_operations cpuinfo_op = {
 583         .start  = c_start,
 584         .next   = c_next,
 585         .stop   = c_stop,
 586         .show   = c_show
 587 };