2 * handle transition of Linux booting another kernel
3 * Copyright (C) 2002-2005 Eric Biederman <ebiederm@xmission.com>
5 * This source code is licensed under the GNU General Public License,
6 * Version 2. See the file COPYING for more details.
9 #define pr_fmt(fmt) "kexec: " fmt
12 #include <linux/kexec.h>
13 #include <linux/string.h>
14 #include <linux/gfp.h>
15 #include <linux/reboot.h>
16 #include <linux/numa.h>
17 #include <linux/ftrace.h>
19 #include <linux/suspend.h>
22 #include <asm/pgtable.h>
23 #include <asm/tlbflush.h>
24 #include <asm/mmu_context.h>
25 #include <asm/debugreg.h>
26 #include <asm/kexec-bzimage64.h>
28 #ifdef CONFIG_KEXEC_FILE
29 static struct kexec_file_ops
*kexec_file_loaders
[] = {
34 static void free_transition_pgtable(struct kimage
*image
)
36 free_page((unsigned long)image
->arch
.pud
);
37 free_page((unsigned long)image
->arch
.pmd
);
38 free_page((unsigned long)image
->arch
.pte
);
41 static int init_transition_pgtable(struct kimage
*image
, pgd_t
*pgd
)
46 unsigned long vaddr
, paddr
;
49 vaddr
= (unsigned long)relocate_kernel
;
50 paddr
= __pa(page_address(image
->control_code_page
)+PAGE_SIZE
);
51 pgd
+= pgd_index(vaddr
);
52 if (!pgd_present(*pgd
)) {
53 pud
= (pud_t
*)get_zeroed_page(GFP_KERNEL
);
56 image
->arch
.pud
= pud
;
57 set_pgd(pgd
, __pgd(__pa(pud
) | _KERNPG_TABLE
));
59 pud
= pud_offset(pgd
, vaddr
);
60 if (!pud_present(*pud
)) {
61 pmd
= (pmd_t
*)get_zeroed_page(GFP_KERNEL
);
64 image
->arch
.pmd
= pmd
;
65 set_pud(pud
, __pud(__pa(pmd
) | _KERNPG_TABLE
));
67 pmd
= pmd_offset(pud
, vaddr
);
68 if (!pmd_present(*pmd
)) {
69 pte
= (pte_t
*)get_zeroed_page(GFP_KERNEL
);
72 image
->arch
.pte
= pte
;
73 set_pmd(pmd
, __pmd(__pa(pte
) | _KERNPG_TABLE
));
75 pte
= pte_offset_kernel(pmd
, vaddr
);
76 set_pte(pte
, pfn_pte(paddr
>> PAGE_SHIFT
, PAGE_KERNEL_EXEC
));
79 free_transition_pgtable(image
);
83 static void *alloc_pgt_page(void *data
)
85 struct kimage
*image
= (struct kimage
*)data
;
89 page
= kimage_alloc_control_pages(image
, 0);
91 p
= page_address(page
);
98 static int init_pgtable(struct kimage
*image
, unsigned long start_pgtable
)
100 struct x86_mapping_info info
= {
101 .alloc_pgt_page
= alloc_pgt_page
,
103 .pmd_flag
= __PAGE_KERNEL_LARGE_EXEC
,
105 unsigned long mstart
, mend
;
110 level4p
= (pgd_t
*)__va(start_pgtable
);
112 for (i
= 0; i
< nr_pfn_mapped
; i
++) {
113 mstart
= pfn_mapped
[i
].start
<< PAGE_SHIFT
;
114 mend
= pfn_mapped
[i
].end
<< PAGE_SHIFT
;
116 result
= kernel_ident_mapping_init(&info
,
117 level4p
, mstart
, mend
);
123 * segments's mem ranges could be outside 0 ~ max_pfn,
124 * for example when jump back to original kernel from kexeced kernel.
125 * or first kernel is booted with user mem map, and second kernel
126 * could be loaded out of that range.
128 for (i
= 0; i
< image
->nr_segments
; i
++) {
129 mstart
= image
->segment
[i
].mem
;
130 mend
= mstart
+ image
->segment
[i
].memsz
;
132 result
= kernel_ident_mapping_init(&info
,
133 level4p
, mstart
, mend
);
139 return init_transition_pgtable(image
, level4p
);
142 static void set_idt(void *newidt
, u16 limit
)
144 struct desc_ptr curidt
;
146 /* x86-64 supports unaliged loads & stores */
148 curidt
.address
= (unsigned long)newidt
;
150 __asm__
__volatile__ (
157 static void set_gdt(void *newgdt
, u16 limit
)
159 struct desc_ptr curgdt
;
161 /* x86-64 supports unaligned loads & stores */
163 curgdt
.address
= (unsigned long)newgdt
;
165 __asm__
__volatile__ (
171 static void load_segments(void)
173 __asm__
__volatile__ (
179 : : "a" (__KERNEL_DS
) : "memory"
183 #ifdef CONFIG_KEXEC_FILE
184 /* Update purgatory as needed after various image segments have been prepared */
185 static int arch_update_purgatory(struct kimage
*image
)
189 if (!image
->file_mode
)
192 /* Setup copying of backup region */
193 if (image
->type
== KEXEC_TYPE_CRASH
) {
194 ret
= kexec_purgatory_get_set_symbol(image
, "backup_dest",
195 &image
->arch
.backup_load_addr
,
196 sizeof(image
->arch
.backup_load_addr
), 0);
200 ret
= kexec_purgatory_get_set_symbol(image
, "backup_src",
201 &image
->arch
.backup_src_start
,
202 sizeof(image
->arch
.backup_src_start
), 0);
206 ret
= kexec_purgatory_get_set_symbol(image
, "backup_sz",
207 &image
->arch
.backup_src_sz
,
208 sizeof(image
->arch
.backup_src_sz
), 0);
215 #else /* !CONFIG_KEXEC_FILE */
216 static inline int arch_update_purgatory(struct kimage
*image
)
220 #endif /* CONFIG_KEXEC_FILE */
222 int machine_kexec_prepare(struct kimage
*image
)
224 unsigned long start_pgtable
;
227 /* Calculate the offsets */
228 start_pgtable
= page_to_pfn(image
->control_code_page
) << PAGE_SHIFT
;
230 /* Setup the identity mapped 64bit page table */
231 result
= init_pgtable(image
, start_pgtable
);
235 /* update purgatory as needed */
236 result
= arch_update_purgatory(image
);
243 void machine_kexec_cleanup(struct kimage
*image
)
245 free_transition_pgtable(image
);
249 * Do not allocate memory (or fail in any way) in machine_kexec().
250 * We are past the point of no return, committed to rebooting now.
252 void machine_kexec(struct kimage
*image
)
254 unsigned long page_list
[PAGES_NR
];
256 int save_ftrace_enabled
;
258 #ifdef CONFIG_KEXEC_JUMP
259 if (image
->preserve_context
)
260 save_processor_state();
263 save_ftrace_enabled
= __ftrace_enabled_save();
265 /* Interrupts aren't acceptable while we reboot */
267 hw_breakpoint_disable();
269 if (image
->preserve_context
) {
270 #ifdef CONFIG_X86_IO_APIC
272 * We need to put APICs in legacy mode so that we can
273 * get timer interrupts in second kernel. kexec/kdump
274 * paths already have calls to disable_IO_APIC() in
275 * one form or other. kexec jump path also need
282 control_page
= page_address(image
->control_code_page
) + PAGE_SIZE
;
283 memcpy(control_page
, relocate_kernel
, KEXEC_CONTROL_CODE_MAX_SIZE
);
285 page_list
[PA_CONTROL_PAGE
] = virt_to_phys(control_page
);
286 page_list
[VA_CONTROL_PAGE
] = (unsigned long)control_page
;
287 page_list
[PA_TABLE_PAGE
] =
288 (unsigned long)__pa(page_address(image
->control_code_page
));
290 if (image
->type
== KEXEC_TYPE_DEFAULT
)
291 page_list
[PA_SWAP_PAGE
] = (page_to_pfn(image
->swap_page
)
295 * The segment registers are funny things, they have both a
296 * visible and an invisible part. Whenever the visible part is
297 * set to a specific selector, the invisible part is loaded
298 * with from a table in memory. At no other time is the
299 * descriptor table in memory accessed.
301 * I take advantage of this here by force loading the
302 * segments, before I zap the gdt with an invalid value.
306 * The gdt & idt are now invalid.
307 * If you want to load them you must set up your own idt & gdt.
309 set_gdt(phys_to_virt(0), 0);
310 set_idt(phys_to_virt(0), 0);
313 image
->start
= relocate_kernel((unsigned long)image
->head
,
314 (unsigned long)page_list
,
316 image
->preserve_context
);
318 #ifdef CONFIG_KEXEC_JUMP
319 if (image
->preserve_context
)
320 restore_processor_state();
323 __ftrace_enabled_restore(save_ftrace_enabled
);
326 void arch_crash_save_vmcoreinfo(void)
328 VMCOREINFO_SYMBOL(phys_base
);
329 VMCOREINFO_SYMBOL(init_level4_pgt
);
332 VMCOREINFO_SYMBOL(node_data
);
333 VMCOREINFO_LENGTH(node_data
, MAX_NUMNODES
);
335 vmcoreinfo_append_str("KERNELOFFSET=%lx\n",
336 (unsigned long)&_text
- __START_KERNEL
);
339 /* arch-dependent functionality related to kexec file-based syscall */
341 #ifdef CONFIG_KEXEC_FILE
342 int arch_kexec_kernel_image_probe(struct kimage
*image
, void *buf
,
343 unsigned long buf_len
)
345 int i
, ret
= -ENOEXEC
;
346 struct kexec_file_ops
*fops
;
348 for (i
= 0; i
< ARRAY_SIZE(kexec_file_loaders
); i
++) {
349 fops
= kexec_file_loaders
[i
];
350 if (!fops
|| !fops
->probe
)
353 ret
= fops
->probe(buf
, buf_len
);
363 void *arch_kexec_kernel_image_load(struct kimage
*image
)
365 vfree(image
->arch
.elf_headers
);
366 image
->arch
.elf_headers
= NULL
;
368 if (!image
->fops
|| !image
->fops
->load
)
369 return ERR_PTR(-ENOEXEC
);
371 return image
->fops
->load(image
, image
->kernel_buf
,
372 image
->kernel_buf_len
, image
->initrd_buf
,
373 image
->initrd_buf_len
, image
->cmdline_buf
,
374 image
->cmdline_buf_len
);
377 int arch_kimage_file_post_load_cleanup(struct kimage
*image
)
379 if (!image
->fops
|| !image
->fops
->cleanup
)
382 return image
->fops
->cleanup(image
->image_loader_data
);
385 int arch_kexec_kernel_verify_sig(struct kimage
*image
, void *kernel
,
386 unsigned long kernel_len
)
388 if (!image
->fops
|| !image
->fops
->verify_sig
) {
389 pr_debug("kernel loader does not support signature verification.");
390 return -EKEYREJECTED
;
393 return image
->fops
->verify_sig(kernel
, kernel_len
);
397 * Apply purgatory relocations.
399 * ehdr: Pointer to elf headers
400 * sechdrs: Pointer to section headers.
401 * relsec: section index of SHT_RELA section.
403 * TODO: Some of the code belongs to generic code. Move that in kexec.c.
405 int arch_kexec_apply_relocations_add(const Elf64_Ehdr
*ehdr
,
406 Elf64_Shdr
*sechdrs
, unsigned int relsec
)
412 Elf64_Shdr
*section
, *symtabsec
;
413 unsigned long address
, sec_base
, value
;
414 const char *strtab
, *name
, *shstrtab
;
417 * ->sh_offset has been modified to keep the pointer to section
420 rel
= (void *)sechdrs
[relsec
].sh_offset
;
422 /* Section to which relocations apply */
423 section
= &sechdrs
[sechdrs
[relsec
].sh_info
];
425 pr_debug("Applying relocate section %u to %u\n", relsec
,
426 sechdrs
[relsec
].sh_info
);
428 /* Associated symbol table */
429 symtabsec
= &sechdrs
[sechdrs
[relsec
].sh_link
];
432 if (symtabsec
->sh_link
>= ehdr
->e_shnum
) {
433 /* Invalid strtab section number */
434 pr_err("Invalid string table section index %d\n",
439 strtab
= (char *)sechdrs
[symtabsec
->sh_link
].sh_offset
;
441 /* section header string table */
442 shstrtab
= (char *)sechdrs
[ehdr
->e_shstrndx
].sh_offset
;
444 for (i
= 0; i
< sechdrs
[relsec
].sh_size
/ sizeof(*rel
); i
++) {
447 * rel[i].r_offset contains byte offset from beginning
448 * of section to the storage unit affected.
450 * This is location to update (->sh_offset). This is temporary
451 * buffer where section is currently loaded. This will finally
452 * be loaded to a different address later, pointed to by
453 * ->sh_addr. kexec takes care of moving it
454 * (kexec_load_segment()).
456 location
= (void *)(section
->sh_offset
+ rel
[i
].r_offset
);
458 /* Final address of the location */
459 address
= section
->sh_addr
+ rel
[i
].r_offset
;
462 * rel[i].r_info contains information about symbol table index
463 * w.r.t which relocation must be made and type of relocation
464 * to apply. ELF64_R_SYM() and ELF64_R_TYPE() macros get
465 * these respectively.
467 sym
= (Elf64_Sym
*)symtabsec
->sh_offset
+
468 ELF64_R_SYM(rel
[i
].r_info
);
471 name
= strtab
+ sym
->st_name
;
473 name
= shstrtab
+ sechdrs
[sym
->st_shndx
].sh_name
;
475 pr_debug("Symbol: %s info: %02x shndx: %02x value=%llx size: %llx\n",
476 name
, sym
->st_info
, sym
->st_shndx
, sym
->st_value
,
479 if (sym
->st_shndx
== SHN_UNDEF
) {
480 pr_err("Undefined symbol: %s\n", name
);
484 if (sym
->st_shndx
== SHN_COMMON
) {
485 pr_err("symbol '%s' in common section\n", name
);
489 if (sym
->st_shndx
== SHN_ABS
)
491 else if (sym
->st_shndx
>= ehdr
->e_shnum
) {
492 pr_err("Invalid section %d for symbol %s\n",
493 sym
->st_shndx
, name
);
496 sec_base
= sechdrs
[sym
->st_shndx
].sh_addr
;
498 value
= sym
->st_value
;
500 value
+= rel
[i
].r_addend
;
502 switch (ELF64_R_TYPE(rel
[i
].r_info
)) {
506 *(u64
*)location
= value
;
509 *(u32
*)location
= value
;
510 if (value
!= *(u32
*)location
)
514 *(s32
*)location
= value
;
515 if ((s64
)value
!= *(s32
*)location
)
519 value
-= (u64
)address
;
520 *(u32
*)location
= value
;
523 pr_err("Unknown rela relocation: %llu\n",
524 ELF64_R_TYPE(rel
[i
].r_info
));
531 pr_err("Overflow in relocation type %d value 0x%lx\n",
532 (int)ELF64_R_TYPE(rel
[i
].r_info
), value
);
535 #endif /* CONFIG_KEXEC_FILE */