Commit | Line | Data |
---|---|---|
279b991b MR |
1 | /* |
2 | * This file is subject to the terms and conditions of the GNU General Public | |
3 | * License. See the file "COPYING" in the main directory of this archive | |
4 | * for more details. | |
5 | * | |
6 | * Support for Kernel relocation at boot time | |
7 | * | |
8 | * Copyright (C) 2015, Imagination Technologies Ltd. | |
9 | * Authors: Matt Redfearn (matt.redfearn@imgtec.com) | |
10 | */ | |
405bc8fd | 11 | #include <asm/bootinfo.h> |
279b991b | 12 | #include <asm/cacheflush.h> |
405bc8fd | 13 | #include <asm/fw/fw.h> |
279b991b MR |
14 | #include <asm/sections.h> |
15 | #include <asm/setup.h> | |
16 | #include <asm/timex.h> | |
17 | #include <linux/elf.h> | |
18 | #include <linux/kernel.h> | |
405bc8fd MR |
19 | #include <linux/libfdt.h> |
20 | #include <linux/of_fdt.h> | |
279b991b MR |
21 | #include <linux/sched.h> |
22 | #include <linux/start_kernel.h> | |
23 | #include <linux/string.h> | |
405bc8fd | 24 | #include <linux/printk.h> |
279b991b MR |
25 | |
26 | #define RELOCATED(x) ((void *)((long)x + offset)) | |
27 | ||
28 | extern u32 _relocation_start[]; /* End kernel image / start relocation table */ | |
29 | extern u32 _relocation_end[]; /* End relocation table */ | |
30 | ||
31 | extern long __start___ex_table; /* Start exception table */ | |
32 | extern long __stop___ex_table; /* End exception table */ | |
33 | ||
34 | static inline u32 __init get_synci_step(void) | |
35 | { | |
36 | u32 res; | |
37 | ||
38 | __asm__("rdhwr %0, $1" : "=r" (res)); | |
39 | ||
40 | return res; | |
41 | } | |
42 | ||
43 | static void __init sync_icache(void *kbase, unsigned long kernel_length) | |
44 | { | |
45 | void *kend = kbase + kernel_length; | |
46 | u32 step = get_synci_step(); | |
47 | ||
48 | do { | |
49 | __asm__ __volatile__( | |
50 | "synci 0(%0)" | |
51 | : /* no output */ | |
52 | : "r" (kbase)); | |
53 | ||
54 | kbase += step; | |
55 | } while (kbase < kend); | |
56 | ||
57 | /* Completion barrier */ | |
58 | __sync(); | |
59 | } | |
60 | ||
61 | static int __init apply_r_mips_64_rel(u32 *loc_orig, u32 *loc_new, long offset) | |
62 | { | |
63 | *(u64 *)loc_new += offset; | |
64 | ||
65 | return 0; | |
66 | } | |
67 | ||
68 | static int __init apply_r_mips_32_rel(u32 *loc_orig, u32 *loc_new, long offset) | |
69 | { | |
70 | *loc_new += offset; | |
71 | ||
72 | return 0; | |
73 | } | |
74 | ||
75 | static int __init apply_r_mips_26_rel(u32 *loc_orig, u32 *loc_new, long offset) | |
76 | { | |
77 | unsigned long target_addr = (*loc_orig) & 0x03ffffff; | |
78 | ||
79 | if (offset % 4) { | |
80 | pr_err("Dangerous R_MIPS_26 REL relocation\n"); | |
81 | return -ENOEXEC; | |
82 | } | |
83 | ||
84 | /* Original target address */ | |
85 | target_addr <<= 2; | |
86 | target_addr += (unsigned long)loc_orig & ~0x03ffffff; | |
87 | ||
88 | /* Get the new target address */ | |
89 | target_addr += offset; | |
90 | ||
91 | if ((target_addr & 0xf0000000) != ((unsigned long)loc_new & 0xf0000000)) { | |
92 | pr_err("R_MIPS_26 REL relocation overflow\n"); | |
93 | return -ENOEXEC; | |
94 | } | |
95 | ||
96 | target_addr -= (unsigned long)loc_new & ~0x03ffffff; | |
97 | target_addr >>= 2; | |
98 | ||
99 | *loc_new = (*loc_new & ~0x03ffffff) | (target_addr & 0x03ffffff); | |
100 | ||
101 | return 0; | |
102 | } | |
103 | ||
104 | ||
105 | static int __init apply_r_mips_hi16_rel(u32 *loc_orig, u32 *loc_new, long offset) | |
106 | { | |
107 | unsigned long insn = *loc_orig; | |
108 | unsigned long target = (insn & 0xffff) << 16; /* high 16bits of target */ | |
109 | ||
110 | target += offset; | |
111 | ||
112 | *loc_new = (insn & ~0xffff) | ((target >> 16) & 0xffff); | |
113 | return 0; | |
114 | } | |
115 | ||
116 | static int (*reloc_handlers_rel[]) (u32 *, u32 *, long) __initdata = { | |
117 | [R_MIPS_64] = apply_r_mips_64_rel, | |
118 | [R_MIPS_32] = apply_r_mips_32_rel, | |
119 | [R_MIPS_26] = apply_r_mips_26_rel, | |
120 | [R_MIPS_HI16] = apply_r_mips_hi16_rel, | |
121 | }; | |
122 | ||
123 | int __init do_relocations(void *kbase_old, void *kbase_new, long offset) | |
124 | { | |
125 | u32 *r; | |
126 | u32 *loc_orig; | |
127 | u32 *loc_new; | |
128 | int type; | |
129 | int res; | |
130 | ||
131 | for (r = _relocation_start; r < _relocation_end; r++) { | |
132 | /* Sentinel for last relocation */ | |
133 | if (*r == 0) | |
134 | break; | |
135 | ||
136 | type = (*r >> 24) & 0xff; | |
137 | loc_orig = (void *)(kbase_old + ((*r & 0x00ffffff) << 2)); | |
138 | loc_new = RELOCATED(loc_orig); | |
139 | ||
140 | if (reloc_handlers_rel[type] == NULL) { | |
141 | /* Unsupported relocation */ | |
142 | pr_err("Unhandled relocation type %d at 0x%pK\n", | |
143 | type, loc_orig); | |
144 | return -ENOEXEC; | |
145 | } | |
146 | ||
147 | res = reloc_handlers_rel[type](loc_orig, loc_new, offset); | |
148 | if (res) | |
149 | return res; | |
150 | } | |
151 | ||
152 | return 0; | |
153 | } | |
154 | ||
155 | /* | |
156 | * The exception table is filled in by the relocs tool after vmlinux is linked. | |
157 | * It must be relocated separately since there will not be any relocation | |
158 | * information for it filled in by the linker. | |
159 | */ | |
160 | static int __init relocate_exception_table(long offset) | |
161 | { | |
162 | unsigned long *etable_start, *etable_end, *e; | |
163 | ||
164 | etable_start = RELOCATED(&__start___ex_table); | |
165 | etable_end = RELOCATED(&__stop___ex_table); | |
166 | ||
167 | for (e = etable_start; e < etable_end; e++) | |
168 | *e += offset; | |
169 | ||
170 | return 0; | |
171 | } | |
172 | ||
405bc8fd MR |
173 | #ifdef CONFIG_RANDOMIZE_BASE |
174 | ||
175 | static inline __init unsigned long rotate_xor(unsigned long hash, | |
176 | const void *area, size_t size) | |
177 | { | |
178 | size_t i; | |
179 | unsigned long *ptr = (unsigned long *)area; | |
180 | ||
181 | for (i = 0; i < size / sizeof(hash); i++) { | |
182 | /* Rotate by odd number of bits and XOR. */ | |
183 | hash = (hash << ((sizeof(hash) * 8) - 7)) | (hash >> 7); | |
184 | hash ^= ptr[i]; | |
185 | } | |
186 | ||
187 | return hash; | |
188 | } | |
189 | ||
190 | static inline __init unsigned long get_random_boot(void) | |
191 | { | |
192 | unsigned long entropy = random_get_entropy(); | |
193 | unsigned long hash = 0; | |
194 | ||
195 | /* Attempt to create a simple but unpredictable starting entropy. */ | |
196 | hash = rotate_xor(hash, linux_banner, strlen(linux_banner)); | |
197 | ||
198 | /* Add in any runtime entropy we can get */ | |
199 | hash = rotate_xor(hash, &entropy, sizeof(entropy)); | |
200 | ||
201 | #if defined(CONFIG_USE_OF) | |
202 | /* Get any additional entropy passed in device tree */ | |
203 | { | |
204 | int node, len; | |
205 | u64 *prop; | |
206 | ||
207 | node = fdt_path_offset(initial_boot_params, "/chosen"); | |
208 | if (node >= 0) { | |
209 | prop = fdt_getprop_w(initial_boot_params, node, | |
210 | "kaslr-seed", &len); | |
211 | if (prop && (len == sizeof(u64))) | |
212 | hash = rotate_xor(hash, prop, sizeof(*prop)); | |
213 | } | |
214 | } | |
215 | #endif /* CONFIG_USE_OF */ | |
216 | ||
217 | return hash; | |
218 | } | |
219 | ||
220 | static inline __init bool kaslr_disabled(void) | |
221 | { | |
222 | char *str; | |
223 | ||
224 | #if defined(CONFIG_CMDLINE_BOOL) | |
225 | const char *builtin_cmdline = CONFIG_CMDLINE; | |
226 | ||
227 | str = strstr(builtin_cmdline, "nokaslr"); | |
228 | if (str == builtin_cmdline || | |
229 | (str > builtin_cmdline && *(str - 1) == ' ')) | |
230 | return true; | |
231 | #endif | |
232 | str = strstr(arcs_cmdline, "nokaslr"); | |
233 | if (str == arcs_cmdline || (str > arcs_cmdline && *(str - 1) == ' ')) | |
234 | return true; | |
235 | ||
236 | return false; | |
237 | } | |
238 | ||
239 | static inline void __init *determine_relocation_address(void) | |
240 | { | |
241 | /* Choose a new address for the kernel */ | |
242 | unsigned long kernel_length; | |
243 | void *dest = &_text; | |
244 | unsigned long offset; | |
245 | ||
246 | if (kaslr_disabled()) | |
247 | return dest; | |
248 | ||
249 | kernel_length = (long)_end - (long)(&_text); | |
250 | ||
251 | offset = get_random_boot() << 16; | |
252 | offset &= (CONFIG_RANDOMIZE_BASE_MAX_OFFSET - 1); | |
253 | if (offset < kernel_length) | |
254 | offset += ALIGN(kernel_length, 0xffff); | |
255 | ||
256 | return RELOCATED(dest); | |
257 | } | |
258 | ||
259 | #else | |
260 | ||
279b991b MR |
261 | static inline void __init *determine_relocation_address(void) |
262 | { | |
263 | /* | |
264 | * Choose a new address for the kernel | |
265 | * For now we'll hard code the destination | |
266 | */ | |
267 | return (void *)0xffffffff81000000; | |
268 | } | |
269 | ||
405bc8fd MR |
270 | #endif |
271 | ||
279b991b MR |
272 | static inline int __init relocation_addr_valid(void *loc_new) |
273 | { | |
274 | if ((unsigned long)loc_new & 0x0000ffff) { | |
275 | /* Inappropriately aligned new location */ | |
276 | return 0; | |
277 | } | |
278 | if ((unsigned long)loc_new < (unsigned long)&_end) { | |
279 | /* New location overlaps original kernel */ | |
280 | return 0; | |
281 | } | |
282 | return 1; | |
283 | } | |
284 | ||
285 | void *__init relocate_kernel(void) | |
286 | { | |
287 | void *loc_new; | |
288 | unsigned long kernel_length; | |
289 | unsigned long bss_length; | |
290 | long offset = 0; | |
291 | int res = 1; | |
292 | /* Default to original kernel entry point */ | |
293 | void *kernel_entry = start_kernel; | |
294 | ||
405bc8fd MR |
295 | /* Get the command line */ |
296 | fw_init_cmdline(); | |
297 | #if defined(CONFIG_USE_OF) | |
298 | /* Deal with the device tree */ | |
299 | early_init_dt_scan(plat_get_fdt()); | |
300 | if (boot_command_line[0]) { | |
301 | /* Boot command line was passed in device tree */ | |
302 | strlcpy(arcs_cmdline, boot_command_line, COMMAND_LINE_SIZE); | |
303 | } | |
304 | #endif /* CONFIG_USE_OF */ | |
305 | ||
279b991b MR |
306 | kernel_length = (long)(&_relocation_start) - (long)(&_text); |
307 | bss_length = (long)&__bss_stop - (long)&__bss_start; | |
308 | ||
309 | loc_new = determine_relocation_address(); | |
310 | ||
311 | /* Sanity check relocation address */ | |
312 | if (relocation_addr_valid(loc_new)) | |
313 | offset = (unsigned long)loc_new - (unsigned long)(&_text); | |
314 | ||
405bc8fd MR |
315 | /* Reset the command line now so we don't end up with a duplicate */ |
316 | arcs_cmdline[0] = '\0'; | |
317 | ||
279b991b MR |
318 | if (offset) { |
319 | /* Copy the kernel to it's new location */ | |
320 | memcpy(loc_new, &_text, kernel_length); | |
321 | ||
322 | /* Perform relocations on the new kernel */ | |
323 | res = do_relocations(&_text, loc_new, offset); | |
324 | if (res < 0) | |
325 | goto out; | |
326 | ||
327 | /* Sync the caches ready for execution of new kernel */ | |
328 | sync_icache(loc_new, kernel_length); | |
329 | ||
330 | res = relocate_exception_table(offset); | |
331 | if (res < 0) | |
332 | goto out; | |
333 | ||
334 | /* | |
335 | * The original .bss has already been cleared, and | |
336 | * some variables such as command line parameters | |
337 | * stored to it so make a copy in the new location. | |
338 | */ | |
339 | memcpy(RELOCATED(&__bss_start), &__bss_start, bss_length); | |
340 | ||
341 | /* The current thread is now within the relocated image */ | |
342 | __current_thread_info = RELOCATED(&init_thread_union); | |
343 | ||
344 | /* Return the new kernel's entry point */ | |
345 | kernel_entry = RELOCATED(start_kernel); | |
346 | } | |
347 | out: | |
348 | return kernel_entry; | |
349 | } | |
405bc8fd MR |
350 | |
351 | /* | |
352 | * Show relocation information on panic. | |
353 | */ | |
354 | void show_kernel_relocation(const char *level) | |
355 | { | |
356 | unsigned long offset; | |
357 | ||
358 | offset = __pa_symbol(_text) - __pa_symbol(VMLINUX_LOAD_ADDRESS); | |
359 | ||
360 | if (IS_ENABLED(CONFIG_RELOCATABLE) && offset > 0) { | |
361 | printk(level); | |
362 | pr_cont("Kernel relocated by 0x%pK\n", (void *)offset); | |
363 | pr_cont(" .text @ 0x%pK\n", _text); | |
364 | pr_cont(" .data @ 0x%pK\n", _sdata); | |
365 | pr_cont(" .bss @ 0x%pK\n", __bss_start); | |
366 | } | |
367 | } | |
368 | ||
369 | static int kernel_location_notifier_fn(struct notifier_block *self, | |
370 | unsigned long v, void *p) | |
371 | { | |
372 | show_kernel_relocation(KERN_EMERG); | |
373 | return NOTIFY_DONE; | |
374 | } | |
375 | ||
376 | static struct notifier_block kernel_location_notifier = { | |
377 | .notifier_call = kernel_location_notifier_fn | |
378 | }; | |
379 | ||
380 | static int __init register_kernel_offset_dumper(void) | |
381 | { | |
382 | atomic_notifier_chain_register(&panic_notifier_list, | |
383 | &kernel_location_notifier); | |
384 | return 0; | |
385 | } | |
386 | __initcall(register_kernel_offset_dumper); |