Commit | Line | Data |
---|---|---|
53631b54 CM |
1 | /* |
2 | * FP/SIMD context switching and fault handling | |
3 | * | |
4 | * Copyright (C) 2012 ARM Ltd. | |
5 | * Author: Catalin Marinas <catalin.marinas@arm.com> | |
6 | * | |
7 | * This program is free software; you can redistribute it and/or modify | |
8 | * it under the terms of the GNU General Public License version 2 as | |
9 | * published by the Free Software Foundation. | |
10 | * | |
11 | * This program is distributed in the hope that it will be useful, | |
12 | * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
13 | * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
14 | * GNU General Public License for more details. | |
15 | * | |
16 | * You should have received a copy of the GNU General Public License | |
17 | * along with this program. If not, see <http://www.gnu.org/licenses/>. | |
18 | */ | |
19 | ||
32365e64 | 20 | #include <linux/cpu.h> |
fb1ab1ab | 21 | #include <linux/cpu_pm.h> |
53631b54 CM |
22 | #include <linux/kernel.h> |
23 | #include <linux/init.h> | |
24 | #include <linux/sched.h> | |
25 | #include <linux/signal.h> | |
4cfb3613 | 26 | #include <linux/hardirq.h> |
53631b54 CM |
27 | |
28 | #include <asm/fpsimd.h> | |
29 | #include <asm/cputype.h> | |
30 | ||
31 | #define FPEXC_IOF (1 << 0) | |
32 | #define FPEXC_DZF (1 << 1) | |
33 | #define FPEXC_OFF (1 << 2) | |
34 | #define FPEXC_UFF (1 << 3) | |
35 | #define FPEXC_IXF (1 << 4) | |
36 | #define FPEXC_IDF (1 << 7) | |
37 | ||
005f78cd AB |
38 | /* |
39 | * In order to reduce the number of times the FPSIMD state is needlessly saved | |
40 | * and restored, we need to keep track of two things: | |
41 | * (a) for each task, we need to remember which CPU was the last one to have | |
42 | * the task's FPSIMD state loaded into its FPSIMD registers; | |
43 | * (b) for each CPU, we need to remember which task's userland FPSIMD state has | |
44 | * been loaded into its FPSIMD registers most recently, or whether it has | |
45 | * been used to perform kernel mode NEON in the meantime. | |
46 | * | |
47 | * For (a), we add a 'cpu' field to struct fpsimd_state, which gets updated to | |
48 | * the id of the current CPU everytime the state is loaded onto a CPU. For (b), | |
49 | * we add the per-cpu variable 'fpsimd_last_state' (below), which contains the | |
50 | * address of the userland FPSIMD state of the task that was loaded onto the CPU | |
51 | * the most recently, or NULL if kernel mode NEON has been performed after that. | |
52 | * | |
53 | * With this in place, we no longer have to restore the next FPSIMD state right | |
54 | * when switching between tasks. Instead, we can defer this check to userland | |
55 | * resume, at which time we verify whether the CPU's fpsimd_last_state and the | |
56 | * task's fpsimd_state.cpu are still mutually in sync. If this is the case, we | |
57 | * can omit the FPSIMD restore. | |
58 | * | |
59 | * As an optimization, we use the thread_info flag TIF_FOREIGN_FPSTATE to | |
60 | * indicate whether or not the userland FPSIMD state of the current task is | |
61 | * present in the registers. The flag is set unless the FPSIMD registers of this | |
62 | * CPU currently contain the most recent userland FPSIMD state of the current | |
63 | * task. | |
64 | * | |
65 | * For a certain task, the sequence may look something like this: | |
66 | * - the task gets scheduled in; if both the task's fpsimd_state.cpu field | |
67 | * contains the id of the current CPU, and the CPU's fpsimd_last_state per-cpu | |
68 | * variable points to the task's fpsimd_state, the TIF_FOREIGN_FPSTATE flag is | |
69 | * cleared, otherwise it is set; | |
70 | * | |
71 | * - the task returns to userland; if TIF_FOREIGN_FPSTATE is set, the task's | |
72 | * userland FPSIMD state is copied from memory to the registers, the task's | |
73 | * fpsimd_state.cpu field is set to the id of the current CPU, the current | |
74 | * CPU's fpsimd_last_state pointer is set to this task's fpsimd_state and the | |
75 | * TIF_FOREIGN_FPSTATE flag is cleared; | |
76 | * | |
77 | * - the task executes an ordinary syscall; upon return to userland, the | |
78 | * TIF_FOREIGN_FPSTATE flag will still be cleared, so no FPSIMD state is | |
79 | * restored; | |
80 | * | |
81 | * - the task executes a syscall which executes some NEON instructions; this is | |
82 | * preceded by a call to kernel_neon_begin(), which copies the task's FPSIMD | |
83 | * register contents to memory, clears the fpsimd_last_state per-cpu variable | |
84 | * and sets the TIF_FOREIGN_FPSTATE flag; | |
85 | * | |
86 | * - the task gets preempted after kernel_neon_end() is called; as we have not | |
87 | * returned from the 2nd syscall yet, TIF_FOREIGN_FPSTATE is still set so | |
88 | * whatever is in the FPSIMD registers is not saved to memory, but discarded. | |
89 | */ | |
90 | static DEFINE_PER_CPU(struct fpsimd_state *, fpsimd_last_state); | |
91 | ||
53631b54 CM |
92 | /* |
93 | * Trapped FP/ASIMD access. | |
94 | */ | |
95 | void do_fpsimd_acc(unsigned int esr, struct pt_regs *regs) | |
96 | { | |
97 | /* TODO: implement lazy context saving/restoring */ | |
98 | WARN_ON(1); | |
99 | } | |
100 | ||
101 | /* | |
102 | * Raise a SIGFPE for the current process. | |
103 | */ | |
104 | void do_fpsimd_exc(unsigned int esr, struct pt_regs *regs) | |
105 | { | |
106 | siginfo_t info; | |
107 | unsigned int si_code = 0; | |
108 | ||
109 | if (esr & FPEXC_IOF) | |
110 | si_code = FPE_FLTINV; | |
111 | else if (esr & FPEXC_DZF) | |
112 | si_code = FPE_FLTDIV; | |
113 | else if (esr & FPEXC_OFF) | |
114 | si_code = FPE_FLTOVF; | |
115 | else if (esr & FPEXC_UFF) | |
116 | si_code = FPE_FLTUND; | |
117 | else if (esr & FPEXC_IXF) | |
118 | si_code = FPE_FLTRES; | |
119 | ||
120 | memset(&info, 0, sizeof(info)); | |
121 | info.si_signo = SIGFPE; | |
122 | info.si_code = si_code; | |
123 | info.si_addr = (void __user *)instruction_pointer(regs); | |
124 | ||
125 | send_sig_info(SIGFPE, &info, current); | |
126 | } | |
127 | ||
128 | void fpsimd_thread_switch(struct task_struct *next) | |
129 | { | |
005f78cd AB |
130 | /* |
131 | * Save the current FPSIMD state to memory, but only if whatever is in | |
132 | * the registers is in fact the most recent userland FPSIMD state of | |
133 | * 'current'. | |
134 | */ | |
135 | if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) | |
53631b54 | 136 | fpsimd_save_state(¤t->thread.fpsimd_state); |
005f78cd AB |
137 | |
138 | if (next->mm) { | |
139 | /* | |
140 | * If we are switching to a task whose most recent userland | |
141 | * FPSIMD state is already in the registers of *this* cpu, | |
142 | * we can skip loading the state from memory. Otherwise, set | |
143 | * the TIF_FOREIGN_FPSTATE flag so the state will be loaded | |
144 | * upon the next return to userland. | |
145 | */ | |
146 | struct fpsimd_state *st = &next->thread.fpsimd_state; | |
147 | ||
148 | if (__this_cpu_read(fpsimd_last_state) == st | |
149 | && st->cpu == smp_processor_id()) | |
150 | clear_ti_thread_flag(task_thread_info(next), | |
151 | TIF_FOREIGN_FPSTATE); | |
152 | else | |
153 | set_ti_thread_flag(task_thread_info(next), | |
154 | TIF_FOREIGN_FPSTATE); | |
155 | } | |
53631b54 CM |
156 | } |
157 | ||
158 | void fpsimd_flush_thread(void) | |
159 | { | |
160 | memset(¤t->thread.fpsimd_state, 0, sizeof(struct fpsimd_state)); | |
674c242c | 161 | fpsimd_flush_task_state(current); |
005f78cd | 162 | set_thread_flag(TIF_FOREIGN_FPSTATE); |
53631b54 CM |
163 | } |
164 | ||
c51f9269 | 165 | /* |
005f78cd AB |
166 | * Save the userland FPSIMD state of 'current' to memory, but only if the state |
167 | * currently held in the registers does in fact belong to 'current' | |
c51f9269 AB |
168 | */ |
169 | void fpsimd_preserve_current_state(void) | |
170 | { | |
171 | preempt_disable(); | |
005f78cd AB |
172 | if (!test_thread_flag(TIF_FOREIGN_FPSTATE)) |
173 | fpsimd_save_state(¤t->thread.fpsimd_state); | |
c51f9269 AB |
174 | preempt_enable(); |
175 | } | |
176 | ||
177 | /* | |
005f78cd AB |
178 | * Load the userland FPSIMD state of 'current' from memory, but only if the |
179 | * FPSIMD state already held in the registers is /not/ the most recent FPSIMD | |
180 | * state of 'current' | |
181 | */ | |
182 | void fpsimd_restore_current_state(void) | |
183 | { | |
184 | preempt_disable(); | |
185 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { | |
186 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; | |
187 | ||
188 | fpsimd_load_state(st); | |
189 | this_cpu_write(fpsimd_last_state, st); | |
190 | st->cpu = smp_processor_id(); | |
191 | } | |
192 | preempt_enable(); | |
193 | } | |
194 | ||
195 | /* | |
196 | * Load an updated userland FPSIMD state for 'current' from memory and set the | |
197 | * flag that indicates that the FPSIMD register contents are the most recent | |
198 | * FPSIMD state of 'current' | |
c51f9269 AB |
199 | */ |
200 | void fpsimd_update_current_state(struct fpsimd_state *state) | |
201 | { | |
202 | preempt_disable(); | |
203 | fpsimd_load_state(state); | |
005f78cd AB |
204 | if (test_and_clear_thread_flag(TIF_FOREIGN_FPSTATE)) { |
205 | struct fpsimd_state *st = ¤t->thread.fpsimd_state; | |
206 | ||
207 | this_cpu_write(fpsimd_last_state, st); | |
208 | st->cpu = smp_processor_id(); | |
209 | } | |
c51f9269 AB |
210 | preempt_enable(); |
211 | } | |
212 | ||
005f78cd AB |
213 | /* |
214 | * Invalidate live CPU copies of task t's FPSIMD state | |
215 | */ | |
216 | void fpsimd_flush_task_state(struct task_struct *t) | |
217 | { | |
218 | t->thread.fpsimd_state.cpu = NR_CPUS; | |
219 | } | |
220 | ||
4cfb3613 AB |
221 | #ifdef CONFIG_KERNEL_MODE_NEON |
222 | ||
190f1ca8 AB |
223 | static DEFINE_PER_CPU(struct fpsimd_partial_state, hardirq_fpsimdstate); |
224 | static DEFINE_PER_CPU(struct fpsimd_partial_state, softirq_fpsimdstate); | |
225 | ||
4cfb3613 AB |
226 | /* |
227 | * Kernel-side NEON support functions | |
228 | */ | |
190f1ca8 | 229 | void kernel_neon_begin_partial(u32 num_regs) |
4cfb3613 | 230 | { |
190f1ca8 AB |
231 | if (in_interrupt()) { |
232 | struct fpsimd_partial_state *s = this_cpu_ptr( | |
233 | in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate); | |
4cfb3613 | 234 | |
190f1ca8 AB |
235 | BUG_ON(num_regs > 32); |
236 | fpsimd_save_partial_state(s, roundup(num_regs, 2)); | |
237 | } else { | |
238 | /* | |
239 | * Save the userland FPSIMD state if we have one and if we | |
240 | * haven't done so already. Clear fpsimd_last_state to indicate | |
241 | * that there is no longer userland FPSIMD state in the | |
242 | * registers. | |
243 | */ | |
244 | preempt_disable(); | |
245 | if (current->mm && | |
246 | !test_and_set_thread_flag(TIF_FOREIGN_FPSTATE)) | |
247 | fpsimd_save_state(¤t->thread.fpsimd_state); | |
248 | this_cpu_write(fpsimd_last_state, NULL); | |
249 | } | |
4cfb3613 | 250 | } |
190f1ca8 | 251 | EXPORT_SYMBOL(kernel_neon_begin_partial); |
4cfb3613 AB |
252 | |
253 | void kernel_neon_end(void) | |
254 | { | |
190f1ca8 AB |
255 | if (in_interrupt()) { |
256 | struct fpsimd_partial_state *s = this_cpu_ptr( | |
257 | in_irq() ? &hardirq_fpsimdstate : &softirq_fpsimdstate); | |
258 | fpsimd_load_partial_state(s); | |
259 | } else { | |
260 | preempt_enable(); | |
261 | } | |
4cfb3613 AB |
262 | } |
263 | EXPORT_SYMBOL(kernel_neon_end); | |
264 | ||
265 | #endif /* CONFIG_KERNEL_MODE_NEON */ | |
266 | ||
fb1ab1ab LP |
267 | #ifdef CONFIG_CPU_PM |
268 | static int fpsimd_cpu_pm_notifier(struct notifier_block *self, | |
269 | unsigned long cmd, void *v) | |
270 | { | |
271 | switch (cmd) { | |
272 | case CPU_PM_ENTER: | |
005f78cd | 273 | if (current->mm && !test_thread_flag(TIF_FOREIGN_FPSTATE)) |
fb1ab1ab | 274 | fpsimd_save_state(¤t->thread.fpsimd_state); |
7c68a9cc | 275 | this_cpu_write(fpsimd_last_state, NULL); |
fb1ab1ab LP |
276 | break; |
277 | case CPU_PM_EXIT: | |
278 | if (current->mm) | |
005f78cd | 279 | set_thread_flag(TIF_FOREIGN_FPSTATE); |
fb1ab1ab LP |
280 | break; |
281 | case CPU_PM_ENTER_FAILED: | |
282 | default: | |
283 | return NOTIFY_DONE; | |
284 | } | |
285 | return NOTIFY_OK; | |
286 | } | |
287 | ||
288 | static struct notifier_block fpsimd_cpu_pm_notifier_block = { | |
289 | .notifier_call = fpsimd_cpu_pm_notifier, | |
290 | }; | |
291 | ||
292 | static void fpsimd_pm_init(void) | |
293 | { | |
294 | cpu_pm_register_notifier(&fpsimd_cpu_pm_notifier_block); | |
295 | } | |
296 | ||
297 | #else | |
298 | static inline void fpsimd_pm_init(void) { } | |
299 | #endif /* CONFIG_CPU_PM */ | |
300 | ||
32365e64 JL |
301 | #ifdef CONFIG_HOTPLUG_CPU |
302 | static int fpsimd_cpu_hotplug_notifier(struct notifier_block *nfb, | |
303 | unsigned long action, | |
304 | void *hcpu) | |
305 | { | |
306 | unsigned int cpu = (long)hcpu; | |
307 | ||
308 | switch (action) { | |
309 | case CPU_DEAD: | |
310 | case CPU_DEAD_FROZEN: | |
311 | per_cpu(fpsimd_last_state, cpu) = NULL; | |
312 | break; | |
313 | } | |
314 | return NOTIFY_OK; | |
315 | } | |
316 | ||
317 | static struct notifier_block fpsimd_cpu_hotplug_notifier_block = { | |
318 | .notifier_call = fpsimd_cpu_hotplug_notifier, | |
319 | }; | |
320 | ||
321 | static inline void fpsimd_hotplug_init(void) | |
322 | { | |
323 | register_cpu_notifier(&fpsimd_cpu_hotplug_notifier_block); | |
324 | } | |
325 | ||
326 | #else | |
327 | static inline void fpsimd_hotplug_init(void) { } | |
328 | #endif | |
329 | ||
53631b54 CM |
330 | /* |
331 | * FP/SIMD support code initialisation. | |
332 | */ | |
333 | static int __init fpsimd_init(void) | |
334 | { | |
335 | u64 pfr = read_cpuid(ID_AA64PFR0_EL1); | |
336 | ||
337 | if (pfr & (0xf << 16)) { | |
338 | pr_notice("Floating-point is not implemented\n"); | |
339 | return 0; | |
340 | } | |
341 | elf_hwcap |= HWCAP_FP; | |
342 | ||
343 | if (pfr & (0xf << 20)) | |
344 | pr_notice("Advanced SIMD is not implemented\n"); | |
345 | else | |
346 | elf_hwcap |= HWCAP_ASIMD; | |
347 | ||
fb1ab1ab | 348 | fpsimd_pm_init(); |
32365e64 | 349 | fpsimd_hotplug_init(); |
fb1ab1ab | 350 | |
53631b54 CM |
351 | return 0; |
352 | } | |
353 | late_initcall(fpsimd_init); |