Commit | Line | Data |
---|---|---|
1eeaed76 RM |
1 | /* |
2 | * Copyright (C) 1994 Linus Torvalds | |
3 | * | |
4 | * Pentium III FXSR, SSE support | |
5 | * General FPU state handling cleanups | |
6 | * Gareth Hughes <gareth@valinux.com>, May 2000 | |
7 | * x86-64 work by Andi Kleen 2002 | |
8 | */ | |
9 | ||
10 | #ifndef _ASM_X86_I387_H | |
11 | #define _ASM_X86_I387_H | |
12 | ||
13 | #include <linux/sched.h> | |
14 | #include <linux/kernel_stat.h> | |
15 | #include <linux/regset.h> | |
92c37fa3 | 16 | #include <asm/asm.h> |
1eeaed76 RM |
17 | #include <asm/processor.h> |
18 | #include <asm/sigcontext.h> | |
19 | #include <asm/user.h> | |
20 | #include <asm/uaccess.h> | |
21 | ||
22 | extern void fpu_init(void); | |
1eeaed76 | 23 | extern void mxcsr_feature_mask_init(void); |
aa283f49 | 24 | extern int init_fpu(struct task_struct *child); |
1eeaed76 | 25 | extern asmlinkage void math_state_restore(void); |
61c4628b | 26 | extern void init_thread_xstate(void); |
1eeaed76 RM |
27 | |
28 | extern user_regset_active_fn fpregs_active, xfpregs_active; | |
29 | extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get; | |
30 | extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set; | |
31 | ||
32 | #ifdef CONFIG_IA32_EMULATION | |
33 | struct _fpstate_ia32; | |
34 | extern int save_i387_ia32(struct _fpstate_ia32 __user *buf); | |
35 | extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf); | |
36 | #endif | |
37 | ||
38 | #ifdef CONFIG_X86_64 | |
39 | ||
40 | /* Ignore delayed exceptions from user space */ | |
41 | static inline void tolerant_fwait(void) | |
42 | { | |
43 | asm volatile("1: fwait\n" | |
44 | "2:\n" | |
affe6637 | 45 | _ASM_EXTABLE(1b, 2b)); |
1eeaed76 RM |
46 | } |
47 | ||
48 | static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) | |
49 | { | |
50 | int err; | |
51 | ||
52 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" | |
53 | "2:\n" | |
54 | ".section .fixup,\"ax\"\n" | |
55 | "3: movl $-1,%[err]\n" | |
56 | " jmp 2b\n" | |
57 | ".previous\n" | |
affe6637 | 58 | _ASM_EXTABLE(1b, 3b) |
1eeaed76 RM |
59 | : [err] "=r" (err) |
60 | #if 0 /* See comment in __save_init_fpu() below. */ | |
61 | : [fx] "r" (fx), "m" (*fx), "0" (0)); | |
62 | #else | |
63 | : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0)); | |
64 | #endif | |
65 | if (unlikely(err)) | |
66 | init_fpu(current); | |
67 | return err; | |
68 | } | |
69 | ||
70 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ | |
71 | ||
72 | /* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception | |
73 | is pending. Clear the x87 state here by setting it to fixed | |
74 | values. The kernel data segment can be sometimes 0 and sometimes | |
75 | new user value. Both should be ok. | |
76 | Use the PDA as safe address because it should be already in L1. */ | |
77 | static inline void clear_fpu_state(struct i387_fxsave_struct *fx) | |
78 | { | |
79 | if (unlikely(fx->swd & X87_FSW_ES)) | |
affe6637 | 80 | asm volatile("fnclex"); |
1eeaed76 | 81 | alternative_input(ASM_NOP8 ASM_NOP2, |
affe6637 JP |
82 | " emms\n" /* clear stack tags */ |
83 | " fildl %%gs:0", /* load to clear state */ | |
84 | X86_FEATURE_FXSAVE_LEAK); | |
1eeaed76 RM |
85 | } |
86 | ||
87 | static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) | |
88 | { | |
89 | int err; | |
90 | ||
91 | asm volatile("1: rex64/fxsave (%[fx])\n\t" | |
92 | "2:\n" | |
93 | ".section .fixup,\"ax\"\n" | |
94 | "3: movl $-1,%[err]\n" | |
95 | " jmp 2b\n" | |
96 | ".previous\n" | |
affe6637 | 97 | _ASM_EXTABLE(1b, 3b) |
1eeaed76 RM |
98 | : [err] "=r" (err), "=m" (*fx) |
99 | #if 0 /* See comment in __fxsave_clear() below. */ | |
100 | : [fx] "r" (fx), "0" (0)); | |
101 | #else | |
102 | : [fx] "cdaSDb" (fx), "0" (0)); | |
103 | #endif | |
affe6637 JP |
104 | if (unlikely(err) && |
105 | __clear_user(fx, sizeof(struct i387_fxsave_struct))) | |
1eeaed76 RM |
106 | err = -EFAULT; |
107 | /* No need to clear here because the caller clears USED_MATH */ | |
108 | return err; | |
109 | } | |
110 | ||
111 | static inline void __save_init_fpu(struct task_struct *tsk) | |
112 | { | |
113 | /* Using "rex64; fxsave %0" is broken because, if the memory operand | |
114 | uses any extended registers for addressing, a second REX prefix | |
115 | will be generated (to the assembler, rex64 followed by semicolon | |
116 | is a separate instruction), and hence the 64-bitness is lost. */ | |
117 | #if 0 | |
118 | /* Using "fxsaveq %0" would be the ideal choice, but is only supported | |
119 | starting with gas 2.16. */ | |
120 | __asm__ __volatile__("fxsaveq %0" | |
61c4628b | 121 | : "=m" (tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
122 | #elif 0 |
123 | /* Using, as a workaround, the properly prefixed form below isn't | |
124 | accepted by any binutils version so far released, complaining that | |
125 | the same type of prefix is used twice if an extended register is | |
126 | needed for addressing (fix submitted to mainline 2005-11-21). */ | |
127 | __asm__ __volatile__("rex64/fxsave %0" | |
61c4628b | 128 | : "=m" (tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
129 | #else |
130 | /* This, however, we can work around by forcing the compiler to select | |
131 | an addressing mode that doesn't require extended registers. */ | |
61c4628b SS |
132 | __asm__ __volatile__("rex64/fxsave (%1)" |
133 | : "=m" (tsk->thread.xstate->fxsave) | |
134 | : "cdaSDb" (&tsk->thread.xstate->fxsave)); | |
1eeaed76 | 135 | #endif |
61c4628b | 136 | clear_fpu_state(&tsk->thread.xstate->fxsave); |
1eeaed76 RM |
137 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
138 | } | |
139 | ||
140 | /* | |
141 | * Signal frame handlers. | |
142 | */ | |
143 | ||
144 | static inline int save_i387(struct _fpstate __user *buf) | |
145 | { | |
146 | struct task_struct *tsk = current; | |
147 | int err = 0; | |
148 | ||
149 | BUILD_BUG_ON(sizeof(struct user_i387_struct) != | |
61c4628b | 150 | sizeof(tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
151 | |
152 | if ((unsigned long)buf % 16) | |
153 | printk("save_i387: bad fpstate %p\n", buf); | |
154 | ||
155 | if (!used_math()) | |
156 | return 0; | |
157 | clear_used_math(); /* trigger finit */ | |
158 | if (task_thread_info(tsk)->status & TS_USEDFPU) { | |
affe6637 JP |
159 | err = save_i387_checking((struct i387_fxsave_struct __user *) |
160 | buf); | |
161 | if (err) | |
162 | return err; | |
1eeaed76 RM |
163 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
164 | stts(); | |
165 | } else { | |
61c4628b | 166 | if (__copy_to_user(buf, &tsk->thread.xstate->fxsave, |
1eeaed76 RM |
167 | sizeof(struct i387_fxsave_struct))) |
168 | return -1; | |
169 | } | |
170 | return 1; | |
171 | } | |
172 | ||
173 | /* | |
174 | * This restores directly out of user space. Exceptions are handled. | |
175 | */ | |
176 | static inline int restore_i387(struct _fpstate __user *buf) | |
177 | { | |
fd3c3ed5 SS |
178 | struct task_struct *tsk = current; |
179 | int err; | |
180 | ||
181 | if (!used_math()) { | |
182 | err = init_fpu(tsk); | |
183 | if (err) | |
184 | return err; | |
185 | } | |
186 | ||
1eeaed76 RM |
187 | if (!(task_thread_info(current)->status & TS_USEDFPU)) { |
188 | clts(); | |
189 | task_thread_info(current)->status |= TS_USEDFPU; | |
190 | } | |
191 | return restore_fpu_checking((__force struct i387_fxsave_struct *)buf); | |
192 | } | |
193 | ||
194 | #else /* CONFIG_X86_32 */ | |
195 | ||
196 | static inline void tolerant_fwait(void) | |
197 | { | |
198 | asm volatile("fnclex ; fwait"); | |
199 | } | |
200 | ||
201 | static inline void restore_fpu(struct task_struct *tsk) | |
202 | { | |
203 | /* | |
204 | * The "nop" is needed to make the instructions the same | |
205 | * length. | |
206 | */ | |
207 | alternative_input( | |
208 | "nop ; frstor %1", | |
209 | "fxrstor %1", | |
210 | X86_FEATURE_FXSR, | |
61c4628b | 211 | "m" (tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
212 | } |
213 | ||
214 | /* We need a safe address that is cheap to find and that is already | |
215 | in L1 during context switch. The best choices are unfortunately | |
216 | different for UP and SMP */ | |
217 | #ifdef CONFIG_SMP | |
218 | #define safe_address (__per_cpu_offset[0]) | |
219 | #else | |
220 | #define safe_address (kstat_cpu(0).cpustat.user) | |
221 | #endif | |
222 | ||
223 | /* | |
224 | * These must be called with preempt disabled | |
225 | */ | |
226 | static inline void __save_init_fpu(struct task_struct *tsk) | |
227 | { | |
228 | /* Use more nops than strictly needed in case the compiler | |
229 | varies code */ | |
230 | alternative_input( | |
231 | "fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4, | |
232 | "fxsave %[fx]\n" | |
233 | "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", | |
234 | X86_FEATURE_FXSR, | |
61c4628b SS |
235 | [fx] "m" (tsk->thread.xstate->fxsave), |
236 | [fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory"); | |
1eeaed76 RM |
237 | /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception |
238 | is pending. Clear the x87 state here by setting it to fixed | |
239 | values. safe_address is a random variable that should be in L1 */ | |
240 | alternative_input( | |
241 | GENERIC_NOP8 GENERIC_NOP2, | |
242 | "emms\n\t" /* clear stack tags */ | |
243 | "fildl %[addr]", /* set F?P to defined value */ | |
244 | X86_FEATURE_FXSAVE_LEAK, | |
245 | [addr] "m" (safe_address)); | |
246 | task_thread_info(tsk)->status &= ~TS_USEDFPU; | |
247 | } | |
248 | ||
249 | /* | |
250 | * Signal frame handlers... | |
251 | */ | |
252 | extern int save_i387(struct _fpstate __user *buf); | |
253 | extern int restore_i387(struct _fpstate __user *buf); | |
254 | ||
255 | #endif /* CONFIG_X86_64 */ | |
256 | ||
257 | static inline void __unlazy_fpu(struct task_struct *tsk) | |
258 | { | |
259 | if (task_thread_info(tsk)->status & TS_USEDFPU) { | |
260 | __save_init_fpu(tsk); | |
261 | stts(); | |
262 | } else | |
263 | tsk->fpu_counter = 0; | |
264 | } | |
265 | ||
266 | static inline void __clear_fpu(struct task_struct *tsk) | |
267 | { | |
268 | if (task_thread_info(tsk)->status & TS_USEDFPU) { | |
269 | tolerant_fwait(); | |
270 | task_thread_info(tsk)->status &= ~TS_USEDFPU; | |
271 | stts(); | |
272 | } | |
273 | } | |
274 | ||
275 | static inline void kernel_fpu_begin(void) | |
276 | { | |
277 | struct thread_info *me = current_thread_info(); | |
278 | preempt_disable(); | |
279 | if (me->status & TS_USEDFPU) | |
280 | __save_init_fpu(me->task); | |
281 | else | |
282 | clts(); | |
283 | } | |
284 | ||
285 | static inline void kernel_fpu_end(void) | |
286 | { | |
287 | stts(); | |
288 | preempt_enable(); | |
289 | } | |
290 | ||
291 | #ifdef CONFIG_X86_64 | |
292 | ||
293 | static inline void save_init_fpu(struct task_struct *tsk) | |
294 | { | |
295 | __save_init_fpu(tsk); | |
296 | stts(); | |
297 | } | |
298 | ||
299 | #define unlazy_fpu __unlazy_fpu | |
300 | #define clear_fpu __clear_fpu | |
301 | ||
302 | #else /* CONFIG_X86_32 */ | |
303 | ||
304 | /* | |
305 | * These disable preemption on their own and are safe | |
306 | */ | |
307 | static inline void save_init_fpu(struct task_struct *tsk) | |
308 | { | |
309 | preempt_disable(); | |
310 | __save_init_fpu(tsk); | |
311 | stts(); | |
312 | preempt_enable(); | |
313 | } | |
314 | ||
315 | static inline void unlazy_fpu(struct task_struct *tsk) | |
316 | { | |
317 | preempt_disable(); | |
318 | __unlazy_fpu(tsk); | |
319 | preempt_enable(); | |
320 | } | |
321 | ||
322 | static inline void clear_fpu(struct task_struct *tsk) | |
323 | { | |
324 | preempt_disable(); | |
325 | __clear_fpu(tsk); | |
326 | preempt_enable(); | |
327 | } | |
328 | ||
329 | #endif /* CONFIG_X86_64 */ | |
330 | ||
1eeaed76 RM |
331 | /* |
332 | * i387 state interaction | |
333 | */ | |
334 | static inline unsigned short get_fpu_cwd(struct task_struct *tsk) | |
335 | { | |
336 | if (cpu_has_fxsr) { | |
61c4628b | 337 | return tsk->thread.xstate->fxsave.cwd; |
1eeaed76 | 338 | } else { |
1679f271 | 339 | return (unsigned short)tsk->thread.xstate->fsave.cwd; |
1eeaed76 RM |
340 | } |
341 | } | |
342 | ||
343 | static inline unsigned short get_fpu_swd(struct task_struct *tsk) | |
344 | { | |
345 | if (cpu_has_fxsr) { | |
61c4628b | 346 | return tsk->thread.xstate->fxsave.swd; |
1eeaed76 | 347 | } else { |
1679f271 | 348 | return (unsigned short)tsk->thread.xstate->fsave.swd; |
1eeaed76 RM |
349 | } |
350 | } | |
351 | ||
352 | static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) | |
353 | { | |
354 | if (cpu_has_xmm) { | |
61c4628b | 355 | return tsk->thread.xstate->fxsave.mxcsr; |
1eeaed76 RM |
356 | } else { |
357 | return MXCSR_DEFAULT; | |
358 | } | |
359 | } | |
360 | ||
361 | #endif /* _ASM_X86_I387_H */ |