Commit | Line | Data |
---|---|---|
1eeaed76 RM |
1 | /* |
2 | * Copyright (C) 1994 Linus Torvalds | |
3 | * | |
4 | * Pentium III FXSR, SSE support | |
5 | * General FPU state handling cleanups | |
6 | * Gareth Hughes <gareth@valinux.com>, May 2000 | |
7 | * x86-64 work by Andi Kleen 2002 | |
8 | */ | |
9 | ||
10 | #ifndef _ASM_X86_I387_H | |
11 | #define _ASM_X86_I387_H | |
12 | ||
13 | #include <linux/sched.h> | |
14 | #include <linux/kernel_stat.h> | |
15 | #include <linux/regset.h> | |
e4914012 | 16 | #include <linux/hardirq.h> |
92c37fa3 | 17 | #include <asm/asm.h> |
1eeaed76 RM |
18 | #include <asm/processor.h> |
19 | #include <asm/sigcontext.h> | |
20 | #include <asm/user.h> | |
21 | #include <asm/uaccess.h> | |
22 | ||
23 | extern void fpu_init(void); | |
1eeaed76 | 24 | extern void mxcsr_feature_mask_init(void); |
aa283f49 | 25 | extern int init_fpu(struct task_struct *child); |
1eeaed76 | 26 | extern asmlinkage void math_state_restore(void); |
61c4628b | 27 | extern void init_thread_xstate(void); |
1eeaed76 RM |
28 | |
29 | extern user_regset_active_fn fpregs_active, xfpregs_active; | |
30 | extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get; | |
31 | extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set; | |
32 | ||
33 | #ifdef CONFIG_IA32_EMULATION | |
34 | struct _fpstate_ia32; | |
35 | extern int save_i387_ia32(struct _fpstate_ia32 __user *buf); | |
36 | extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf); | |
37 | #endif | |
38 | ||
39 | #ifdef CONFIG_X86_64 | |
40 | ||
41 | /* Ignore delayed exceptions from user space */ | |
42 | static inline void tolerant_fwait(void) | |
43 | { | |
44 | asm volatile("1: fwait\n" | |
45 | "2:\n" | |
affe6637 | 46 | _ASM_EXTABLE(1b, 2b)); |
1eeaed76 RM |
47 | } |
48 | ||
49 | static inline int restore_fpu_checking(struct i387_fxsave_struct *fx) | |
50 | { | |
51 | int err; | |
52 | ||
53 | asm volatile("1: rex64/fxrstor (%[fx])\n\t" | |
54 | "2:\n" | |
55 | ".section .fixup,\"ax\"\n" | |
56 | "3: movl $-1,%[err]\n" | |
57 | " jmp 2b\n" | |
58 | ".previous\n" | |
affe6637 | 59 | _ASM_EXTABLE(1b, 3b) |
1eeaed76 RM |
60 | : [err] "=r" (err) |
61 | #if 0 /* See comment in __save_init_fpu() below. */ | |
62 | : [fx] "r" (fx), "m" (*fx), "0" (0)); | |
63 | #else | |
64 | : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0)); | |
65 | #endif | |
1eeaed76 RM |
66 | return err; |
67 | } | |
68 | ||
69 | #define X87_FSW_ES (1 << 7) /* Exception Summary */ | |
70 | ||
71 | /* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception | |
72 | is pending. Clear the x87 state here by setting it to fixed | |
73 | values. The kernel data segment can be sometimes 0 and sometimes | |
74 | new user value. Both should be ok. | |
75 | Use the PDA as safe address because it should be already in L1. */ | |
76 | static inline void clear_fpu_state(struct i387_fxsave_struct *fx) | |
77 | { | |
78 | if (unlikely(fx->swd & X87_FSW_ES)) | |
affe6637 | 79 | asm volatile("fnclex"); |
1eeaed76 | 80 | alternative_input(ASM_NOP8 ASM_NOP2, |
affe6637 JP |
81 | " emms\n" /* clear stack tags */ |
82 | " fildl %%gs:0", /* load to clear state */ | |
83 | X86_FEATURE_FXSAVE_LEAK); | |
1eeaed76 RM |
84 | } |
85 | ||
86 | static inline int save_i387_checking(struct i387_fxsave_struct __user *fx) | |
87 | { | |
88 | int err; | |
89 | ||
90 | asm volatile("1: rex64/fxsave (%[fx])\n\t" | |
91 | "2:\n" | |
92 | ".section .fixup,\"ax\"\n" | |
93 | "3: movl $-1,%[err]\n" | |
94 | " jmp 2b\n" | |
95 | ".previous\n" | |
affe6637 | 96 | _ASM_EXTABLE(1b, 3b) |
1eeaed76 RM |
97 | : [err] "=r" (err), "=m" (*fx) |
98 | #if 0 /* See comment in __fxsave_clear() below. */ | |
99 | : [fx] "r" (fx), "0" (0)); | |
100 | #else | |
101 | : [fx] "cdaSDb" (fx), "0" (0)); | |
102 | #endif | |
affe6637 JP |
103 | if (unlikely(err) && |
104 | __clear_user(fx, sizeof(struct i387_fxsave_struct))) | |
1eeaed76 RM |
105 | err = -EFAULT; |
106 | /* No need to clear here because the caller clears USED_MATH */ | |
107 | return err; | |
108 | } | |
109 | ||
110 | static inline void __save_init_fpu(struct task_struct *tsk) | |
111 | { | |
112 | /* Using "rex64; fxsave %0" is broken because, if the memory operand | |
113 | uses any extended registers for addressing, a second REX prefix | |
114 | will be generated (to the assembler, rex64 followed by semicolon | |
115 | is a separate instruction), and hence the 64-bitness is lost. */ | |
116 | #if 0 | |
117 | /* Using "fxsaveq %0" would be the ideal choice, but is only supported | |
118 | starting with gas 2.16. */ | |
119 | __asm__ __volatile__("fxsaveq %0" | |
61c4628b | 120 | : "=m" (tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
121 | #elif 0 |
122 | /* Using, as a workaround, the properly prefixed form below isn't | |
123 | accepted by any binutils version so far released, complaining that | |
124 | the same type of prefix is used twice if an extended register is | |
125 | needed for addressing (fix submitted to mainline 2005-11-21). */ | |
126 | __asm__ __volatile__("rex64/fxsave %0" | |
61c4628b | 127 | : "=m" (tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
128 | #else |
129 | /* This, however, we can work around by forcing the compiler to select | |
130 | an addressing mode that doesn't require extended registers. */ | |
61c4628b SS |
131 | __asm__ __volatile__("rex64/fxsave (%1)" |
132 | : "=m" (tsk->thread.xstate->fxsave) | |
133 | : "cdaSDb" (&tsk->thread.xstate->fxsave)); | |
1eeaed76 | 134 | #endif |
61c4628b | 135 | clear_fpu_state(&tsk->thread.xstate->fxsave); |
1eeaed76 RM |
136 | task_thread_info(tsk)->status &= ~TS_USEDFPU; |
137 | } | |
138 | ||
1eeaed76 RM |
139 | #else /* CONFIG_X86_32 */ |
140 | ||
e8a496ac SS |
141 | extern void finit(void); |
142 | ||
1eeaed76 RM |
143 | static inline void tolerant_fwait(void) |
144 | { | |
145 | asm volatile("fnclex ; fwait"); | |
146 | } | |
147 | ||
148 | static inline void restore_fpu(struct task_struct *tsk) | |
149 | { | |
150 | /* | |
151 | * The "nop" is needed to make the instructions the same | |
152 | * length. | |
153 | */ | |
154 | alternative_input( | |
155 | "nop ; frstor %1", | |
156 | "fxrstor %1", | |
157 | X86_FEATURE_FXSR, | |
61c4628b | 158 | "m" (tsk->thread.xstate->fxsave)); |
1eeaed76 RM |
159 | } |
160 | ||
161 | /* We need a safe address that is cheap to find and that is already | |
162 | in L1 during context switch. The best choices are unfortunately | |
163 | different for UP and SMP */ | |
164 | #ifdef CONFIG_SMP | |
165 | #define safe_address (__per_cpu_offset[0]) | |
166 | #else | |
167 | #define safe_address (kstat_cpu(0).cpustat.user) | |
168 | #endif | |
169 | ||
170 | /* | |
171 | * These must be called with preempt disabled | |
172 | */ | |
173 | static inline void __save_init_fpu(struct task_struct *tsk) | |
174 | { | |
175 | /* Use more nops than strictly needed in case the compiler | |
176 | varies code */ | |
177 | alternative_input( | |
178 | "fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4, | |
179 | "fxsave %[fx]\n" | |
180 | "bt $7,%[fsw] ; jnc 1f ; fnclex\n1:", | |
181 | X86_FEATURE_FXSR, | |
61c4628b SS |
182 | [fx] "m" (tsk->thread.xstate->fxsave), |
183 | [fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory"); | |
1eeaed76 RM |
184 | /* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception |
185 | is pending. Clear the x87 state here by setting it to fixed | |
186 | values. safe_address is a random variable that should be in L1 */ | |
187 | alternative_input( | |
188 | GENERIC_NOP8 GENERIC_NOP2, | |
189 | "emms\n\t" /* clear stack tags */ | |
190 | "fildl %[addr]", /* set F?P to defined value */ | |
191 | X86_FEATURE_FXSAVE_LEAK, | |
192 | [addr] "m" (safe_address)); | |
193 | task_thread_info(tsk)->status &= ~TS_USEDFPU; | |
194 | } | |
195 | ||
196 | /* | |
197 | * Signal frame handlers... | |
198 | */ | |
199 | extern int save_i387(struct _fpstate __user *buf); | |
200 | extern int restore_i387(struct _fpstate __user *buf); | |
201 | ||
202 | #endif /* CONFIG_X86_64 */ | |
203 | ||
204 | static inline void __unlazy_fpu(struct task_struct *tsk) | |
205 | { | |
206 | if (task_thread_info(tsk)->status & TS_USEDFPU) { | |
207 | __save_init_fpu(tsk); | |
208 | stts(); | |
209 | } else | |
210 | tsk->fpu_counter = 0; | |
211 | } | |
212 | ||
213 | static inline void __clear_fpu(struct task_struct *tsk) | |
214 | { | |
215 | if (task_thread_info(tsk)->status & TS_USEDFPU) { | |
216 | tolerant_fwait(); | |
217 | task_thread_info(tsk)->status &= ~TS_USEDFPU; | |
218 | stts(); | |
219 | } | |
220 | } | |
221 | ||
222 | static inline void kernel_fpu_begin(void) | |
223 | { | |
224 | struct thread_info *me = current_thread_info(); | |
225 | preempt_disable(); | |
226 | if (me->status & TS_USEDFPU) | |
227 | __save_init_fpu(me->task); | |
228 | else | |
229 | clts(); | |
230 | } | |
231 | ||
232 | static inline void kernel_fpu_end(void) | |
233 | { | |
234 | stts(); | |
235 | preempt_enable(); | |
236 | } | |
237 | ||
e4914012 SS |
238 | /* |
239 | * Some instructions like VIA's padlock instructions generate a spurious | |
240 | * DNA fault but don't modify SSE registers. And these instructions | |
241 | * get used from interrupt context aswell. To prevent these kernel instructions | |
242 | * in interrupt context interact wrongly with other user/kernel fpu usage, we | |
243 | * should use them only in the context of irq_ts_save/restore() | |
244 | */ | |
245 | static inline int irq_ts_save(void) | |
246 | { | |
247 | /* | |
248 | * If we are in process context, we are ok to take a spurious DNA fault. | |
249 | * Otherwise, doing clts() in process context require pre-emption to | |
250 | * be disabled or some heavy lifting like kernel_fpu_begin() | |
251 | */ | |
252 | if (!in_interrupt()) | |
253 | return 0; | |
254 | ||
255 | if (read_cr0() & X86_CR0_TS) { | |
256 | clts(); | |
257 | return 1; | |
258 | } | |
259 | ||
260 | return 0; | |
261 | } | |
262 | ||
263 | static inline void irq_ts_restore(int TS_state) | |
264 | { | |
265 | if (TS_state) | |
266 | stts(); | |
267 | } | |
268 | ||
1eeaed76 RM |
269 | #ifdef CONFIG_X86_64 |
270 | ||
271 | static inline void save_init_fpu(struct task_struct *tsk) | |
272 | { | |
273 | __save_init_fpu(tsk); | |
274 | stts(); | |
275 | } | |
276 | ||
277 | #define unlazy_fpu __unlazy_fpu | |
278 | #define clear_fpu __clear_fpu | |
279 | ||
280 | #else /* CONFIG_X86_32 */ | |
281 | ||
282 | /* | |
283 | * These disable preemption on their own and are safe | |
284 | */ | |
285 | static inline void save_init_fpu(struct task_struct *tsk) | |
286 | { | |
287 | preempt_disable(); | |
288 | __save_init_fpu(tsk); | |
289 | stts(); | |
290 | preempt_enable(); | |
291 | } | |
292 | ||
293 | static inline void unlazy_fpu(struct task_struct *tsk) | |
294 | { | |
295 | preempt_disable(); | |
296 | __unlazy_fpu(tsk); | |
297 | preempt_enable(); | |
298 | } | |
299 | ||
300 | static inline void clear_fpu(struct task_struct *tsk) | |
301 | { | |
302 | preempt_disable(); | |
303 | __clear_fpu(tsk); | |
304 | preempt_enable(); | |
305 | } | |
306 | ||
307 | #endif /* CONFIG_X86_64 */ | |
308 | ||
1eeaed76 RM |
309 | /* |
310 | * i387 state interaction | |
311 | */ | |
312 | static inline unsigned short get_fpu_cwd(struct task_struct *tsk) | |
313 | { | |
314 | if (cpu_has_fxsr) { | |
61c4628b | 315 | return tsk->thread.xstate->fxsave.cwd; |
1eeaed76 | 316 | } else { |
1679f271 | 317 | return (unsigned short)tsk->thread.xstate->fsave.cwd; |
1eeaed76 RM |
318 | } |
319 | } | |
320 | ||
321 | static inline unsigned short get_fpu_swd(struct task_struct *tsk) | |
322 | { | |
323 | if (cpu_has_fxsr) { | |
61c4628b | 324 | return tsk->thread.xstate->fxsave.swd; |
1eeaed76 | 325 | } else { |
1679f271 | 326 | return (unsigned short)tsk->thread.xstate->fsave.swd; |
1eeaed76 RM |
327 | } |
328 | } | |
329 | ||
330 | static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk) | |
331 | { | |
332 | if (cpu_has_xmm) { | |
61c4628b | 333 | return tsk->thread.xstate->fxsave.mxcsr; |
1eeaed76 RM |
334 | } else { |
335 | return MXCSR_DEFAULT; | |
336 | } | |
337 | } | |
338 | ||
339 | #endif /* _ASM_X86_I387_H */ |