2 * Copyright (C) 1994 Linus Torvalds
4 * Pentium III FXSR, SSE support
5 * General FPU state handling cleanups
6 * Gareth Hughes <gareth@valinux.com>, May 2000
8 #include <linux/module.h>
9 #include <linux/regset.h>
10 #include <linux/sched.h>
11 #include <linux/slab.h>
13 #include <asm/sigcontext.h>
14 #include <asm/processor.h>
15 #include <asm/math_emu.h>
16 #include <asm/uaccess.h>
17 #include <asm/ptrace.h>
19 #include <asm/fpu-internal.h>
23 * Were we in an interrupt that interrupted kernel mode?
25 * On others, we can do a kernel_fpu_begin/end() pair *ONLY* if that
26 * pair does nothing at all: the thread must not have fpu (so
27 * that we don't try to save the FPU state), and TS must
28 * be set (so that the clts/stts pair does nothing that is
29 * visible in the interrupted kernel thread).
31 * Except for the eagerfpu case when we return 1 unless we've already
32 * been eager and saved the state in kernel_fpu_begin().
34 static inline bool interrupted_kernel_fpu_idle(void)
37 return __thread_has_fpu(current
);
39 return !__thread_has_fpu(current
) &&
40 (read_cr0() & X86_CR0_TS
);
44 * Were we in user mode (or vm86 mode) when we were
47 * Doing kernel_fpu_begin/end() is ok if we are running
48 * in an interrupt context from user mode - we'll just
49 * save the FPU state as required.
51 static inline bool interrupted_user_mode(void)
53 struct pt_regs
*regs
= get_irq_regs();
54 return regs
&& user_mode_vm(regs
);
58 * Can we use the FPU in kernel mode with the
59 * whole "kernel_fpu_begin/end()" sequence?
61 * It's always ok in process context (ie "not interrupt")
62 * but it is sometimes ok even from an irq.
64 bool irq_fpu_usable(void)
66 return !in_interrupt() ||
67 interrupted_user_mode() ||
68 interrupted_kernel_fpu_idle();
70 EXPORT_SYMBOL(irq_fpu_usable
);
72 void __kernel_fpu_begin(void)
74 struct task_struct
*me
= current
;
76 if (__thread_has_fpu(me
)) {
77 __thread_clear_has_fpu(me
);
79 /* We do 'stts()' in __kernel_fpu_end() */
80 } else if (!use_eager_fpu()) {
81 this_cpu_write(fpu_owner_task
, NULL
);
85 EXPORT_SYMBOL(__kernel_fpu_begin
);
87 void __kernel_fpu_end(void)
89 if (use_eager_fpu()) {
91 * For eager fpu, most the time, tsk_used_math() is true.
92 * Restore the user math as we are done with the kernel usage.
93 * At few instances during thread exit, signal handling etc,
94 * tsk_used_math() is false. Those few places will take proper
95 * actions, so we don't need to restore the math here.
97 if (likely(tsk_used_math(current
)))
103 EXPORT_SYMBOL(__kernel_fpu_end
);
105 void unlazy_fpu(struct task_struct
*tsk
)
108 if (__thread_has_fpu(tsk
)) {
109 if (use_eager_fpu()) {
112 __save_init_fpu(tsk
);
113 __thread_fpu_end(tsk
);
118 EXPORT_SYMBOL(unlazy_fpu
);
120 unsigned int mxcsr_feature_mask __read_mostly
= 0xffffffffu
;
121 unsigned int xstate_size
;
122 EXPORT_SYMBOL_GPL(xstate_size
);
123 static struct i387_fxsave_struct fx_scratch
;
125 static void mxcsr_feature_mask_init(void)
127 unsigned long mask
= 0;
130 memset(&fx_scratch
, 0, sizeof(struct i387_fxsave_struct
));
131 asm volatile("fxsave %0" : "+m" (fx_scratch
));
132 mask
= fx_scratch
.mxcsr_mask
;
136 mxcsr_feature_mask
&= mask
;
139 static void init_thread_xstate(void)
142 * Note that xstate_size might be overwriten later during
148 * Disable xsave as we do not support it if i387
149 * emulation is enabled.
151 setup_clear_cpu_cap(X86_FEATURE_XSAVE
);
152 setup_clear_cpu_cap(X86_FEATURE_XSAVEOPT
);
153 xstate_size
= sizeof(struct i387_soft_struct
);
158 xstate_size
= sizeof(struct i387_fxsave_struct
);
160 xstate_size
= sizeof(struct i387_fsave_struct
);
164 * Called at bootup to set up the initial FPU state that is later cloned
165 * into all processes.
171 unsigned long cr4_mask
= 0;
173 #ifndef CONFIG_MATH_EMULATION
175 pr_emerg("No FPU found and no math emulation present\n");
176 pr_emerg("Giving up\n");
182 cr4_mask
|= X86_CR4_OSFXSR
;
184 cr4_mask
|= X86_CR4_OSXMMEXCPT
;
186 set_in_cr4(cr4_mask
);
189 cr0
&= ~(X86_CR0_TS
|X86_CR0_EM
); /* clear TS and EM */
195 * init_thread_xstate is only called once to avoid overriding
196 * xstate_size during boot time or during CPU hotplug.
198 if (xstate_size
== 0)
199 init_thread_xstate();
201 mxcsr_feature_mask_init();
206 void fpu_finit(struct fpu
*fpu
)
209 finit_soft_fpu(&fpu
->state
->soft
);
214 fx_finit(&fpu
->state
->fxsave
);
216 struct i387_fsave_struct
*fp
= &fpu
->state
->fsave
;
217 memset(fp
, 0, xstate_size
);
218 fp
->cwd
= 0xffff037fu
;
219 fp
->swd
= 0xffff0000u
;
220 fp
->twd
= 0xffffffffu
;
221 fp
->fos
= 0xffff0000u
;
224 EXPORT_SYMBOL_GPL(fpu_finit
);
227 * The _current_ task is using the FPU for the first time
228 * so initialize it and set the mxcsr to its default
229 * value at reset if we support XMM instructions and then
230 * remember the current task has used the FPU.
232 int init_fpu(struct task_struct
*tsk
)
236 if (tsk_used_math(tsk
)) {
237 if (cpu_has_fpu
&& tsk
== current
)
239 tsk
->thread
.fpu
.last_cpu
= ~0;
244 * Memory allocation at the first usage of the FPU and other state.
246 ret
= fpu_alloc(&tsk
->thread
.fpu
);
250 fpu_finit(&tsk
->thread
.fpu
);
252 set_stopped_child_used_math(tsk
);
255 EXPORT_SYMBOL_GPL(init_fpu
);
258 * The xstateregs_active() routine is the same as the fpregs_active() routine,
259 * as the "regset->n" for the xstate regset will be updated based on the feature
260 * capabilites supported by the xsave.
262 int fpregs_active(struct task_struct
*target
, const struct user_regset
*regset
)
264 return tsk_used_math(target
) ? regset
->n
: 0;
267 int xfpregs_active(struct task_struct
*target
, const struct user_regset
*regset
)
269 return (cpu_has_fxsr
&& tsk_used_math(target
)) ? regset
->n
: 0;
272 int xfpregs_get(struct task_struct
*target
, const struct user_regset
*regset
,
273 unsigned int pos
, unsigned int count
,
274 void *kbuf
, void __user
*ubuf
)
281 ret
= init_fpu(target
);
285 sanitize_i387_state(target
);
287 return user_regset_copyout(&pos
, &count
, &kbuf
, &ubuf
,
288 &target
->thread
.fpu
.state
->fxsave
, 0, -1);
291 int xfpregs_set(struct task_struct
*target
, const struct user_regset
*regset
,
292 unsigned int pos
, unsigned int count
,
293 const void *kbuf
, const void __user
*ubuf
)
300 ret
= init_fpu(target
);
304 sanitize_i387_state(target
);
306 ret
= user_regset_copyin(&pos
, &count
, &kbuf
, &ubuf
,
307 &target
->thread
.fpu
.state
->fxsave
, 0, -1);
310 * mxcsr reserved bits must be masked to zero for security reasons.
312 target
->thread
.fpu
.state
->fxsave
.mxcsr
&= mxcsr_feature_mask
;
315 * update the header bits in the xsave header, indicating the
316 * presence of FP and SSE state.
319 target
->thread
.fpu
.state
->xsave
.xsave_hdr
.xstate_bv
|= XSTATE_FPSSE
;
324 int xstateregs_get(struct task_struct
*target
, const struct user_regset
*regset
,
325 unsigned int pos
, unsigned int count
,
326 void *kbuf
, void __user
*ubuf
)
333 ret
= init_fpu(target
);
338 * Copy the 48bytes defined by the software first into the xstate
339 * memory layout in the thread struct, so that we can copy the entire
340 * xstateregs to the user using one user_regset_copyout().
342 memcpy(&target
->thread
.fpu
.state
->fxsave
.sw_reserved
,
343 xstate_fx_sw_bytes
, sizeof(xstate_fx_sw_bytes
));
346 * Copy the xstate memory layout.
348 ret
= user_regset_copyout(&pos
, &count
, &kbuf
, &ubuf
,
349 &target
->thread
.fpu
.state
->xsave
, 0, -1);
353 int xstateregs_set(struct task_struct
*target
, const struct user_regset
*regset
,
354 unsigned int pos
, unsigned int count
,
355 const void *kbuf
, const void __user
*ubuf
)
358 struct xsave_hdr_struct
*xsave_hdr
;
363 ret
= init_fpu(target
);
367 ret
= user_regset_copyin(&pos
, &count
, &kbuf
, &ubuf
,
368 &target
->thread
.fpu
.state
->xsave
, 0, -1);
371 * mxcsr reserved bits must be masked to zero for security reasons.
373 target
->thread
.fpu
.state
->fxsave
.mxcsr
&= mxcsr_feature_mask
;
375 xsave_hdr
= &target
->thread
.fpu
.state
->xsave
.xsave_hdr
;
377 xsave_hdr
->xstate_bv
&= pcntxt_mask
;
379 * These bits must be zero.
381 memset(xsave_hdr
->reserved
, 0, 48);
386 #if defined CONFIG_X86_32 || defined CONFIG_IA32_EMULATION
389 * FPU tag word conversions.
392 static inline unsigned short twd_i387_to_fxsr(unsigned short twd
)
394 unsigned int tmp
; /* to avoid 16 bit prefixes in the code */
396 /* Transform each pair of bits into 01 (valid) or 00 (empty) */
398 tmp
= (tmp
| (tmp
>>1)) & 0x5555; /* 0V0V0V0V0V0V0V0V */
399 /* and move the valid bits to the lower byte. */
400 tmp
= (tmp
| (tmp
>> 1)) & 0x3333; /* 00VV00VV00VV00VV */
401 tmp
= (tmp
| (tmp
>> 2)) & 0x0f0f; /* 0000VVVV0000VVVV */
402 tmp
= (tmp
| (tmp
>> 4)) & 0x00ff; /* 00000000VVVVVVVV */
407 #define FPREG_ADDR(f, n) ((void *)&(f)->st_space + (n) * 16)
408 #define FP_EXP_TAG_VALID 0
409 #define FP_EXP_TAG_ZERO 1
410 #define FP_EXP_TAG_SPECIAL 2
411 #define FP_EXP_TAG_EMPTY 3
413 static inline u32
twd_fxsr_to_i387(struct i387_fxsave_struct
*fxsave
)
416 u32 tos
= (fxsave
->swd
>> 11) & 7;
417 u32 twd
= (unsigned long) fxsave
->twd
;
419 u32 ret
= 0xffff0000u
;
422 for (i
= 0; i
< 8; i
++, twd
>>= 1) {
424 st
= FPREG_ADDR(fxsave
, (i
- tos
) & 7);
426 switch (st
->exponent
& 0x7fff) {
428 tag
= FP_EXP_TAG_SPECIAL
;
431 if (!st
->significand
[0] &&
432 !st
->significand
[1] &&
433 !st
->significand
[2] &&
435 tag
= FP_EXP_TAG_ZERO
;
437 tag
= FP_EXP_TAG_SPECIAL
;
440 if (st
->significand
[3] & 0x8000)
441 tag
= FP_EXP_TAG_VALID
;
443 tag
= FP_EXP_TAG_SPECIAL
;
447 tag
= FP_EXP_TAG_EMPTY
;
449 ret
|= tag
<< (2 * i
);
455 * FXSR floating point environment conversions.
459 convert_from_fxsr(struct user_i387_ia32_struct
*env
, struct task_struct
*tsk
)
461 struct i387_fxsave_struct
*fxsave
= &tsk
->thread
.fpu
.state
->fxsave
;
462 struct _fpreg
*to
= (struct _fpreg
*) &env
->st_space
[0];
463 struct _fpxreg
*from
= (struct _fpxreg
*) &fxsave
->st_space
[0];
466 env
->cwd
= fxsave
->cwd
| 0xffff0000u
;
467 env
->swd
= fxsave
->swd
| 0xffff0000u
;
468 env
->twd
= twd_fxsr_to_i387(fxsave
);
471 env
->fip
= fxsave
->rip
;
472 env
->foo
= fxsave
->rdp
;
474 * should be actually ds/cs at fpu exception time, but
475 * that information is not available in 64bit mode.
477 env
->fcs
= task_pt_regs(tsk
)->cs
;
478 if (tsk
== current
) {
479 savesegment(ds
, env
->fos
);
481 env
->fos
= tsk
->thread
.ds
;
483 env
->fos
|= 0xffff0000;
485 env
->fip
= fxsave
->fip
;
486 env
->fcs
= (u16
) fxsave
->fcs
| ((u32
) fxsave
->fop
<< 16);
487 env
->foo
= fxsave
->foo
;
488 env
->fos
= fxsave
->fos
;
491 for (i
= 0; i
< 8; ++i
)
492 memcpy(&to
[i
], &from
[i
], sizeof(to
[0]));
495 void convert_to_fxsr(struct task_struct
*tsk
,
496 const struct user_i387_ia32_struct
*env
)
499 struct i387_fxsave_struct
*fxsave
= &tsk
->thread
.fpu
.state
->fxsave
;
500 struct _fpreg
*from
= (struct _fpreg
*) &env
->st_space
[0];
501 struct _fpxreg
*to
= (struct _fpxreg
*) &fxsave
->st_space
[0];
504 fxsave
->cwd
= env
->cwd
;
505 fxsave
->swd
= env
->swd
;
506 fxsave
->twd
= twd_i387_to_fxsr(env
->twd
);
507 fxsave
->fop
= (u16
) ((u32
) env
->fcs
>> 16);
509 fxsave
->rip
= env
->fip
;
510 fxsave
->rdp
= env
->foo
;
511 /* cs and ds ignored */
513 fxsave
->fip
= env
->fip
;
514 fxsave
->fcs
= (env
->fcs
& 0xffff);
515 fxsave
->foo
= env
->foo
;
516 fxsave
->fos
= env
->fos
;
519 for (i
= 0; i
< 8; ++i
)
520 memcpy(&to
[i
], &from
[i
], sizeof(from
[0]));
523 int fpregs_get(struct task_struct
*target
, const struct user_regset
*regset
,
524 unsigned int pos
, unsigned int count
,
525 void *kbuf
, void __user
*ubuf
)
527 struct user_i387_ia32_struct env
;
530 ret
= init_fpu(target
);
534 if (!static_cpu_has(X86_FEATURE_FPU
))
535 return fpregs_soft_get(target
, regset
, pos
, count
, kbuf
, ubuf
);
538 return user_regset_copyout(&pos
, &count
, &kbuf
, &ubuf
,
539 &target
->thread
.fpu
.state
->fsave
, 0,
542 sanitize_i387_state(target
);
544 if (kbuf
&& pos
== 0 && count
== sizeof(env
)) {
545 convert_from_fxsr(kbuf
, target
);
549 convert_from_fxsr(&env
, target
);
551 return user_regset_copyout(&pos
, &count
, &kbuf
, &ubuf
, &env
, 0, -1);
554 int fpregs_set(struct task_struct
*target
, const struct user_regset
*regset
,
555 unsigned int pos
, unsigned int count
,
556 const void *kbuf
, const void __user
*ubuf
)
558 struct user_i387_ia32_struct env
;
561 ret
= init_fpu(target
);
565 sanitize_i387_state(target
);
567 if (!static_cpu_has(X86_FEATURE_FPU
))
568 return fpregs_soft_set(target
, regset
, pos
, count
, kbuf
, ubuf
);
571 return user_regset_copyin(&pos
, &count
, &kbuf
, &ubuf
,
572 &target
->thread
.fpu
.state
->fsave
, 0,
575 if (pos
> 0 || count
< sizeof(env
))
576 convert_from_fxsr(&env
, target
);
578 ret
= user_regset_copyin(&pos
, &count
, &kbuf
, &ubuf
, &env
, 0, -1);
580 convert_to_fxsr(target
, &env
);
583 * update the header bit in the xsave header, indicating the
587 target
->thread
.fpu
.state
->xsave
.xsave_hdr
.xstate_bv
|= XSTATE_FP
;
592 * FPU state for core dumps.
593 * This is only used for a.out dumps now.
594 * It is declared generically using elf_fpregset_t (which is
595 * struct user_i387_struct) but is in fact only used for 32-bit
596 * dumps, so on 64-bit it is really struct user_i387_ia32_struct.
598 int dump_fpu(struct pt_regs
*regs
, struct user_i387_struct
*fpu
)
600 struct task_struct
*tsk
= current
;
603 fpvalid
= !!used_math();
605 fpvalid
= !fpregs_get(tsk
, NULL
,
606 0, sizeof(struct user_i387_ia32_struct
),
611 EXPORT_SYMBOL(dump_fpu
);
613 #endif /* CONFIG_X86_32 || CONFIG_IA32_EMULATION */
615 static int __init
no_387(char *s
)
617 setup_clear_cpu_cap(X86_FEATURE_FPU
);
621 __setup("no387", no_387
);
623 void fpu_detect(struct cpuinfo_x86
*c
)
631 cr0
&= ~(X86_CR0_TS
| X86_CR0_EM
);
634 asm volatile("fninit ; fnstsw %0 ; fnstcw %1"
635 : "+m" (fsw
), "+m" (fcw
));
637 if (fsw
== 0 && (fcw
& 0x103f) == 0x003f)
638 set_cpu_cap(c
, X86_FEATURE_FPU
);
640 clear_cpu_cap(c
, X86_FEATURE_FPU
);
642 /* The final cr0 value is set in fpu_init() */