[deliverable/linux.git] / include / asm-x86 / i387.h

/*
 * Copyright (C) 1994 Linus Torvalds
 *
 * Pentium III FXSR, SSE support
 * General FPU state handling cleanups
 *	Gareth Hughes <gareth@valinux.com>, May 2000
 * x86-64 work by Andi Kleen 2002
 */

#ifndef ASM_X86__I387_H
#define ASM_X86__I387_H

#include <linux/sched.h>
#include <linux/kernel_stat.h>
#include <linux/regset.h>
#include <asm/asm.h>
#include <asm/processor.h>
#include <asm/sigcontext.h>
#include <asm/user.h>
#include <asm/uaccess.h>

extern void fpu_init(void);
extern void mxcsr_feature_mask_init(void);
extern int init_fpu(struct task_struct *child);
extern asmlinkage void math_state_restore(void);
extern void init_thread_xstate(void);

extern user_regset_active_fn fpregs_active, xfpregs_active;
extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get;
extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set;

#ifdef CONFIG_IA32_EMULATION
struct _fpstate_ia32;
extern int save_i387_ia32(struct _fpstate_ia32 __user *buf);
extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf);
#endif

#ifdef CONFIG_X86_64

/* Ignore delayed exceptions from user space */
static inline void tolerant_fwait(void)
{
	asm volatile("1: fwait\n"
		     "2:\n"
		     _ASM_EXTABLE(1b, 2b));
}

static inline int restore_fpu_checking(struct i387_fxsave_struct *fx)
{
	int err;

	asm volatile("1:  rex64/fxrstor (%[fx])\n\t"
		     "2:\n"
		     ".section .fixup,\"ax\"\n"
		     "3:  movl $-1,%[err]\n"
		     "    jmp  2b\n"
		     ".previous\n"
		     _ASM_EXTABLE(1b, 3b)
		     : [err] "=r" (err)
#if 0 /* See comment in __save_init_fpu() below. */
		     : [fx] "r" (fx), "m" (*fx), "0" (0));
#else
		     : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0));
#endif
	return err;
}

#define X87_FSW_ES (1 << 7)	/* Exception Summary */

/* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
   is pending. Clear the x87 state here by setting it to fixed
   values. The kernel data segment can be sometimes 0 and sometimes
   new user value. Both should be ok.
   Use the PDA as safe address because it should be already in L1. */
static inline void clear_fpu_state(struct i387_fxsave_struct *fx)
{
	if (unlikely(fx->swd & X87_FSW_ES))
		asm volatile("fnclex");
	alternative_input(ASM_NOP8 ASM_NOP2,
			  "    emms\n"		/* clear stack tags */
			  "    fildl %%gs:0",	/* load to clear state */
			  X86_FEATURE_FXSAVE_LEAK);
}

static inline int save_i387_checking(struct i387_fxsave_struct __user *fx)
{
	int err;

	asm volatile("1:  rex64/fxsave (%[fx])\n\t"
		     "2:\n"
		     ".section .fixup,\"ax\"\n"
		     "3:  movl $-1,%[err]\n"
		     "    jmp  2b\n"
		     ".previous\n"
		     _ASM_EXTABLE(1b, 3b)
		     : [err] "=r" (err), "=m" (*fx)
#if 0 /* See comment in __fxsave_clear() below. */
		     : [fx] "r" (fx), "0" (0));
#else
		     : [fx] "cdaSDb" (fx), "0" (0));
#endif
	if (unlikely(err) &&
	    __clear_user(fx, sizeof(struct i387_fxsave_struct)))
		err = -EFAULT;
	/* No need to clear here because the caller clears USED_MATH */
	return err;
}

static inline void __save_init_fpu(struct task_struct *tsk)
{
	/* Using "rex64; fxsave %0" is broken because, if the memory operand
	   uses any extended registers for addressing, a second REX prefix
	   will be generated (to the assembler, rex64 followed by semicolon
	   is a separate instruction), and hence the 64-bitness is lost. */
#if 0
	/* Using "fxsaveq %0" would be the ideal choice, but is only supported
	   starting with gas 2.16. */
	__asm__ __volatile__("fxsaveq %0"
			     : "=m" (tsk->thread.xstate->fxsave));
#elif 0
	/* Using, as a workaround, the properly prefixed form below isn't
	   accepted by any binutils version so far released, complaining that
	   the same type of prefix is used twice if an extended register is
	   needed for addressing (fix submitted to mainline 2005-11-21). */
	__asm__ __volatile__("rex64/fxsave %0"
			     : "=m" (tsk->thread.xstate->fxsave));
#else
	/* This, however, we can work around by forcing the compiler to select
	   an addressing mode that doesn't require extended registers. */
	__asm__ __volatile__("rex64/fxsave (%1)"
			     : "=m" (tsk->thread.xstate->fxsave)
			     : "cdaSDb" (&tsk->thread.xstate->fxsave));
#endif
	clear_fpu_state(&tsk->thread.xstate->fxsave);
	task_thread_info(tsk)->status &= ~TS_USEDFPU;
}

#else  /* CONFIG_X86_32 */

extern void finit(void);

static inline void tolerant_fwait(void)
{
	asm volatile("fnclex ; fwait");
}

static inline void restore_fpu(struct task_struct *tsk)
{
	/*
	 * The "nop" is needed to make the instructions the same
	 * length.
	 */
	alternative_input(
		"nop ; frstor %1",
		"fxrstor %1",
		X86_FEATURE_FXSR,
		"m" (tsk->thread.xstate->fxsave));
}

/* We need a safe address that is cheap to find and that is already
   in L1 during context switch. The best choices are unfortunately
   different for UP and SMP */
#ifdef CONFIG_SMP
#define safe_address (__per_cpu_offset[0])
#else
#define safe_address (kstat_cpu(0).cpustat.user)
#endif

/*
 * These must be called with preempt disabled
 */
static inline void __save_init_fpu(struct task_struct *tsk)
{
	/* Use more nops than strictly needed in case the compiler
	   varies code */
	alternative_input(
		"fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4,
		"fxsave %[fx]\n"
		"bt $7,%[fsw] ; jnc 1f ; fnclex\n1:",
		X86_FEATURE_FXSR,
		[fx] "m" (tsk->thread.xstate->fxsave),
		[fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory");
	/* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
	   is pending.  Clear the x87 state here by setting it to fixed
	   values. safe_address is a random variable that should be in L1 */
	alternative_input(
		GENERIC_NOP8 GENERIC_NOP2,
		"emms\n\t"	  	/* clear stack tags */
		"fildl %[addr]", 	/* set F?P to defined value */
		X86_FEATURE_FXSAVE_LEAK,
		[addr] "m" (safe_address));
	task_thread_info(tsk)->status &= ~TS_USEDFPU;
}

/*
 * Signal frame handlers...
 */
extern int save_i387(struct _fpstate __user *buf);
extern int restore_i387(struct _fpstate __user *buf);

#endif	/* CONFIG_X86_64 */

static inline void __unlazy_fpu(struct task_struct *tsk)
{
	if (task_thread_info(tsk)->status & TS_USEDFPU) {
		__save_init_fpu(tsk);
		stts();
	} else
		tsk->fpu_counter = 0;
}

static inline void __clear_fpu(struct task_struct *tsk)
{
	if (task_thread_info(tsk)->status & TS_USEDFPU) {
		tolerant_fwait();
		task_thread_info(tsk)->status &= ~TS_USEDFPU;
		stts();
	}
}

static inline void kernel_fpu_begin(void)
{
	struct thread_info *me = current_thread_info();
	preempt_disable();
	if (me->status & TS_USEDFPU)
		__save_init_fpu(me->task);
	else
		clts();
}

static inline void kernel_fpu_end(void)
{
	stts();
	preempt_enable();
}

#ifdef CONFIG_X86_64

static inline void save_init_fpu(struct task_struct *tsk)
{
	__save_init_fpu(tsk);
	stts();
}

#define unlazy_fpu	__unlazy_fpu
#define clear_fpu	__clear_fpu

#else  /* CONFIG_X86_32 */

/*
 * These disable preemption on their own and are safe
 */
static inline void save_init_fpu(struct task_struct *tsk)
{
	preempt_disable();
	__save_init_fpu(tsk);
	stts();
	preempt_enable();
}

static inline void unlazy_fpu(struct task_struct *tsk)
{
	preempt_disable();
	__unlazy_fpu(tsk);
	preempt_enable();
}

static inline void clear_fpu(struct task_struct *tsk)
{
	preempt_disable();
	__clear_fpu(tsk);
	preempt_enable();
}

#endif	/* CONFIG_X86_64 */

/*
 * i387 state interaction
 */
static inline unsigned short get_fpu_cwd(struct task_struct *tsk)
{
	if (cpu_has_fxsr) {
		return tsk->thread.xstate->fxsave.cwd;
	} else {
		return (unsigned short)tsk->thread.xstate->fsave.cwd;
	}
}

static inline unsigned short get_fpu_swd(struct task_struct *tsk)
{
	if (cpu_has_fxsr) {
		return tsk->thread.xstate->fxsave.swd;
	} else {
		return (unsigned short)tsk->thread.xstate->fsave.swd;
	}
}

static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
{
	if (cpu_has_xmm) {
		return tsk->thread.xstate->fxsave.mxcsr;
	} else {
		return MXCSR_DEFAULT;
	}
}

#endif /* ASM_X86__I387_H */
Commit	Line	Data
1eeaed76 RM	1	/*
	2	* Copyright (C) 1994 Linus Torvalds
	3	*
	4	* Pentium III FXSR, SSE support
	5	* General FPU state handling cleanups
	6	* Gareth Hughes <gareth@valinux.com>, May 2000
	7	* x86-64 work by Andi Kleen 2002
	8	*/
	9
77ef50a5 VN	10	#ifndef ASM_X86__I387_H
77ef50a5 VN	11	#define ASM_X86__I387_H
1eeaed76 RM	12
	13	#include <linux/sched.h>
	14	#include <linux/kernel_stat.h>
	15	#include <linux/regset.h>
92c37fa3	16	#include <asm/asm.h>
1eeaed76 RM	17	#include <asm/processor.h>
	18	#include <asm/sigcontext.h>
	19	#include <asm/user.h>
	20	#include <asm/uaccess.h>
	21
	22	extern void fpu_init(void);
1eeaed76	23	extern void mxcsr_feature_mask_init(void);
aa283f49	24	extern int init_fpu(struct task_struct *child);
1eeaed76	25	extern asmlinkage void math_state_restore(void);
61c4628b	26	extern void init_thread_xstate(void);
1eeaed76 RM	27
	28	extern user_regset_active_fn fpregs_active, xfpregs_active;
	29	extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get;
	30	extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set;
	31
	32	#ifdef CONFIG_IA32_EMULATION
	33	struct _fpstate_ia32;
	34	extern int save_i387_ia32(struct _fpstate_ia32 __user *buf);
	35	extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf);
	36	#endif
	37
	38	#ifdef CONFIG_X86_64
	39
	40	/* Ignore delayed exceptions from user space */
	41	static inline void tolerant_fwait(void)
	42	{
	43	asm volatile("1: fwait\n"
	44	"2:\n"
affe6637	45	_ASM_EXTABLE(1b, 2b));
1eeaed76 RM	46	}
	47
	48	static inline int restore_fpu_checking(struct i387_fxsave_struct *fx)
	49	{
	50	int err;
	51
	52	asm volatile("1: rex64/fxrstor (%[fx])\n\t"
	53	"2:\n"
	54	".section .fixup,\"ax\"\n"
	55	"3: movl $-1,%[err]\n"
	56	" jmp 2b\n"
	57	".previous\n"
affe6637	58	_ASM_EXTABLE(1b, 3b)
1eeaed76 RM	59	: [err] "=r" (err)
	60	#if 0 /* See comment in __save_init_fpu() below. */
	61	: [fx] "r" (fx), "m" (*fx), "0" (0));
	62	#else
	63	: [fx] "cdaSDb" (fx), "m" (*fx), "0" (0));
	64	#endif
1eeaed76 RM	65	return err;
	66	}
	67
	68	#define X87_FSW_ES (1 << 7) /* Exception Summary */
	69
	70	/* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
	71	is pending. Clear the x87 state here by setting it to fixed
	72	values. The kernel data segment can be sometimes 0 and sometimes
	73	new user value. Both should be ok.
	74	Use the PDA as safe address because it should be already in L1. */
	75	static inline void clear_fpu_state(struct i387_fxsave_struct *fx)
	76	{
	77	if (unlikely(fx->swd & X87_FSW_ES))
affe6637	78	asm volatile("fnclex");
1eeaed76	79	alternative_input(ASM_NOP8 ASM_NOP2,
affe6637 JP	80	" emms\n" /* clear stack tags */
	81	" fildl %%gs:0", /* load to clear state */
	82	X86_FEATURE_FXSAVE_LEAK);
1eeaed76 RM	83	}
	84
	85	static inline int save_i387_checking(struct i387_fxsave_struct __user *fx)
	86	{
	87	int err;
	88
	89	asm volatile("1: rex64/fxsave (%[fx])\n\t"
	90	"2:\n"
	91	".section .fixup,\"ax\"\n"
	92	"3: movl $-1,%[err]\n"
	93	" jmp 2b\n"
	94	".previous\n"
affe6637	95	_ASM_EXTABLE(1b, 3b)
1eeaed76 RM	96	: [err] "=r" (err), "=m" (*fx)
	97	#if 0 /* See comment in __fxsave_clear() below. */
	98	: [fx] "r" (fx), "0" (0));
	99	#else
	100	: [fx] "cdaSDb" (fx), "0" (0));
	101	#endif
affe6637 JP	102	if (unlikely(err) &&
affe6637 JP	103	__clear_user(fx, sizeof(struct i387_fxsave_struct)))
1eeaed76 RM	104	err = -EFAULT;
	105	/* No need to clear here because the caller clears USED_MATH */
	106	return err;
	107	}
	108
	109	static inline void __save_init_fpu(struct task_struct *tsk)
	110	{
	111	/* Using "rex64; fxsave %0" is broken because, if the memory operand
	112	uses any extended registers for addressing, a second REX prefix
	113	will be generated (to the assembler, rex64 followed by semicolon
	114	is a separate instruction), and hence the 64-bitness is lost. */
	115	#if 0
	116	/* Using "fxsaveq %0" would be the ideal choice, but is only supported
	117	starting with gas 2.16. */
	118	__asm__ __volatile__("fxsaveq %0"
61c4628b	119	: "=m" (tsk->thread.xstate->fxsave));
1eeaed76 RM	120	#elif 0
	121	/* Using, as a workaround, the properly prefixed form below isn't
	122	accepted by any binutils version so far released, complaining that
	123	the same type of prefix is used twice if an extended register is
	124	needed for addressing (fix submitted to mainline 2005-11-21). */
	125	__asm__ __volatile__("rex64/fxsave %0"
61c4628b	126	: "=m" (tsk->thread.xstate->fxsave));
1eeaed76 RM	127	#else
	128	/* This, however, we can work around by forcing the compiler to select
	129	an addressing mode that doesn't require extended registers. */
61c4628b SS	130	__asm__ __volatile__("rex64/fxsave (%1)"
	131	: "=m" (tsk->thread.xstate->fxsave)
	132	: "cdaSDb" (&tsk->thread.xstate->fxsave));
1eeaed76	133	#endif
61c4628b	134	clear_fpu_state(&tsk->thread.xstate->fxsave);
1eeaed76 RM	135	task_thread_info(tsk)->status &= ~TS_USEDFPU;
	136	}
	137
1eeaed76 RM	138	#else /* CONFIG_X86_32 */
1eeaed76 RM	139
e8a496ac SS	140	extern void finit(void);
e8a496ac SS	141
1eeaed76 RM	142	static inline void tolerant_fwait(void)
	143	{
	144	asm volatile("fnclex ; fwait");
	145	}
	146
	147	static inline void restore_fpu(struct task_struct *tsk)
	148	{
	149	/*
	150	* The "nop" is needed to make the instructions the same
	151	* length.
	152	*/
	153	alternative_input(
	154	"nop ; frstor %1",
	155	"fxrstor %1",
	156	X86_FEATURE_FXSR,
61c4628b	157	"m" (tsk->thread.xstate->fxsave));
1eeaed76 RM	158	}
	159
	160	/* We need a safe address that is cheap to find and that is already
	161	in L1 during context switch. The best choices are unfortunately
	162	different for UP and SMP */
	163	#ifdef CONFIG_SMP
	164	#define safe_address (__per_cpu_offset[0])
	165	#else
	166	#define safe_address (kstat_cpu(0).cpustat.user)
	167	#endif
	168
	169	/*
	170	* These must be called with preempt disabled
	171	*/
	172	static inline void __save_init_fpu(struct task_struct *tsk)
	173	{
	174	/* Use more nops than strictly needed in case the compiler
	175	varies code */
	176	alternative_input(
	177	"fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4,
	178	"fxsave %[fx]\n"
	179	"bt $7,%[fsw] ; jnc 1f ; fnclex\n1:",
	180	X86_FEATURE_FXSR,
61c4628b SS	181	[fx] "m" (tsk->thread.xstate->fxsave),
61c4628b SS	182	[fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory");
1eeaed76 RM	183	/* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
	184	is pending. Clear the x87 state here by setting it to fixed
	185	values. safe_address is a random variable that should be in L1 */
	186	alternative_input(
	187	GENERIC_NOP8 GENERIC_NOP2,
	188	"emms\n\t" /* clear stack tags */
	189	"fildl %[addr]", /* set F?P to defined value */
	190	X86_FEATURE_FXSAVE_LEAK,
	191	[addr] "m" (safe_address));
	192	task_thread_info(tsk)->status &= ~TS_USEDFPU;
	193	}
	194
	195	/*
	196	* Signal frame handlers...
	197	*/
	198	extern int save_i387(struct _fpstate __user *buf);
	199	extern int restore_i387(struct _fpstate __user *buf);
	200
	201	#endif /* CONFIG_X86_64 */
	202
	203	static inline void __unlazy_fpu(struct task_struct *tsk)
	204	{
	205	if (task_thread_info(tsk)->status & TS_USEDFPU) {
	206	__save_init_fpu(tsk);
	207	stts();
	208	} else
	209	tsk->fpu_counter = 0;
	210	}
	211
	212	static inline void __clear_fpu(struct task_struct *tsk)
	213	{
	214	if (task_thread_info(tsk)->status & TS_USEDFPU) {
	215	tolerant_fwait();
	216	task_thread_info(tsk)->status &= ~TS_USEDFPU;
	217	stts();
	218	}
	219	}
	220
	221	static inline void kernel_fpu_begin(void)
	222	{
	223	struct thread_info *me = current_thread_info();
	224	preempt_disable();
	225	if (me->status & TS_USEDFPU)
	226	__save_init_fpu(me->task);
	227	else
	228	clts();
	229	}
	230
	231	static inline void kernel_fpu_end(void)
	232	{
	233	stts();
	234	preempt_enable();
	235	}
	236
	237	#ifdef CONFIG_X86_64
	238
	239	static inline void save_init_fpu(struct task_struct *tsk)
	240	{
	241	__save_init_fpu(tsk);
	242	stts();
	243	}
	244
	245	#define unlazy_fpu __unlazy_fpu
	246	#define clear_fpu __clear_fpu
247
248	#else /* CONFIG_X86_32 */
249
250	/*
251	* These disable preemption on their own and are safe
252	*/
253	static inline void save_init_fpu(struct task_struct *tsk)
254	{
255	preempt_disable();
256	__save_init_fpu(tsk);
257	stts();
258	preempt_enable();
259	}
260
261	static inline void unlazy_fpu(struct task_struct *tsk)
262	{
263	preempt_disable();
264	__unlazy_fpu(tsk);
265	preempt_enable();
266	}
267
268	static inline void clear_fpu(struct task_struct *tsk)
269	{
270	preempt_disable();
271	__clear_fpu(tsk);
272	preempt_enable();
273	}
274
275	#endif /* CONFIG_X86_64 */
276
1eeaed76 RM	277	/*
	278	* i387 state interaction
	279	*/
	280	static inline unsigned short get_fpu_cwd(struct task_struct *tsk)
	281	{
	282	if (cpu_has_fxsr) {
61c4628b	283	return tsk->thread.xstate->fxsave.cwd;
1eeaed76	284	} else {
1679f271	285	return (unsigned short)tsk->thread.xstate->fsave.cwd;
1eeaed76 RM	286	}
	287	}
	288
	289	static inline unsigned short get_fpu_swd(struct task_struct *tsk)
	290	{
	291	if (cpu_has_fxsr) {
61c4628b	292	return tsk->thread.xstate->fxsave.swd;
1eeaed76	293	} else {
1679f271	294	return (unsigned short)tsk->thread.xstate->fsave.swd;
1eeaed76 RM	295	}
	296	}
	297
	298	static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
	299	{
	300	if (cpu_has_xmm) {
61c4628b	301	return tsk->thread.xstate->fxsave.mxcsr;
1eeaed76 RM	302	} else {
	303	return MXCSR_DEFAULT;
	304	}
	305	}
	306
77ef50a5	307	#endif /* ASM_X86__I387_H */