[deliverable/linux.git] / include / asm-x86 / i387.h

/*
 * Copyright (C) 1994 Linus Torvalds
 *
 * Pentium III FXSR, SSE support
 * General FPU state handling cleanups
 *	Gareth Hughes <gareth@valinux.com>, May 2000
 * x86-64 work by Andi Kleen 2002
 */

#ifndef _ASM_X86_I387_H
#define _ASM_X86_I387_H

#include <linux/sched.h>
#include <linux/kernel_stat.h>
#include <linux/regset.h>
#include <linux/hardirq.h>
#include <asm/asm.h>
#include <asm/processor.h>
#include <asm/sigcontext.h>
#include <asm/user.h>
#include <asm/uaccess.h>

extern void fpu_init(void);
extern void mxcsr_feature_mask_init(void);
extern int init_fpu(struct task_struct *child);
extern asmlinkage void math_state_restore(void);
extern void init_thread_xstate(void);

extern user_regset_active_fn fpregs_active, xfpregs_active;
extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get;
extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set;

#ifdef CONFIG_IA32_EMULATION
struct _fpstate_ia32;
extern int save_i387_ia32(struct _fpstate_ia32 __user *buf);
extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf);
#endif

#ifdef CONFIG_X86_64

/* Ignore delayed exceptions from user space */
static inline void tolerant_fwait(void)
{
	asm volatile("1: fwait\n"
		     "2:\n"
		     _ASM_EXTABLE(1b, 2b));
}

static inline int restore_fpu_checking(struct i387_fxsave_struct *fx)
{
	int err;

	asm volatile("1:  rex64/fxrstor (%[fx])\n\t"
		     "2:\n"
		     ".section .fixup,\"ax\"\n"
		     "3:  movl $-1,%[err]\n"
		     "    jmp  2b\n"
		     ".previous\n"
		     _ASM_EXTABLE(1b, 3b)
		     : [err] "=r" (err)
#if 0 /* See comment in __save_init_fpu() below. */
		     : [fx] "r" (fx), "m" (*fx), "0" (0));
#else
		     : [fx] "cdaSDb" (fx), "m" (*fx), "0" (0));
#endif
	return err;
}

#define X87_FSW_ES (1 << 7)	/* Exception Summary */

/* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
   is pending. Clear the x87 state here by setting it to fixed
   values. The kernel data segment can be sometimes 0 and sometimes
   new user value. Both should be ok.
   Use the PDA as safe address because it should be already in L1. */
static inline void clear_fpu_state(struct i387_fxsave_struct *fx)
{
	if (unlikely(fx->swd & X87_FSW_ES))
		asm volatile("fnclex");
	alternative_input(ASM_NOP8 ASM_NOP2,
			  "    emms\n"		/* clear stack tags */
			  "    fildl %%gs:0",	/* load to clear state */
			  X86_FEATURE_FXSAVE_LEAK);
}

static inline int save_i387_checking(struct i387_fxsave_struct __user *fx)
{
	int err;

	asm volatile("1:  rex64/fxsave (%[fx])\n\t"
		     "2:\n"
		     ".section .fixup,\"ax\"\n"
		     "3:  movl $-1,%[err]\n"
		     "    jmp  2b\n"
		     ".previous\n"
		     _ASM_EXTABLE(1b, 3b)
		     : [err] "=r" (err), "=m" (*fx)
#if 0 /* See comment in __fxsave_clear() below. */
		     : [fx] "r" (fx), "0" (0));
#else
		     : [fx] "cdaSDb" (fx), "0" (0));
#endif
	if (unlikely(err) &&
	    __clear_user(fx, sizeof(struct i387_fxsave_struct)))
		err = -EFAULT;
	/* No need to clear here because the caller clears USED_MATH */
	return err;
}

static inline void __save_init_fpu(struct task_struct *tsk)
{
	/* Using "rex64; fxsave %0" is broken because, if the memory operand
	   uses any extended registers for addressing, a second REX prefix
	   will be generated (to the assembler, rex64 followed by semicolon
	   is a separate instruction), and hence the 64-bitness is lost. */
#if 0
	/* Using "fxsaveq %0" would be the ideal choice, but is only supported
	   starting with gas 2.16. */
	__asm__ __volatile__("fxsaveq %0"
			     : "=m" (tsk->thread.xstate->fxsave));
#elif 0
	/* Using, as a workaround, the properly prefixed form below isn't
	   accepted by any binutils version so far released, complaining that
	   the same type of prefix is used twice if an extended register is
	   needed for addressing (fix submitted to mainline 2005-11-21). */
	__asm__ __volatile__("rex64/fxsave %0"
			     : "=m" (tsk->thread.xstate->fxsave));
#else
	/* This, however, we can work around by forcing the compiler to select
	   an addressing mode that doesn't require extended registers. */
	__asm__ __volatile__("rex64/fxsave (%1)"
			     : "=m" (tsk->thread.xstate->fxsave)
			     : "cdaSDb" (&tsk->thread.xstate->fxsave));
#endif
	clear_fpu_state(&tsk->thread.xstate->fxsave);
	task_thread_info(tsk)->status &= ~TS_USEDFPU;
}

#else  /* CONFIG_X86_32 */

extern void finit(void);

static inline void tolerant_fwait(void)
{
	asm volatile("fnclex ; fwait");
}

static inline void restore_fpu(struct task_struct *tsk)
{
	/*
	 * The "nop" is needed to make the instructions the same
	 * length.
	 */
	alternative_input(
		"nop ; frstor %1",
		"fxrstor %1",
		X86_FEATURE_FXSR,
		"m" (tsk->thread.xstate->fxsave));
}

/* We need a safe address that is cheap to find and that is already
   in L1 during context switch. The best choices are unfortunately
   different for UP and SMP */
#ifdef CONFIG_SMP
#define safe_address (__per_cpu_offset[0])
#else
#define safe_address (kstat_cpu(0).cpustat.user)
#endif

/*
 * These must be called with preempt disabled
 */
static inline void __save_init_fpu(struct task_struct *tsk)
{
	/* Use more nops than strictly needed in case the compiler
	   varies code */
	alternative_input(
		"fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4,
		"fxsave %[fx]\n"
		"bt $7,%[fsw] ; jnc 1f ; fnclex\n1:",
		X86_FEATURE_FXSR,
		[fx] "m" (tsk->thread.xstate->fxsave),
		[fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory");
	/* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
	   is pending.  Clear the x87 state here by setting it to fixed
	   values. safe_address is a random variable that should be in L1 */
	alternative_input(
		GENERIC_NOP8 GENERIC_NOP2,
		"emms\n\t"	  	/* clear stack tags */
		"fildl %[addr]", 	/* set F?P to defined value */
		X86_FEATURE_FXSAVE_LEAK,
		[addr] "m" (safe_address));
	task_thread_info(tsk)->status &= ~TS_USEDFPU;
}

/*
 * Signal frame handlers...
 */
extern int save_i387(struct _fpstate __user *buf);
extern int restore_i387(struct _fpstate __user *buf);

#endif	/* CONFIG_X86_64 */

static inline void __unlazy_fpu(struct task_struct *tsk)
{
	if (task_thread_info(tsk)->status & TS_USEDFPU) {
		__save_init_fpu(tsk);
		stts();
	} else
		tsk->fpu_counter = 0;
}

static inline void __clear_fpu(struct task_struct *tsk)
{
	if (task_thread_info(tsk)->status & TS_USEDFPU) {
		tolerant_fwait();
		task_thread_info(tsk)->status &= ~TS_USEDFPU;
		stts();
	}
}

static inline void kernel_fpu_begin(void)
{
	struct thread_info *me = current_thread_info();
	preempt_disable();
	if (me->status & TS_USEDFPU)
		__save_init_fpu(me->task);
	else
		clts();
}

static inline void kernel_fpu_end(void)
{
	stts();
	preempt_enable();
}

/*
 * Some instructions like VIA's padlock instructions generate a spurious
 * DNA fault but don't modify SSE registers. And these instructions
 * get used from interrupt context aswell. To prevent these kernel instructions
 * in interrupt context interact wrongly with other user/kernel fpu usage, we
 * should use them only in the context of irq_ts_save/restore()
 */
static inline int irq_ts_save(void)
{
	/*
	 * If we are in process context, we are ok to take a spurious DNA fault.
	 * Otherwise, doing clts() in process context require pre-emption to
	 * be disabled or some heavy lifting like kernel_fpu_begin()
	 */
	if (!in_interrupt())
		return 0;

	if (read_cr0() & X86_CR0_TS) {
		clts();
		return 1;
	}

	return 0;
}

static inline void irq_ts_restore(int TS_state)
{
	if (TS_state)
		stts();
}

#ifdef CONFIG_X86_64

static inline void save_init_fpu(struct task_struct *tsk)
{
	__save_init_fpu(tsk);
	stts();
}

#define unlazy_fpu	__unlazy_fpu
#define clear_fpu	__clear_fpu

#else  /* CONFIG_X86_32 */

/*
 * These disable preemption on their own and are safe
 */
static inline void save_init_fpu(struct task_struct *tsk)
{
	preempt_disable();
	__save_init_fpu(tsk);
	stts();
	preempt_enable();
}

static inline void unlazy_fpu(struct task_struct *tsk)
{
	preempt_disable();
	__unlazy_fpu(tsk);
	preempt_enable();
}

static inline void clear_fpu(struct task_struct *tsk)
{
	preempt_disable();
	__clear_fpu(tsk);
	preempt_enable();
}

#endif	/* CONFIG_X86_64 */

/*
 * i387 state interaction
 */
static inline unsigned short get_fpu_cwd(struct task_struct *tsk)
{
	if (cpu_has_fxsr) {
		return tsk->thread.xstate->fxsave.cwd;
	} else {
		return (unsigned short)tsk->thread.xstate->fsave.cwd;
	}
}

static inline unsigned short get_fpu_swd(struct task_struct *tsk)
{
	if (cpu_has_fxsr) {
		return tsk->thread.xstate->fxsave.swd;
	} else {
		return (unsigned short)tsk->thread.xstate->fsave.swd;
	}
}

static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
{
	if (cpu_has_xmm) {
		return tsk->thread.xstate->fxsave.mxcsr;
	} else {
		return MXCSR_DEFAULT;
	}
}

#endif	/* _ASM_X86_I387_H */
Commit	Line	Data
1eeaed76 RM	1	/*
	2	* Copyright (C) 1994 Linus Torvalds
	3	*
	4	* Pentium III FXSR, SSE support
	5	* General FPU state handling cleanups
	6	* Gareth Hughes <gareth@valinux.com>, May 2000
	7	* x86-64 work by Andi Kleen 2002
	8	*/
	9
	10	#ifndef _ASM_X86_I387_H
	11	#define _ASM_X86_I387_H
	12
	13	#include <linux/sched.h>
	14	#include <linux/kernel_stat.h>
	15	#include <linux/regset.h>
e4914012	16	#include <linux/hardirq.h>
92c37fa3	17	#include <asm/asm.h>
1eeaed76 RM	18	#include <asm/processor.h>
	19	#include <asm/sigcontext.h>
	20	#include <asm/user.h>
	21	#include <asm/uaccess.h>
	22
	23	extern void fpu_init(void);
1eeaed76	24	extern void mxcsr_feature_mask_init(void);
aa283f49	25	extern int init_fpu(struct task_struct *child);
1eeaed76	26	extern asmlinkage void math_state_restore(void);
61c4628b	27	extern void init_thread_xstate(void);
1eeaed76 RM	28
	29	extern user_regset_active_fn fpregs_active, xfpregs_active;
	30	extern user_regset_get_fn fpregs_get, xfpregs_get, fpregs_soft_get;
	31	extern user_regset_set_fn fpregs_set, xfpregs_set, fpregs_soft_set;
	32
	33	#ifdef CONFIG_IA32_EMULATION
	34	struct _fpstate_ia32;
	35	extern int save_i387_ia32(struct _fpstate_ia32 __user *buf);
	36	extern int restore_i387_ia32(struct _fpstate_ia32 __user *buf);
	37	#endif
	38
	39	#ifdef CONFIG_X86_64
	40
	41	/* Ignore delayed exceptions from user space */
	42	static inline void tolerant_fwait(void)
	43	{
	44	asm volatile("1: fwait\n"
	45	"2:\n"
affe6637	46	_ASM_EXTABLE(1b, 2b));
1eeaed76 RM	47	}
	48
	49	static inline int restore_fpu_checking(struct i387_fxsave_struct *fx)
	50	{
	51	int err;
	52
	53	asm volatile("1: rex64/fxrstor (%[fx])\n\t"
	54	"2:\n"
	55	".section .fixup,\"ax\"\n"
	56	"3: movl $-1,%[err]\n"
	57	" jmp 2b\n"
	58	".previous\n"
affe6637	59	_ASM_EXTABLE(1b, 3b)
1eeaed76 RM	60	: [err] "=r" (err)
	61	#if 0 /* See comment in __save_init_fpu() below. */
	62	: [fx] "r" (fx), "m" (*fx), "0" (0));
	63	#else
	64	: [fx] "cdaSDb" (fx), "m" (*fx), "0" (0));
	65	#endif
1eeaed76 RM	66	return err;
	67	}
	68
	69	#define X87_FSW_ES (1 << 7) /* Exception Summary */
	70
	71	/* AMD CPUs don't save/restore FDP/FIP/FOP unless an exception
	72	is pending. Clear the x87 state here by setting it to fixed
	73	values. The kernel data segment can be sometimes 0 and sometimes
	74	new user value. Both should be ok.
	75	Use the PDA as safe address because it should be already in L1. */
	76	static inline void clear_fpu_state(struct i387_fxsave_struct *fx)
	77	{
	78	if (unlikely(fx->swd & X87_FSW_ES))
affe6637	79	asm volatile("fnclex");
1eeaed76	80	alternative_input(ASM_NOP8 ASM_NOP2,
affe6637 JP	81	" emms\n" /* clear stack tags */
	82	" fildl %%gs:0", /* load to clear state */
	83	X86_FEATURE_FXSAVE_LEAK);
1eeaed76 RM	84	}
	85
	86	static inline int save_i387_checking(struct i387_fxsave_struct __user *fx)
	87	{
	88	int err;
	89
	90	asm volatile("1: rex64/fxsave (%[fx])\n\t"
	91	"2:\n"
	92	".section .fixup,\"ax\"\n"
	93	"3: movl $-1,%[err]\n"
	94	" jmp 2b\n"
	95	".previous\n"
affe6637	96	_ASM_EXTABLE(1b, 3b)
1eeaed76 RM	97	: [err] "=r" (err), "=m" (*fx)
	98	#if 0 /* See comment in __fxsave_clear() below. */
	99	: [fx] "r" (fx), "0" (0));
	100	#else
	101	: [fx] "cdaSDb" (fx), "0" (0));
	102	#endif
affe6637 JP	103	if (unlikely(err) &&
affe6637 JP	104	__clear_user(fx, sizeof(struct i387_fxsave_struct)))
1eeaed76 RM	105	err = -EFAULT;
	106	/* No need to clear here because the caller clears USED_MATH */
	107	return err;
	108	}
	109
	110	static inline void __save_init_fpu(struct task_struct *tsk)
	111	{
	112	/* Using "rex64; fxsave %0" is broken because, if the memory operand
	113	uses any extended registers for addressing, a second REX prefix
	114	will be generated (to the assembler, rex64 followed by semicolon
	115	is a separate instruction), and hence the 64-bitness is lost. */
	116	#if 0
	117	/* Using "fxsaveq %0" would be the ideal choice, but is only supported
	118	starting with gas 2.16. */
	119	__asm__ __volatile__("fxsaveq %0"
61c4628b	120	: "=m" (tsk->thread.xstate->fxsave));
1eeaed76 RM	121	#elif 0
	122	/* Using, as a workaround, the properly prefixed form below isn't
	123	accepted by any binutils version so far released, complaining that
	124	the same type of prefix is used twice if an extended register is
	125	needed for addressing (fix submitted to mainline 2005-11-21). */
	126	__asm__ __volatile__("rex64/fxsave %0"
61c4628b	127	: "=m" (tsk->thread.xstate->fxsave));
1eeaed76 RM	128	#else
	129	/* This, however, we can work around by forcing the compiler to select
	130	an addressing mode that doesn't require extended registers. */
61c4628b SS	131	__asm__ __volatile__("rex64/fxsave (%1)"
	132	: "=m" (tsk->thread.xstate->fxsave)
	133	: "cdaSDb" (&tsk->thread.xstate->fxsave));
1eeaed76	134	#endif
61c4628b	135	clear_fpu_state(&tsk->thread.xstate->fxsave);
1eeaed76 RM	136	task_thread_info(tsk)->status &= ~TS_USEDFPU;
	137	}
	138
1eeaed76 RM	139	#else /* CONFIG_X86_32 */
1eeaed76 RM	140
e8a496ac SS	141	extern void finit(void);
e8a496ac SS	142
1eeaed76 RM	143	static inline void tolerant_fwait(void)
	144	{
	145	asm volatile("fnclex ; fwait");
	146	}
	147
	148	static inline void restore_fpu(struct task_struct *tsk)
	149	{
	150	/*
	151	* The "nop" is needed to make the instructions the same
	152	* length.
	153	*/
	154	alternative_input(
	155	"nop ; frstor %1",
	156	"fxrstor %1",
	157	X86_FEATURE_FXSR,
61c4628b	158	"m" (tsk->thread.xstate->fxsave));
1eeaed76 RM	159	}
	160
	161	/* We need a safe address that is cheap to find and that is already
	162	in L1 during context switch. The best choices are unfortunately
	163	different for UP and SMP */
	164	#ifdef CONFIG_SMP
	165	#define safe_address (__per_cpu_offset[0])
	166	#else
	167	#define safe_address (kstat_cpu(0).cpustat.user)
	168	#endif
	169
	170	/*
	171	* These must be called with preempt disabled
	172	*/
	173	static inline void __save_init_fpu(struct task_struct *tsk)
	174	{
	175	/* Use more nops than strictly needed in case the compiler
	176	varies code */
	177	alternative_input(
	178	"fnsave %[fx] ;fwait;" GENERIC_NOP8 GENERIC_NOP4,
	179	"fxsave %[fx]\n"
	180	"bt $7,%[fsw] ; jnc 1f ; fnclex\n1:",
	181	X86_FEATURE_FXSR,
61c4628b SS	182	[fx] "m" (tsk->thread.xstate->fxsave),
61c4628b SS	183	[fsw] "m" (tsk->thread.xstate->fxsave.swd) : "memory");
1eeaed76 RM	184	/* AMD K7/K8 CPUs don't save/restore FDP/FIP/FOP unless an exception
	185	is pending. Clear the x87 state here by setting it to fixed
	186	values. safe_address is a random variable that should be in L1 */
	187	alternative_input(
	188	GENERIC_NOP8 GENERIC_NOP2,
	189	"emms\n\t" /* clear stack tags */
	190	"fildl %[addr]", /* set F?P to defined value */
	191	X86_FEATURE_FXSAVE_LEAK,
	192	[addr] "m" (safe_address));
	193	task_thread_info(tsk)->status &= ~TS_USEDFPU;
	194	}
	195
	196	/*
	197	* Signal frame handlers...
	198	*/
	199	extern int save_i387(struct _fpstate __user *buf);
	200	extern int restore_i387(struct _fpstate __user *buf);
	201
	202	#endif /* CONFIG_X86_64 */
	203
	204	static inline void __unlazy_fpu(struct task_struct *tsk)
	205	{
	206	if (task_thread_info(tsk)->status & TS_USEDFPU) {
	207	__save_init_fpu(tsk);
	208	stts();
	209	} else
	210	tsk->fpu_counter = 0;
	211	}
	212
	213	static inline void __clear_fpu(struct task_struct *tsk)
	214	{
	215	if (task_thread_info(tsk)->status & TS_USEDFPU) {
	216	tolerant_fwait();
	217	task_thread_info(tsk)->status &= ~TS_USEDFPU;
	218	stts();
	219	}
	220	}
	221
	222	static inline void kernel_fpu_begin(void)
	223	{
	224	struct thread_info *me = current_thread_info();
	225	preempt_disable();
	226	if (me->status & TS_USEDFPU)
	227	__save_init_fpu(me->task);
	228	else
	229	clts();
	230	}
	231
	232	static inline void kernel_fpu_end(void)
	233	{
	234	stts();
	235	preempt_enable();
	236	}
	237
e4914012 SS	238	/*
	239	* Some instructions like VIA's padlock instructions generate a spurious
	240	* DNA fault but don't modify SSE registers. And these instructions
	241	* get used from interrupt context aswell. To prevent these kernel instructions
	242	* in interrupt context interact wrongly with other user/kernel fpu usage, we
	243	* should use them only in the context of irq_ts_save/restore()
	244	*/
	245	static inline int irq_ts_save(void)
	246	{
	247	/*
	248	* If we are in process context, we are ok to take a spurious DNA fault.
	249	* Otherwise, doing clts() in process context require pre-emption to
	250	* be disabled or some heavy lifting like kernel_fpu_begin()
	251	*/
	252	if (!in_interrupt())
	253	return 0;
	254
	255	if (read_cr0() & X86_CR0_TS) {
	256	clts();
	257	return 1;
	258	}
	259
	260	return 0;
	261	}
	262
	263	static inline void irq_ts_restore(int TS_state)
	264	{
	265	if (TS_state)
	266	stts();
	267	}
	268
1eeaed76 RM	269	#ifdef CONFIG_X86_64
	270
	271	static inline void save_init_fpu(struct task_struct *tsk)
	272	{
	273	__save_init_fpu(tsk);
	274	stts();
	275	}
	276
	277	#define unlazy_fpu __unlazy_fpu
	278	#define clear_fpu __clear_fpu
	279
	280	#else /* CONFIG_X86_32 */
	281
	282	/*
	283	* These disable preemption on their own and are safe
	284	*/
	285	static inline void save_init_fpu(struct task_struct *tsk)
	286	{
	287	preempt_disable();
	288	__save_init_fpu(tsk);
	289	stts();
	290	preempt_enable();
	291	}
	292
	293	static inline void unlazy_fpu(struct task_struct *tsk)
	294	{
	295	preempt_disable();
	296	__unlazy_fpu(tsk);
	297	preempt_enable();
	298	}
	299
	300	static inline void clear_fpu(struct task_struct *tsk)
	301	{
	302	preempt_disable();
	303	__clear_fpu(tsk);
	304	preempt_enable();
	305	}
	306
	307	#endif /* CONFIG_X86_64 */
	308
1eeaed76 RM	309	/*
	310	* i387 state interaction
	311	*/
	312	static inline unsigned short get_fpu_cwd(struct task_struct *tsk)
	313	{
	314	if (cpu_has_fxsr) {
61c4628b	315	return tsk->thread.xstate->fxsave.cwd;
1eeaed76	316	} else {
1679f271	317	return (unsigned short)tsk->thread.xstate->fsave.cwd;
1eeaed76 RM	318	}
	319	}
	320
	321	static inline unsigned short get_fpu_swd(struct task_struct *tsk)
	322	{
	323	if (cpu_has_fxsr) {
61c4628b	324	return tsk->thread.xstate->fxsave.swd;
1eeaed76	325	} else {
1679f271	326	return (unsigned short)tsk->thread.xstate->fsave.swd;
1eeaed76 RM	327	}
	328	}
	329
	330	static inline unsigned short get_fpu_mxcsr(struct task_struct *tsk)
	331	{
	332	if (cpu_has_xmm) {
61c4628b	333	return tsk->thread.xstate->fxsave.mxcsr;
1eeaed76 RM	334	} else {
	335	return MXCSR_DEFAULT;
	336	}
	337	}
	338
	339	#endif /* _ASM_X86_I387_H */