[deliverable/linux.git] / arch / x86 / include / asm / uaccess_64.h

#ifndef _ASM_X86_UACCESS_64_H
#define _ASM_X86_UACCESS_64_H

/*
 * User space memory access functions
 */
#include <linux/compiler.h>
#include <linux/errno.h>
#include <linux/prefetch.h>
#include <linux/lockdep.h>
#include <asm/alternative.h>
#include <asm/cpufeature.h>
#include <asm/page.h>

/*
 * Copy To/From Userspace
 */

/* Handles exceptions in both to and from, but doesn't do access_ok */
__must_check unsigned long
copy_user_generic_string(void *to, const void *from, unsigned len);
__must_check unsigned long
copy_user_generic_unrolled(void *to, const void *from, unsigned len);

static __always_inline __must_check unsigned long
copy_user_generic(void *to, const void *from, unsigned len)
{
	unsigned ret;

	alternative_call(copy_user_generic_unrolled,
			 copy_user_generic_string,
			 X86_FEATURE_REP_GOOD,
			 ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
				     "=d" (len)),
			 "1" (to), "2" (from), "3" (len)
			 : "memory", "rcx", "r8", "r9", "r10", "r11");
	return ret;
}

__must_check unsigned long
_copy_to_user(void __user *to, const void *from, unsigned len);
__must_check unsigned long
_copy_from_user(void *to, const void __user *from, unsigned len);
__must_check unsigned long
copy_in_user(void __user *to, const void __user *from, unsigned len);

static inline unsigned long __must_check copy_from_user(void *to,
					  const void __user *from,
					  unsigned long n)
{
	int sz = __compiletime_object_size(to);

	might_fault();
	if (likely(sz == -1 || sz >= n))
		n = _copy_from_user(to, from, n);
#ifdef CONFIG_DEBUG_VM
	else
		WARN(1, "Buffer overflow detected!\n");
#endif
	return n;
}

static __always_inline __must_check
int copy_to_user(void __user *dst, const void *src, unsigned size)
{
	might_fault();

	return _copy_to_user(dst, src, size);
}

static __always_inline __must_check
int __copy_from_user(void *dst, const void __user *src, unsigned size)
{
	int ret = 0;

	might_fault();
	if (!__builtin_constant_p(size))
		return copy_user_generic(dst, (__force void *)src, size);
	switch (size) {
	case 1:__get_user_asm(*(u8 *)dst, (u8 __user *)src,
			      ret, "b", "b", "=q", 1);
		return ret;
	case 2:__get_user_asm(*(u16 *)dst, (u16 __user *)src,
			      ret, "w", "w", "=r", 2);
		return ret;
	case 4:__get_user_asm(*(u32 *)dst, (u32 __user *)src,
			      ret, "l", "k", "=r", 4);
		return ret;
	case 8:__get_user_asm(*(u64 *)dst, (u64 __user *)src,
			      ret, "q", "", "=r", 8);
		return ret;
	case 10:
		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
			       ret, "q", "", "=r", 10);
		if (unlikely(ret))
			return ret;
		__get_user_asm(*(u16 *)(8 + (char *)dst),
			       (u16 __user *)(8 + (char __user *)src),
			       ret, "w", "w", "=r", 2);
		return ret;
	case 16:
		__get_user_asm(*(u64 *)dst, (u64 __user *)src,
			       ret, "q", "", "=r", 16);
		if (unlikely(ret))
			return ret;
		__get_user_asm(*(u64 *)(8 + (char *)dst),
			       (u64 __user *)(8 + (char __user *)src),
			       ret, "q", "", "=r", 8);
		return ret;
	default:
		return copy_user_generic(dst, (__force void *)src, size);
	}
}

static __always_inline __must_check
int __copy_to_user(void __user *dst, const void *src, unsigned size)
{
	int ret = 0;

	might_fault();
	if (!__builtin_constant_p(size))
		return copy_user_generic((__force void *)dst, src, size);
	switch (size) {
	case 1:__put_user_asm(*(u8 *)src, (u8 __user *)dst,
			      ret, "b", "b", "iq", 1);
		return ret;
	case 2:__put_user_asm(*(u16 *)src, (u16 __user *)dst,
			      ret, "w", "w", "ir", 2);
		return ret;
	case 4:__put_user_asm(*(u32 *)src, (u32 __user *)dst,
			      ret, "l", "k", "ir", 4);
		return ret;
	case 8:__put_user_asm(*(u64 *)src, (u64 __user *)dst,
			      ret, "q", "", "er", 8);
		return ret;
	case 10:
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
			       ret, "q", "", "er", 10);
		if (unlikely(ret))
			return ret;
		asm("":::"memory");
		__put_user_asm(4[(u16 *)src], 4 + (u16 __user *)dst,
			       ret, "w", "w", "ir", 2);
		return ret;
	case 16:
		__put_user_asm(*(u64 *)src, (u64 __user *)dst,
			       ret, "q", "", "er", 16);
		if (unlikely(ret))
			return ret;
		asm("":::"memory");
		__put_user_asm(1[(u64 *)src], 1 + (u64 __user *)dst,
			       ret, "q", "", "er", 8);
		return ret;
	default:
		return copy_user_generic((__force void *)dst, src, size);
	}
}

static __always_inline __must_check
int __copy_in_user(void __user *dst, const void __user *src, unsigned size)
{
	int ret = 0;

	might_fault();
	if (!__builtin_constant_p(size))
		return copy_user_generic((__force void *)dst,
					 (__force void *)src, size);
	switch (size) {
	case 1: {
		u8 tmp;
		__get_user_asm(tmp, (u8 __user *)src,
			       ret, "b", "b", "=q", 1);
		if (likely(!ret))
			__put_user_asm(tmp, (u8 __user *)dst,
				       ret, "b", "b", "iq", 1);
		return ret;
	}
	case 2: {
		u16 tmp;
		__get_user_asm(tmp, (u16 __user *)src,
			       ret, "w", "w", "=r", 2);
		if (likely(!ret))
			__put_user_asm(tmp, (u16 __user *)dst,
				       ret, "w", "w", "ir", 2);
		return ret;
	}

	case 4: {
		u32 tmp;
		__get_user_asm(tmp, (u32 __user *)src,
			       ret, "l", "k", "=r", 4);
		if (likely(!ret))
			__put_user_asm(tmp, (u32 __user *)dst,
				       ret, "l", "k", "ir", 4);
		return ret;
	}
	case 8: {
		u64 tmp;
		__get_user_asm(tmp, (u64 __user *)src,
			       ret, "q", "", "=r", 8);
		if (likely(!ret))
			__put_user_asm(tmp, (u64 __user *)dst,
				       ret, "q", "", "er", 8);
		return ret;
	}
	default:
		return copy_user_generic((__force void *)dst,
					 (__force void *)src, size);
	}
}

__must_check long
strncpy_from_user(char *dst, const char __user *src, long count);
__must_check long
__strncpy_from_user(char *dst, const char __user *src, long count);
__must_check long strnlen_user(const char __user *str, long n);
__must_check long __strnlen_user(const char __user *str, long n);
__must_check long strlen_user(const char __user *str);
__must_check unsigned long clear_user(void __user *mem, unsigned long len);
__must_check unsigned long __clear_user(void __user *mem, unsigned long len);

static __must_check __always_inline int
__copy_from_user_inatomic(void *dst, const void __user *src, unsigned size)
{
	return copy_user_generic(dst, (__force const void *)src, size);
}

static __must_check __always_inline int
__copy_to_user_inatomic(void __user *dst, const void *src, unsigned size)
{
	return copy_user_generic((__force void *)dst, src, size);
}

extern long __copy_user_nocache(void *dst, const void __user *src,
				unsigned size, int zerorest);

static inline int
__copy_from_user_nocache(void *dst, const void __user *src, unsigned size)
{
	might_sleep();
	return __copy_user_nocache(dst, src, size, 1);
}

static inline int
__copy_from_user_inatomic_nocache(void *dst, const void __user *src,
				  unsigned size)
{
	return __copy_user_nocache(dst, src, size, 0);
}

unsigned long
copy_user_handle_tail(char *to, char *from, unsigned len, unsigned zerorest);

#endif /* _ASM_X86_UACCESS_64_H */
Commit	Line	Data
1965aae3 PA	1	#ifndef _ASM_X86_UACCESS_64_H
1965aae3 PA	2	#define _ASM_X86_UACCESS_64_H
1da177e4 LT	3
	4	/*
	5	* User space memory access functions
	6	*/
1da177e4 LT	7	#include <linux/compiler.h>
1da177e4 LT	8	#include <linux/errno.h>
1da177e4	9	#include <linux/prefetch.h>
16dbc6c9	10	#include <linux/lockdep.h>
1b1d9258 JB	11	#include <asm/alternative.h>
1b1d9258 JB	12	#include <asm/cpufeature.h>
1da177e4 LT	13	#include <asm/page.h>
1da177e4 LT	14
1da177e4 LT	15	/*
	16	* Copy To/From Userspace
	17	*/
	18
	19	/* Handles exceptions in both to and from, but doesn't do access_ok */
95912008	20	__must_check unsigned long
1b1d9258 JB	21	copy_user_generic_string(void to, const void from, unsigned len);
	22	__must_check unsigned long
	23	copy_user_generic_unrolled(void to, const void from, unsigned len);
	24
	25	static __always_inline __must_check unsigned long
	26	copy_user_generic(void to, const void from, unsigned len)
	27	{
	28	unsigned ret;
	29
	30	alternative_call(copy_user_generic_unrolled,
	31	copy_user_generic_string,
	32	X86_FEATURE_REP_GOOD,
	33	ASM_OUTPUT2("=a" (ret), "=D" (to), "=S" (from),
	34	"=d" (len)),
	35	"1" (to), "2" (from), "3" (len)
	36	: "memory", "rcx", "r8", "r9", "r10", "r11");
	37	return ret;
	38	}
95912008 AK	39
95912008 AK	40	__must_check unsigned long
3c93ca00	41	_copy_to_user(void __user to, const void from, unsigned len);
95912008	42	__must_check unsigned long
9f0cf4ad	43	_copy_from_user(void to, const void __user from, unsigned len);
95912008 AK	44	__must_check unsigned long
	45	copy_in_user(void __user to, const void __user from, unsigned len);
	46
9f0cf4ad AV	47	static inline unsigned long __must_check copy_from_user(void *to,
	48	const void __user *from,
	49	unsigned long n)
	50	{
	51	int sz = __compiletime_object_size(to);
9f0cf4ad	52
3c93ca00	53	might_fault();
9f0cf4ad	54	if (likely(sz == -1 \|\| sz >= n))
409d02ef	55	n = _copy_from_user(to, from, n);
9f0cf4ad AV	56	#ifdef CONFIG_DEBUG_VM
	57	else
	58	WARN(1, "Buffer overflow detected!\n");
	59	#endif
409d02ef	60	return n;
9f0cf4ad AV	61	}
9f0cf4ad AV	62
3c93ca00 FW	63	static __always_inline __must_check
	64	int copy_to_user(void __user dst, const void src, unsigned size)
	65	{
	66	might_fault();
	67
	68	return _copy_to_user(dst, src, size);
	69	}
9f0cf4ad	70
95912008 AK	71	static __always_inline __must_check
95912008 AK	72	int __copy_from_user(void dst, const void __user src, unsigned size)
b896313e	73	{
383d079b	74	int ret = 0;
c10d38dd	75
3ee1afa3	76	might_fault();
1da177e4	77	if (!__builtin_constant_p(size))
b896313e JP	78	return copy_user_generic(dst, (__force void *)src, size);
	79	switch (size) {
	80	case 1:__get_user_asm((u8 )dst, (u8 __user *)src,
	81	ret, "b", "b", "=q", 1);
1da177e4	82	return ret;
b896313e JP	83	case 2:__get_user_asm((u16 )dst, (u16 __user *)src,
b896313e JP	84	ret, "w", "w", "=r", 2);
1da177e4	85	return ret;
b896313e JP	86	case 4:__get_user_asm((u32 )dst, (u32 __user *)src,
	87	ret, "l", "k", "=r", 4);
	88	return ret;
	89	case 8:__get_user_asm((u64 )dst, (u64 __user *)src,
	90	ret, "q", "", "=r", 8);
1da177e4	91	return ret;
1da177e4	92	case 10:
b896313e	93	__get_user_asm((u64 )dst, (u64 __user *)src,
20a4a236	94	ret, "q", "", "=r", 10);
b896313e JP	95	if (unlikely(ret))
	96	return ret;
	97	__get_user_asm((u16 )(8 + (char *)dst),
	98	(u16 __user )(8 + (char __user )src),
	99	ret, "w", "w", "=r", 2);
	100	return ret;
1da177e4	101	case 16:
b896313e JP	102	__get_user_asm((u64 )dst, (u64 __user *)src,
	103	ret, "q", "", "=r", 16);
	104	if (unlikely(ret))
	105	return ret;
	106	__get_user_asm((u64 )(8 + (char *)dst),
	107	(u64 __user )(8 + (char __user )src),
	108	ret, "q", "", "=r", 8);
	109	return ret;
1da177e4	110	default:
b896313e	111	return copy_user_generic(dst, (__force void *)src, size);
1da177e4	112	}
b896313e	113	}
1da177e4	114
95912008 AK	115	static __always_inline __must_check
95912008 AK	116	int __copy_to_user(void __user dst, const void src, unsigned size)
b896313e	117	{
383d079b	118	int ret = 0;
c10d38dd	119
3ee1afa3	120	might_fault();
1da177e4	121	if (!__builtin_constant_p(size))
b896313e JP	122	return copy_user_generic((__force void *)dst, src, size);
	123	switch (size) {
	124	case 1:__put_user_asm((u8 )src, (u8 __user *)dst,
	125	ret, "b", "b", "iq", 1);
1da177e4	126	return ret;
b896313e JP	127	case 2:__put_user_asm((u16 )src, (u16 __user *)dst,
b896313e JP	128	ret, "w", "w", "ir", 2);
1da177e4	129	return ret;
b896313e JP	130	case 4:__put_user_asm((u32 )src, (u32 __user *)dst,
	131	ret, "l", "k", "ir", 4);
	132	return ret;
	133	case 8:__put_user_asm((u64 )src, (u64 __user *)dst,
155b7352	134	ret, "q", "", "er", 8);
1da177e4	135	return ret;
1da177e4	136	case 10:
b896313e	137	__put_user_asm((u64 )src, (u64 __user *)dst,
155b7352	138	ret, "q", "", "er", 10);
b896313e JP	139	if (unlikely(ret))
b896313e JP	140	return ret;
1da177e4	141	asm("":::"memory");
b896313e JP	142	__put_user_asm(4[(u16 )src], 4 + (u16 __user )dst,
	143	ret, "w", "w", "ir", 2);
	144	return ret;
1da177e4	145	case 16:
b896313e	146	__put_user_asm((u64 )src, (u64 __user *)dst,
155b7352	147	ret, "q", "", "er", 16);
b896313e JP	148	if (unlikely(ret))
b896313e JP	149	return ret;
1da177e4	150	asm("":::"memory");
b896313e	151	__put_user_asm(1[(u64 )src], 1 + (u64 __user )dst,
155b7352	152	ret, "q", "", "er", 8);
b896313e	153	return ret;
1da177e4	154	default:
b896313e	155	return copy_user_generic((__force void *)dst, src, size);
1da177e4	156	}
b896313e	157	}
1da177e4	158
95912008 AK	159	static __always_inline __must_check
95912008 AK	160	int __copy_in_user(void __user dst, const void __user src, unsigned size)
b896313e	161	{
383d079b	162	int ret = 0;
c10d38dd	163
3ee1afa3	164	might_fault();
1da177e4	165	if (!__builtin_constant_p(size))
b896313e JP	166	return copy_user_generic((__force void *)dst,
	167	(__force void *)src, size);
	168	switch (size) {
	169	case 1: {
1da177e4	170	u8 tmp;
b896313e JP	171	__get_user_asm(tmp, (u8 __user *)src,
b896313e JP	172	ret, "b", "b", "=q", 1);
1da177e4	173	if (likely(!ret))
b896313e JP	174	__put_user_asm(tmp, (u8 __user *)dst,
b896313e JP	175	ret, "b", "b", "iq", 1);
1da177e4 LT	176	return ret;
1da177e4 LT	177	}
b896313e	178	case 2: {
1da177e4	179	u16 tmp;
b896313e JP	180	__get_user_asm(tmp, (u16 __user *)src,
b896313e JP	181	ret, "w", "w", "=r", 2);
1da177e4	182	if (likely(!ret))
b896313e JP	183	__put_user_asm(tmp, (u16 __user *)dst,
b896313e JP	184	ret, "w", "w", "ir", 2);
1da177e4 LT	185	return ret;
	186	}
	187
b896313e	188	case 4: {
1da177e4	189	u32 tmp;
b896313e JP	190	__get_user_asm(tmp, (u32 __user *)src,
b896313e JP	191	ret, "l", "k", "=r", 4);
1da177e4	192	if (likely(!ret))
b896313e JP	193	__put_user_asm(tmp, (u32 __user *)dst,
b896313e JP	194	ret, "l", "k", "ir", 4);
1da177e4 LT	195	return ret;
1da177e4 LT	196	}
b896313e	197	case 8: {
1da177e4	198	u64 tmp;
b896313e JP	199	__get_user_asm(tmp, (u64 __user *)src,
b896313e JP	200	ret, "q", "", "=r", 8);
1da177e4	201	if (likely(!ret))
b896313e	202	__put_user_asm(tmp, (u64 __user *)dst,
155b7352	203	ret, "q", "", "er", 8);
1da177e4 LT	204	return ret;
	205	}
	206	default:
b896313e JP	207	return copy_user_generic((__force void *)dst,
b896313e JP	208	(__force void *)src, size);
1da177e4	209	}
b896313e	210	}
1da177e4	211
b896313e	212	__must_check long
95912008	213	strncpy_from_user(char dst, const char __user src, long count);
b896313e	214	__must_check long
95912008 AK	215	__strncpy_from_user(char dst, const char __user src, long count);
	216	__must_check long strnlen_user(const char __user *str, long n);
	217	__must_check long __strnlen_user(const char __user *str, long n);
	218	__must_check long strlen_user(const char __user *str);
	219	__must_check unsigned long clear_user(void __user *mem, unsigned long len);
	220	__must_check unsigned long __clear_user(void __user *mem, unsigned long len);
	221
14722485 JB	222	static __must_check __always_inline int
	223	__copy_from_user_inatomic(void dst, const void __user src, unsigned size)
	224	{
	225	return copy_user_generic(dst, (__force const void *)src, size);
	226	}
b885808e AK	227
	228	static __must_check __always_inline int
	229	__copy_to_user_inatomic(void __user dst, const void src, unsigned size)
	230	{
	231	return copy_user_generic((__force void *)dst, src, size);
	232	}
1da177e4	233
b896313e JP	234	extern long __copy_user_nocache(void dst, const void __user src,
b896313e JP	235	unsigned size, int zerorest);
0812a579	236
f1800536 IM	237	static inline int
f1800536 IM	238	__copy_from_user_nocache(void dst, const void __user src, unsigned size)
0812a579 AK	239	{
0812a579 AK	240	might_sleep();
f1800536	241	return __copy_user_nocache(dst, src, size, 1);
0812a579 AK	242	}
0812a579 AK	243
f1800536 IM	244	static inline int
	245	__copy_from_user_inatomic_nocache(void dst, const void __user src,
	246	unsigned size)
0812a579	247	{
f1800536	248	return __copy_user_nocache(dst, src, size, 0);
0812a579 AK	249	}
0812a579 AK	250
1129585a VM	251	unsigned long
	252	copy_user_handle_tail(char to, char from, unsigned len, unsigned zerorest);
	253
1965aae3	254	#endif /* _ASM_X86_UACCESS_64_H */