Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | #ifndef __X86_64_UACCESS_H |
2 | #define __X86_64_UACCESS_H | |
3 | ||
4 | /* | |
5 | * User space memory access functions | |
6 | */ | |
1da177e4 LT |
7 | #include <linux/compiler.h> |
8 | #include <linux/errno.h> | |
9 | #include <linux/sched.h> | |
10 | #include <linux/prefetch.h> | |
11 | #include <asm/page.h> | |
12 | ||
13 | #define VERIFY_READ 0 | |
14 | #define VERIFY_WRITE 1 | |
15 | ||
16 | /* | |
17 | * The fs value determines whether argument validity checking should be | |
18 | * performed or not. If get_fs() == USER_DS, checking is performed, with | |
19 | * get_fs() == KERNEL_DS, checking is bypassed. | |
20 | * | |
21 | * For historical reasons, these macros are grossly misnamed. | |
22 | */ | |
23 | ||
24 | #define MAKE_MM_SEG(s) ((mm_segment_t) { (s) }) | |
25 | ||
26 | #define KERNEL_DS MAKE_MM_SEG(0xFFFFFFFFFFFFFFFFUL) | |
27 | #define USER_DS MAKE_MM_SEG(PAGE_OFFSET) | |
28 | ||
29 | #define get_ds() (KERNEL_DS) | |
30 | #define get_fs() (current_thread_info()->addr_limit) | |
31 | #define set_fs(x) (current_thread_info()->addr_limit = (x)) | |
32 | ||
33 | #define segment_eq(a,b) ((a).seg == (b).seg) | |
34 | ||
35 | #define __addr_ok(addr) (!((unsigned long)(addr) & (current_thread_info()->addr_limit.seg))) | |
36 | ||
37 | /* | |
38 | * Uhhuh, this needs 65-bit arithmetic. We have a carry.. | |
39 | */ | |
40 | #define __range_not_ok(addr,size) ({ \ | |
41 | unsigned long flag,sum; \ | |
42 | __chk_user_ptr(addr); \ | |
43 | asm("# range_ok\n\r" \ | |
44 | "addq %3,%1 ; sbbq %0,%0 ; cmpq %1,%4 ; sbbq $0,%0" \ | |
45 | :"=&r" (flag), "=r" (sum) \ | |
46 | :"1" (addr),"g" ((long)(size)),"g" (current_thread_info()->addr_limit.seg)); \ | |
47 | flag; }) | |
48 | ||
49 | #define access_ok(type, addr, size) (__range_not_ok(addr,size) == 0) | |
50 | ||
1da177e4 LT |
51 | /* |
52 | * The exception table consists of pairs of addresses: the first is the | |
53 | * address of an instruction that is allowed to fault, and the second is | |
54 | * the address at which the program should continue. No registers are | |
55 | * modified, so it is entirely up to the continuation code to figure out | |
56 | * what to do. | |
57 | * | |
58 | * All the routines below use bits of fixup code that are out of line | |
59 | * with the main instruction path. This means when everything is well, | |
60 | * we don't even have to jump over them. Further, they do not intrude | |
61 | * on our cache or tlb entries. | |
62 | */ | |
63 | ||
64 | struct exception_table_entry | |
65 | { | |
66 | unsigned long insn, fixup; | |
67 | }; | |
68 | ||
69 | #define ARCH_HAS_SEARCH_EXTABLE | |
70 | ||
71 | /* | |
72 | * These are the main single-value transfer routines. They automatically | |
73 | * use the right size if we just have the right pointer type. | |
74 | * | |
75 | * This gets kind of ugly. We want to return _two_ values in "get_user()" | |
76 | * and yet we don't want to do any pointers, because that is too much | |
77 | * of a performance impact. Thus we have a few rather ugly macros here, | |
78 | * and hide all the ugliness from the user. | |
79 | * | |
80 | * The "__xxx" versions of the user access functions are versions that | |
81 | * do not verify the address space, that must have been done previously | |
82 | * with a separate "access_ok()" call (this is used when we do multiple | |
83 | * accesses to the same area of user memory). | |
84 | */ | |
85 | ||
86 | #define __get_user_x(size,ret,x,ptr) \ | |
87 | __asm__ __volatile__("call __get_user_" #size \ | |
88 | :"=a" (ret),"=d" (x) \ | |
89 | :"c" (ptr) \ | |
90 | :"r8") | |
91 | ||
92 | /* Careful: we have to cast the result to the type of the pointer for sign reasons */ | |
93 | #define get_user(x,ptr) \ | |
94 | ({ unsigned long __val_gu; \ | |
95 | int __ret_gu; \ | |
96 | __chk_user_ptr(ptr); \ | |
97 | switch(sizeof (*(ptr))) { \ | |
98 | case 1: __get_user_x(1,__ret_gu,__val_gu,ptr); break; \ | |
99 | case 2: __get_user_x(2,__ret_gu,__val_gu,ptr); break; \ | |
100 | case 4: __get_user_x(4,__ret_gu,__val_gu,ptr); break; \ | |
101 | case 8: __get_user_x(8,__ret_gu,__val_gu,ptr); break; \ | |
102 | default: __get_user_bad(); break; \ | |
103 | } \ | |
104 | (x) = (__typeof__(*(ptr)))__val_gu; \ | |
105 | __ret_gu; \ | |
106 | }) | |
107 | ||
108 | extern void __put_user_1(void); | |
109 | extern void __put_user_2(void); | |
110 | extern void __put_user_4(void); | |
111 | extern void __put_user_8(void); | |
112 | extern void __put_user_bad(void); | |
113 | ||
114 | #define __put_user_x(size,ret,x,ptr) \ | |
115 | __asm__ __volatile__("call __put_user_" #size \ | |
116 | :"=a" (ret) \ | |
117 | :"c" (ptr),"d" (x) \ | |
118 | :"r8") | |
119 | ||
120 | #define put_user(x,ptr) \ | |
121 | __put_user_check((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr))) | |
122 | ||
123 | #define __get_user(x,ptr) \ | |
124 | __get_user_nocheck((x),(ptr),sizeof(*(ptr))) | |
125 | #define __put_user(x,ptr) \ | |
126 | __put_user_nocheck((__typeof__(*(ptr)))(x),(ptr),sizeof(*(ptr))) | |
127 | ||
128 | #define __get_user_unaligned __get_user | |
129 | #define __put_user_unaligned __put_user | |
130 | ||
131 | #define __put_user_nocheck(x,ptr,size) \ | |
132 | ({ \ | |
133 | int __pu_err; \ | |
134 | __put_user_size((x),(ptr),(size),__pu_err); \ | |
135 | __pu_err; \ | |
136 | }) | |
137 | ||
138 | ||
139 | #define __put_user_check(x,ptr,size) \ | |
140 | ({ \ | |
141 | int __pu_err; \ | |
142 | __typeof__(*(ptr)) __user *__pu_addr = (ptr); \ | |
143 | switch (size) { \ | |
144 | case 1: __put_user_x(1,__pu_err,x,__pu_addr); break; \ | |
145 | case 2: __put_user_x(2,__pu_err,x,__pu_addr); break; \ | |
146 | case 4: __put_user_x(4,__pu_err,x,__pu_addr); break; \ | |
147 | case 8: __put_user_x(8,__pu_err,x,__pu_addr); break; \ | |
148 | default: __put_user_bad(); \ | |
149 | } \ | |
150 | __pu_err; \ | |
151 | }) | |
152 | ||
153 | #define __put_user_size(x,ptr,size,retval) \ | |
154 | do { \ | |
155 | retval = 0; \ | |
156 | __chk_user_ptr(ptr); \ | |
157 | switch (size) { \ | |
158 | case 1: __put_user_asm(x,ptr,retval,"b","b","iq",-EFAULT); break;\ | |
159 | case 2: __put_user_asm(x,ptr,retval,"w","w","ir",-EFAULT); break;\ | |
160 | case 4: __put_user_asm(x,ptr,retval,"l","k","ir",-EFAULT); break;\ | |
161 | case 8: __put_user_asm(x,ptr,retval,"q","","ir",-EFAULT); break;\ | |
162 | default: __put_user_bad(); \ | |
163 | } \ | |
164 | } while (0) | |
165 | ||
166 | /* FIXME: this hack is definitely wrong -AK */ | |
167 | struct __large_struct { unsigned long buf[100]; }; | |
168 | #define __m(x) (*(struct __large_struct __user *)(x)) | |
169 | ||
170 | /* | |
171 | * Tell gcc we read from memory instead of writing: this is because | |
172 | * we do not write to any memory gcc knows about, so there are no | |
173 | * aliasing issues. | |
174 | */ | |
175 | #define __put_user_asm(x, addr, err, itype, rtype, ltype, errno) \ | |
176 | __asm__ __volatile__( \ | |
177 | "1: mov"itype" %"rtype"1,%2\n" \ | |
178 | "2:\n" \ | |
179 | ".section .fixup,\"ax\"\n" \ | |
180 | "3: mov %3,%0\n" \ | |
181 | " jmp 2b\n" \ | |
182 | ".previous\n" \ | |
183 | ".section __ex_table,\"a\"\n" \ | |
184 | " .align 8\n" \ | |
185 | " .quad 1b,3b\n" \ | |
186 | ".previous" \ | |
187 | : "=r"(err) \ | |
188 | : ltype (x), "m"(__m(addr)), "i"(errno), "0"(err)) | |
189 | ||
190 | ||
191 | #define __get_user_nocheck(x,ptr,size) \ | |
192 | ({ \ | |
193 | int __gu_err; \ | |
194 | unsigned long __gu_val; \ | |
195 | __get_user_size(__gu_val,(ptr),(size),__gu_err); \ | |
196 | (x) = (__typeof__(*(ptr)))__gu_val; \ | |
197 | __gu_err; \ | |
198 | }) | |
199 | ||
200 | extern int __get_user_1(void); | |
201 | extern int __get_user_2(void); | |
202 | extern int __get_user_4(void); | |
203 | extern int __get_user_8(void); | |
204 | extern int __get_user_bad(void); | |
205 | ||
206 | #define __get_user_size(x,ptr,size,retval) \ | |
207 | do { \ | |
208 | retval = 0; \ | |
209 | __chk_user_ptr(ptr); \ | |
210 | switch (size) { \ | |
211 | case 1: __get_user_asm(x,ptr,retval,"b","b","=q",-EFAULT); break;\ | |
212 | case 2: __get_user_asm(x,ptr,retval,"w","w","=r",-EFAULT); break;\ | |
213 | case 4: __get_user_asm(x,ptr,retval,"l","k","=r",-EFAULT); break;\ | |
214 | case 8: __get_user_asm(x,ptr,retval,"q","","=r",-EFAULT); break;\ | |
215 | default: (x) = __get_user_bad(); \ | |
216 | } \ | |
217 | } while (0) | |
218 | ||
219 | #define __get_user_asm(x, addr, err, itype, rtype, ltype, errno) \ | |
220 | __asm__ __volatile__( \ | |
221 | "1: mov"itype" %2,%"rtype"1\n" \ | |
222 | "2:\n" \ | |
223 | ".section .fixup,\"ax\"\n" \ | |
224 | "3: mov %3,%0\n" \ | |
225 | " xor"itype" %"rtype"1,%"rtype"1\n" \ | |
226 | " jmp 2b\n" \ | |
227 | ".previous\n" \ | |
228 | ".section __ex_table,\"a\"\n" \ | |
229 | " .align 8\n" \ | |
230 | " .quad 1b,3b\n" \ | |
231 | ".previous" \ | |
232 | : "=r"(err), ltype (x) \ | |
233 | : "m"(__m(addr)), "i"(errno), "0"(err)) | |
234 | ||
235 | /* | |
236 | * Copy To/From Userspace | |
237 | */ | |
238 | ||
239 | /* Handles exceptions in both to and from, but doesn't do access_ok */ | |
240 | extern unsigned long copy_user_generic(void *to, const void *from, unsigned len); | |
3022d734 | 241 | extern unsigned long copy_user_generic_dontzero(void *to, const void *from, unsigned len); |
1da177e4 LT |
242 | |
243 | extern unsigned long copy_to_user(void __user *to, const void *from, unsigned len); | |
244 | extern unsigned long copy_from_user(void *to, const void __user *from, unsigned len); | |
245 | extern unsigned long copy_in_user(void __user *to, const void __user *from, unsigned len); | |
246 | ||
652050ae | 247 | static __always_inline int __copy_from_user(void *dst, const void __user *src, unsigned size) |
1da177e4 LT |
248 | { |
249 | int ret = 0; | |
250 | if (!__builtin_constant_p(size)) | |
251 | return copy_user_generic(dst,(__force void *)src,size); | |
252 | switch (size) { | |
253 | case 1:__get_user_asm(*(u8*)dst,(u8 __user *)src,ret,"b","b","=q",1); | |
254 | return ret; | |
255 | case 2:__get_user_asm(*(u16*)dst,(u16 __user *)src,ret,"w","w","=r",2); | |
256 | return ret; | |
257 | case 4:__get_user_asm(*(u32*)dst,(u32 __user *)src,ret,"l","k","=r",4); | |
258 | return ret; | |
259 | case 8:__get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",8); | |
260 | return ret; | |
261 | case 10: | |
262 | __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); | |
263 | if (unlikely(ret)) return ret; | |
264 | __get_user_asm(*(u16*)(8+(char*)dst),(u16 __user *)(8+(char __user *)src),ret,"w","w","=r",2); | |
265 | return ret; | |
266 | case 16: | |
267 | __get_user_asm(*(u64*)dst,(u64 __user *)src,ret,"q","","=r",16); | |
268 | if (unlikely(ret)) return ret; | |
269 | __get_user_asm(*(u64*)(8+(char*)dst),(u64 __user *)(8+(char __user *)src),ret,"q","","=r",8); | |
270 | return ret; | |
271 | default: | |
272 | return copy_user_generic(dst,(__force void *)src,size); | |
273 | } | |
274 | } | |
275 | ||
652050ae | 276 | static __always_inline int __copy_to_user(void __user *dst, const void *src, unsigned size) |
1da177e4 LT |
277 | { |
278 | int ret = 0; | |
279 | if (!__builtin_constant_p(size)) | |
280 | return copy_user_generic((__force void *)dst,src,size); | |
281 | switch (size) { | |
282 | case 1:__put_user_asm(*(u8*)src,(u8 __user *)dst,ret,"b","b","iq",1); | |
283 | return ret; | |
284 | case 2:__put_user_asm(*(u16*)src,(u16 __user *)dst,ret,"w","w","ir",2); | |
285 | return ret; | |
286 | case 4:__put_user_asm(*(u32*)src,(u32 __user *)dst,ret,"l","k","ir",4); | |
287 | return ret; | |
288 | case 8:__put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",8); | |
289 | return ret; | |
290 | case 10: | |
291 | __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",10); | |
292 | if (unlikely(ret)) return ret; | |
293 | asm("":::"memory"); | |
294 | __put_user_asm(4[(u16*)src],4+(u16 __user *)dst,ret,"w","w","ir",2); | |
295 | return ret; | |
296 | case 16: | |
297 | __put_user_asm(*(u64*)src,(u64 __user *)dst,ret,"q","","ir",16); | |
298 | if (unlikely(ret)) return ret; | |
299 | asm("":::"memory"); | |
300 | __put_user_asm(1[(u64*)src],1+(u64 __user *)dst,ret,"q","","ir",8); | |
301 | return ret; | |
302 | default: | |
303 | return copy_user_generic((__force void *)dst,src,size); | |
304 | } | |
305 | } | |
306 | ||
652050ae | 307 | static __always_inline int __copy_in_user(void __user *dst, const void __user *src, unsigned size) |
1da177e4 LT |
308 | { |
309 | int ret = 0; | |
310 | if (!__builtin_constant_p(size)) | |
311 | return copy_user_generic((__force void *)dst,(__force void *)src,size); | |
312 | switch (size) { | |
313 | case 1: { | |
314 | u8 tmp; | |
315 | __get_user_asm(tmp,(u8 __user *)src,ret,"b","b","=q",1); | |
316 | if (likely(!ret)) | |
317 | __put_user_asm(tmp,(u8 __user *)dst,ret,"b","b","iq",1); | |
318 | return ret; | |
319 | } | |
320 | case 2: { | |
321 | u16 tmp; | |
322 | __get_user_asm(tmp,(u16 __user *)src,ret,"w","w","=r",2); | |
323 | if (likely(!ret)) | |
324 | __put_user_asm(tmp,(u16 __user *)dst,ret,"w","w","ir",2); | |
325 | return ret; | |
326 | } | |
327 | ||
328 | case 4: { | |
329 | u32 tmp; | |
330 | __get_user_asm(tmp,(u32 __user *)src,ret,"l","k","=r",4); | |
331 | if (likely(!ret)) | |
332 | __put_user_asm(tmp,(u32 __user *)dst,ret,"l","k","ir",4); | |
333 | return ret; | |
334 | } | |
335 | case 8: { | |
336 | u64 tmp; | |
337 | __get_user_asm(tmp,(u64 __user *)src,ret,"q","","=r",8); | |
338 | if (likely(!ret)) | |
339 | __put_user_asm(tmp,(u64 __user *)dst,ret,"q","","ir",8); | |
340 | return ret; | |
341 | } | |
342 | default: | |
343 | return copy_user_generic((__force void *)dst,(__force void *)src,size); | |
344 | } | |
345 | } | |
346 | ||
347 | long strncpy_from_user(char *dst, const char __user *src, long count); | |
348 | long __strncpy_from_user(char *dst, const char __user *src, long count); | |
349 | long strnlen_user(const char __user *str, long n); | |
5f1d189f | 350 | long __strnlen_user(const char __user *str, long n); |
1da177e4 LT |
351 | long strlen_user(const char __user *str); |
352 | unsigned long clear_user(void __user *mem, unsigned long len); | |
353 | unsigned long __clear_user(void __user *mem, unsigned long len); | |
354 | ||
3022d734 AK |
355 | extern long __copy_from_user_inatomic(void *dst, const void __user *src, unsigned size); |
356 | #define __copy_to_user_inatomic copy_user_generic | |
1da177e4 LT |
357 | |
358 | #endif /* __X86_64_UACCESS_H */ |