Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* Copyright 2002 Andi Kleen, SuSE Labs. |
2 | * Subject to the GNU Public License v2. | |
3 | * | |
4 | * Functions to copy from and to user space. | |
5 | */ | |
6 | ||
8d379dad JB |
7 | #include <linux/linkage.h> |
8 | #include <asm/dwarf2.h> | |
9 | ||
7bcd3f34 AK |
10 | #define FIX_ALIGNMENT 1 |
11 | ||
3022d734 AK |
12 | #include <asm/current.h> |
13 | #include <asm/asm-offsets.h> | |
14 | #include <asm/thread_info.h> | |
15 | #include <asm/cpufeature.h> | |
16 | ||
17 | .macro ALTERNATIVE_JUMP feature,orig,alt | |
18 | 0: | |
19 | .byte 0xe9 /* 32bit jump */ | |
20 | .long \orig-1f /* by default jump to orig */ | |
21 | 1: | |
22 | .section .altinstr_replacement,"ax" | |
23 | 2: .byte 0xe9 /* near jump with 32bit immediate */ | |
24 | .long \alt-1b /* offset */ /* or alternatively to alt */ | |
25 | .previous | |
26 | .section .altinstructions,"a" | |
27 | .align 8 | |
28 | .quad 0b | |
29 | .quad 2b | |
30 | .byte \feature /* when feature is set */ | |
31 | .byte 5 | |
32 | .byte 5 | |
33 | .previous | |
34 | .endm | |
1da177e4 LT |
35 | |
36 | /* Standard copy_to_user with segment limit checking */ | |
8d379dad JB |
37 | ENTRY(copy_to_user) |
38 | CFI_STARTPROC | |
1da177e4 LT |
39 | GET_THREAD_INFO(%rax) |
40 | movq %rdi,%rcx | |
41 | addq %rdx,%rcx | |
42 | jc bad_to_user | |
43 | cmpq threadinfo_addr_limit(%rax),%rcx | |
44 | jae bad_to_user | |
3022d734 AK |
45 | xorl %eax,%eax /* clear zero flag */ |
46 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string | |
8d379dad | 47 | CFI_ENDPROC |
7bcd3f34 | 48 | |
3022d734 AK |
49 | ENTRY(copy_user_generic) |
50 | CFI_STARTPROC | |
51 | movl $1,%ecx /* set zero flag */ | |
52 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string | |
53 | CFI_ENDPROC | |
54 | ||
55 | ENTRY(__copy_from_user_inatomic) | |
56 | CFI_STARTPROC | |
57 | xorl %ecx,%ecx /* clear zero flag */ | |
58 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string | |
59 | CFI_ENDPROC | |
1da177e4 LT |
60 | |
61 | /* Standard copy_from_user with segment limit checking */ | |
8d379dad JB |
62 | ENTRY(copy_from_user) |
63 | CFI_STARTPROC | |
1da177e4 LT |
64 | GET_THREAD_INFO(%rax) |
65 | movq %rsi,%rcx | |
66 | addq %rdx,%rcx | |
67 | jc bad_from_user | |
68 | cmpq threadinfo_addr_limit(%rax),%rcx | |
69 | jae bad_from_user | |
3022d734 AK |
70 | movl $1,%ecx /* set zero flag */ |
71 | ALTERNATIVE_JUMP X86_FEATURE_REP_GOOD,copy_user_generic_unrolled,copy_user_generic_string | |
8d379dad JB |
72 | CFI_ENDPROC |
73 | ENDPROC(copy_from_user) | |
1da177e4 LT |
74 | |
75 | .section .fixup,"ax" | |
76 | /* must zero dest */ | |
77 | bad_from_user: | |
8d379dad | 78 | CFI_STARTPROC |
1da177e4 LT |
79 | movl %edx,%ecx |
80 | xorl %eax,%eax | |
81 | rep | |
82 | stosb | |
83 | bad_to_user: | |
84 | movl %edx,%eax | |
85 | ret | |
8d379dad JB |
86 | CFI_ENDPROC |
87 | END(bad_from_user) | |
1da177e4 LT |
88 | .previous |
89 | ||
90 | ||
91 | /* | |
3022d734 AK |
92 | * copy_user_generic_unrolled - memory copy with exception handling. |
93 | * This version is for CPUs like P4 that don't have efficient micro code for rep movsq | |
1da177e4 LT |
94 | * |
95 | * Input: | |
96 | * rdi destination | |
97 | * rsi source | |
98 | * rdx count | |
3022d734 | 99 | * ecx zero flag -- if true zero destination on error |
1da177e4 LT |
100 | * |
101 | * Output: | |
102 | * eax uncopied bytes or 0 if successful. | |
103 | */ | |
3022d734 | 104 | ENTRY(copy_user_generic_unrolled) |
8d379dad | 105 | CFI_STARTPROC |
7bcd3f34 | 106 | pushq %rbx |
8d379dad JB |
107 | CFI_ADJUST_CFA_OFFSET 8 |
108 | CFI_REL_OFFSET rbx, 0 | |
3022d734 AK |
109 | pushq %rcx |
110 | CFI_ADJUST_CFA_OFFSET 8 | |
111 | CFI_REL_OFFSET rcx, 0 | |
7bcd3f34 AK |
112 | xorl %eax,%eax /*zero for the exception handler */ |
113 | ||
114 | #ifdef FIX_ALIGNMENT | |
115 | /* check for bad alignment of destination */ | |
116 | movl %edi,%ecx | |
117 | andl $7,%ecx | |
118 | jnz .Lbad_alignment | |
119 | .Lafter_bad_alignment: | |
120 | #endif | |
121 | ||
122 | movq %rdx,%rcx | |
123 | ||
124 | movl $64,%ebx | |
125 | shrq $6,%rdx | |
126 | decq %rdx | |
127 | js .Lhandle_tail | |
128 | ||
129 | .p2align 4 | |
130 | .Lloop: | |
131 | .Ls1: movq (%rsi),%r11 | |
132 | .Ls2: movq 1*8(%rsi),%r8 | |
133 | .Ls3: movq 2*8(%rsi),%r9 | |
134 | .Ls4: movq 3*8(%rsi),%r10 | |
135 | .Ld1: movq %r11,(%rdi) | |
136 | .Ld2: movq %r8,1*8(%rdi) | |
137 | .Ld3: movq %r9,2*8(%rdi) | |
138 | .Ld4: movq %r10,3*8(%rdi) | |
139 | ||
140 | .Ls5: movq 4*8(%rsi),%r11 | |
141 | .Ls6: movq 5*8(%rsi),%r8 | |
142 | .Ls7: movq 6*8(%rsi),%r9 | |
143 | .Ls8: movq 7*8(%rsi),%r10 | |
144 | .Ld5: movq %r11,4*8(%rdi) | |
145 | .Ld6: movq %r8,5*8(%rdi) | |
146 | .Ld7: movq %r9,6*8(%rdi) | |
147 | .Ld8: movq %r10,7*8(%rdi) | |
148 | ||
149 | decq %rdx | |
150 | ||
151 | leaq 64(%rsi),%rsi | |
152 | leaq 64(%rdi),%rdi | |
153 | ||
154 | jns .Lloop | |
155 | ||
156 | .p2align 4 | |
157 | .Lhandle_tail: | |
158 | movl %ecx,%edx | |
159 | andl $63,%ecx | |
160 | shrl $3,%ecx | |
161 | jz .Lhandle_7 | |
162 | movl $8,%ebx | |
163 | .p2align 4 | |
164 | .Lloop_8: | |
165 | .Ls9: movq (%rsi),%r8 | |
166 | .Ld9: movq %r8,(%rdi) | |
167 | decl %ecx | |
168 | leaq 8(%rdi),%rdi | |
169 | leaq 8(%rsi),%rsi | |
170 | jnz .Lloop_8 | |
171 | ||
172 | .Lhandle_7: | |
173 | movl %edx,%ecx | |
174 | andl $7,%ecx | |
175 | jz .Lende | |
176 | .p2align 4 | |
177 | .Lloop_1: | |
178 | .Ls10: movb (%rsi),%bl | |
179 | .Ld10: movb %bl,(%rdi) | |
180 | incq %rdi | |
181 | incq %rsi | |
182 | decl %ecx | |
183 | jnz .Lloop_1 | |
184 | ||
8d379dad | 185 | CFI_REMEMBER_STATE |
7bcd3f34 | 186 | .Lende: |
3022d734 AK |
187 | popq %rcx |
188 | CFI_ADJUST_CFA_OFFSET -8 | |
189 | CFI_RESTORE rcx | |
7bcd3f34 | 190 | popq %rbx |
8d379dad JB |
191 | CFI_ADJUST_CFA_OFFSET -8 |
192 | CFI_RESTORE rbx | |
7bcd3f34 | 193 | ret |
8d379dad | 194 | CFI_RESTORE_STATE |
7bcd3f34 AK |
195 | |
196 | #ifdef FIX_ALIGNMENT | |
197 | /* align destination */ | |
198 | .p2align 4 | |
199 | .Lbad_alignment: | |
200 | movl $8,%r9d | |
201 | subl %ecx,%r9d | |
202 | movl %r9d,%ecx | |
203 | cmpq %r9,%rdx | |
204 | jz .Lhandle_7 | |
205 | js .Lhandle_7 | |
206 | .Lalign_1: | |
207 | .Ls11: movb (%rsi),%bl | |
208 | .Ld11: movb %bl,(%rdi) | |
209 | incq %rsi | |
210 | incq %rdi | |
211 | decl %ecx | |
212 | jnz .Lalign_1 | |
213 | subq %r9,%rdx | |
214 | jmp .Lafter_bad_alignment | |
215 | #endif | |
216 | ||
217 | /* table sorted by exception address */ | |
218 | .section __ex_table,"a" | |
219 | .align 8 | |
42a886af LT |
220 | .quad .Ls1,.Ls1e /* Ls1-Ls4 have copied zero bytes */ |
221 | .quad .Ls2,.Ls1e | |
222 | .quad .Ls3,.Ls1e | |
223 | .quad .Ls4,.Ls1e | |
224 | .quad .Ld1,.Ls1e /* Ld1-Ld4 have copied 0-24 bytes */ | |
7bcd3f34 AK |
225 | .quad .Ld2,.Ls2e |
226 | .quad .Ld3,.Ls3e | |
227 | .quad .Ld4,.Ls4e | |
42a886af LT |
228 | .quad .Ls5,.Ls5e /* Ls5-Ls8 have copied 32 bytes */ |
229 | .quad .Ls6,.Ls5e | |
230 | .quad .Ls7,.Ls5e | |
231 | .quad .Ls8,.Ls5e | |
232 | .quad .Ld5,.Ls5e /* Ld5-Ld8 have copied 32-56 bytes */ | |
7bcd3f34 AK |
233 | .quad .Ld6,.Ls6e |
234 | .quad .Ld7,.Ls7e | |
235 | .quad .Ld8,.Ls8e | |
236 | .quad .Ls9,.Le_quad | |
237 | .quad .Ld9,.Le_quad | |
238 | .quad .Ls10,.Le_byte | |
239 | .quad .Ld10,.Le_byte | |
240 | #ifdef FIX_ALIGNMENT | |
241 | .quad .Ls11,.Lzero_rest | |
242 | .quad .Ld11,.Lzero_rest | |
243 | #endif | |
244 | .quad .Le5,.Le_zero | |
245 | .previous | |
246 | ||
7bcd3f34 | 247 | /* eax: zero, ebx: 64 */ |
42a886af | 248 | .Ls1e: addl $8,%eax /* eax is bytes left uncopied within the loop (Ls1e: 64 .. Ls8e: 8) */ |
7bcd3f34 AK |
249 | .Ls2e: addl $8,%eax |
250 | .Ls3e: addl $8,%eax | |
251 | .Ls4e: addl $8,%eax | |
252 | .Ls5e: addl $8,%eax | |
253 | .Ls6e: addl $8,%eax | |
254 | .Ls7e: addl $8,%eax | |
255 | .Ls8e: addl $8,%eax | |
256 | addq %rbx,%rdi /* +64 */ | |
257 | subq %rax,%rdi /* correct destination with computed offset */ | |
258 | ||
259 | shlq $6,%rdx /* loop counter * 64 (stride length) */ | |
260 | addq %rax,%rdx /* add offset to loopcnt */ | |
261 | andl $63,%ecx /* remaining bytes */ | |
262 | addq %rcx,%rdx /* add them */ | |
263 | jmp .Lzero_rest | |
264 | ||
265 | /* exception on quad word loop in tail handling */ | |
266 | /* ecx: loopcnt/8, %edx: length, rdi: correct */ | |
267 | .Le_quad: | |
268 | shll $3,%ecx | |
269 | andl $7,%edx | |
270 | addl %ecx,%edx | |
271 | /* edx: bytes to zero, rdi: dest, eax:zero */ | |
272 | .Lzero_rest: | |
3022d734 AK |
273 | cmpl $0,(%rsp) |
274 | jz .Le_zero | |
7bcd3f34 AK |
275 | movq %rdx,%rcx |
276 | .Le_byte: | |
277 | xorl %eax,%eax | |
278 | .Le5: rep | |
279 | stosb | |
280 | /* when there is another exception while zeroing the rest just return */ | |
281 | .Le_zero: | |
282 | movq %rdx,%rax | |
283 | jmp .Lende | |
8d379dad JB |
284 | CFI_ENDPROC |
285 | ENDPROC(copy_user_generic) | |
286 | ||
7bcd3f34 AK |
287 | |
288 | /* Some CPUs run faster using the string copy instructions. | |
289 | This is also a lot simpler. Use them when possible. | |
290 | Patch in jmps to this code instead of copying it fully | |
291 | to avoid unwanted aliasing in the exception tables. */ | |
292 | ||
293 | /* rdi destination | |
294 | * rsi source | |
295 | * rdx count | |
3022d734 | 296 | * ecx zero flag |
7bcd3f34 AK |
297 | * |
298 | * Output: | |
299 | * eax uncopied bytes or 0 if successfull. | |
300 | * | |
301 | * Only 4GB of copy is supported. This shouldn't be a problem | |
302 | * because the kernel normally only writes from/to page sized chunks | |
303 | * even if user space passed a longer buffer. | |
304 | * And more would be dangerous because both Intel and AMD have | |
305 | * errata with rep movsq > 4GB. If someone feels the need to fix | |
306 | * this please consider this. | |
3022d734 AK |
307 | */ |
308 | ENTRY(copy_user_generic_string) | |
8d379dad | 309 | CFI_STARTPROC |
3022d734 | 310 | movl %ecx,%r8d /* save zero flag */ |
1da177e4 LT |
311 | movl %edx,%ecx |
312 | shrl $3,%ecx | |
313 | andl $7,%edx | |
3022d734 | 314 | jz 10f |
1da177e4 LT |
315 | 1: rep |
316 | movsq | |
317 | movl %edx,%ecx | |
318 | 2: rep | |
319 | movsb | |
3022d734 | 320 | 9: movl %ecx,%eax |
1da177e4 | 321 | ret |
3022d734 AK |
322 | |
323 | /* multiple of 8 byte */ | |
324 | 10: rep | |
325 | movsq | |
326 | xor %eax,%eax | |
1da177e4 | 327 | ret |
3022d734 AK |
328 | |
329 | /* exception handling */ | |
330 | 3: lea (%rdx,%rcx,8),%rax /* exception on quad loop */ | |
331 | jmp 6f | |
332 | 5: movl %ecx,%eax /* exception on byte loop */ | |
333 | /* eax: left over bytes */ | |
334 | 6: testl %r8d,%r8d /* zero flag set? */ | |
335 | jz 7f | |
336 | movl %eax,%ecx /* initialize x86 loop counter */ | |
337 | push %rax | |
338 | xorl %eax,%eax | |
339 | 8: rep | |
340 | stosb /* zero the rest */ | |
341 | 11: pop %rax | |
342 | 7: ret | |
8d379dad JB |
343 | CFI_ENDPROC |
344 | END(copy_user_generic_c) | |
2cbc9ee3 | 345 | |
1da177e4 LT |
346 | .section __ex_table,"a" |
347 | .quad 1b,3b | |
3022d734 AK |
348 | .quad 2b,5b |
349 | .quad 8b,11b | |
350 | .quad 10b,3b | |
1da177e4 | 351 | .previous |