Commit | Line | Data |
---|---|---|
373cd784 JH |
1 | /* |
2 | * User address space access functions. | |
3 | * The non-inlined parts of asm-metag/uaccess.h are here. | |
4 | * | |
5 | * Copyright (C) 2006, Imagination Technologies. | |
6 | * Copyright (C) 2000, Axis Communications AB. | |
7 | * | |
8 | * Written by Hans-Peter Nilsson. | |
9 | * Pieces used from memcpy, originally by Kenny Ranerup long time ago. | |
10 | * Modified for Meta by Will Newton. | |
11 | */ | |
12 | ||
9da3ee9a | 13 | #include <linux/export.h> |
373cd784 JH |
14 | #include <linux/uaccess.h> |
15 | #include <asm/cache.h> /* def of L1_CACHE_BYTES */ | |
16 | ||
17 | #define USE_RAPF | |
18 | #define RAPF_MIN_BUF_SIZE (3*L1_CACHE_BYTES) | |
19 | ||
20 | ||
21 | /* The "double write" in this code is because the Meta will not fault | |
22 | * immediately unless the memory pipe is forced to by e.g. a data stall or | |
23 | * another memory op. The second write should be discarded by the write | |
24 | * combiner so should have virtually no cost. | |
25 | */ | |
26 | ||
27 | #define __asm_copy_user_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
28 | asm volatile ( \ | |
29 | COPY \ | |
30 | "1:\n" \ | |
31 | " .section .fixup,\"ax\"\n" \ | |
32 | " MOV D1Ar1,#0\n" \ | |
33 | FIXUP \ | |
34 | " MOVT D1Ar1,#HI(1b)\n" \ | |
35 | " JUMP D1Ar1,#LO(1b)\n" \ | |
36 | " .previous\n" \ | |
37 | " .section __ex_table,\"a\"\n" \ | |
38 | TENTRY \ | |
39 | " .previous\n" \ | |
40 | : "=r" (to), "=r" (from), "=r" (ret) \ | |
41 | : "0" (to), "1" (from), "2" (ret) \ | |
42 | : "D1Ar1", "memory") | |
43 | ||
44 | ||
45 | #define __asm_copy_to_user_1(to, from, ret) \ | |
46 | __asm_copy_user_cont(to, from, ret, \ | |
47 | " GETB D1Ar1,[%1++]\n" \ | |
48 | " SETB [%0],D1Ar1\n" \ | |
49 | "2: SETB [%0++],D1Ar1\n", \ | |
50 | "3: ADD %2,%2,#1\n", \ | |
51 | " .long 2b,3b\n") | |
52 | ||
53 | #define __asm_copy_to_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
54 | __asm_copy_user_cont(to, from, ret, \ | |
55 | " GETW D1Ar1,[%1++]\n" \ | |
56 | " SETW [%0],D1Ar1\n" \ | |
57 | "2: SETW [%0++],D1Ar1\n" COPY, \ | |
58 | "3: ADD %2,%2,#2\n" FIXUP, \ | |
59 | " .long 2b,3b\n" TENTRY) | |
60 | ||
61 | #define __asm_copy_to_user_2(to, from, ret) \ | |
62 | __asm_copy_to_user_2x_cont(to, from, ret, "", "", "") | |
63 | ||
64 | #define __asm_copy_to_user_3(to, from, ret) \ | |
65 | __asm_copy_to_user_2x_cont(to, from, ret, \ | |
66 | " GETB D1Ar1,[%1++]\n" \ | |
67 | " SETB [%0],D1Ar1\n" \ | |
68 | "4: SETB [%0++],D1Ar1\n", \ | |
69 | "5: ADD %2,%2,#1\n", \ | |
70 | " .long 4b,5b\n") | |
71 | ||
72 | #define __asm_copy_to_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
73 | __asm_copy_user_cont(to, from, ret, \ | |
74 | " GETD D1Ar1,[%1++]\n" \ | |
75 | " SETD [%0],D1Ar1\n" \ | |
76 | "2: SETD [%0++],D1Ar1\n" COPY, \ | |
77 | "3: ADD %2,%2,#4\n" FIXUP, \ | |
78 | " .long 2b,3b\n" TENTRY) | |
79 | ||
80 | #define __asm_copy_to_user_4(to, from, ret) \ | |
81 | __asm_copy_to_user_4x_cont(to, from, ret, "", "", "") | |
82 | ||
83 | #define __asm_copy_to_user_5(to, from, ret) \ | |
84 | __asm_copy_to_user_4x_cont(to, from, ret, \ | |
85 | " GETB D1Ar1,[%1++]\n" \ | |
86 | " SETB [%0],D1Ar1\n" \ | |
87 | "4: SETB [%0++],D1Ar1\n", \ | |
88 | "5: ADD %2,%2,#1\n", \ | |
89 | " .long 4b,5b\n") | |
90 | ||
91 | #define __asm_copy_to_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
92 | __asm_copy_to_user_4x_cont(to, from, ret, \ | |
93 | " GETW D1Ar1,[%1++]\n" \ | |
94 | " SETW [%0],D1Ar1\n" \ | |
95 | "4: SETW [%0++],D1Ar1\n" COPY, \ | |
96 | "5: ADD %2,%2,#2\n" FIXUP, \ | |
97 | " .long 4b,5b\n" TENTRY) | |
98 | ||
99 | #define __asm_copy_to_user_6(to, from, ret) \ | |
100 | __asm_copy_to_user_6x_cont(to, from, ret, "", "", "") | |
101 | ||
102 | #define __asm_copy_to_user_7(to, from, ret) \ | |
103 | __asm_copy_to_user_6x_cont(to, from, ret, \ | |
104 | " GETB D1Ar1,[%1++]\n" \ | |
105 | " SETB [%0],D1Ar1\n" \ | |
106 | "6: SETB [%0++],D1Ar1\n", \ | |
107 | "7: ADD %2,%2,#1\n", \ | |
108 | " .long 6b,7b\n") | |
109 | ||
110 | #define __asm_copy_to_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
111 | __asm_copy_to_user_4x_cont(to, from, ret, \ | |
112 | " GETD D1Ar1,[%1++]\n" \ | |
113 | " SETD [%0],D1Ar1\n" \ | |
114 | "4: SETD [%0++],D1Ar1\n" COPY, \ | |
115 | "5: ADD %2,%2,#4\n" FIXUP, \ | |
116 | " .long 4b,5b\n" TENTRY) | |
117 | ||
118 | #define __asm_copy_to_user_8(to, from, ret) \ | |
119 | __asm_copy_to_user_8x_cont(to, from, ret, "", "", "") | |
120 | ||
121 | #define __asm_copy_to_user_9(to, from, ret) \ | |
122 | __asm_copy_to_user_8x_cont(to, from, ret, \ | |
123 | " GETB D1Ar1,[%1++]\n" \ | |
124 | " SETB [%0],D1Ar1\n" \ | |
125 | "6: SETB [%0++],D1Ar1\n", \ | |
126 | "7: ADD %2,%2,#1\n", \ | |
127 | " .long 6b,7b\n") | |
128 | ||
129 | #define __asm_copy_to_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
130 | __asm_copy_to_user_8x_cont(to, from, ret, \ | |
131 | " GETW D1Ar1,[%1++]\n" \ | |
132 | " SETW [%0],D1Ar1\n" \ | |
133 | "6: SETW [%0++],D1Ar1\n" COPY, \ | |
134 | "7: ADD %2,%2,#2\n" FIXUP, \ | |
135 | " .long 6b,7b\n" TENTRY) | |
136 | ||
137 | #define __asm_copy_to_user_10(to, from, ret) \ | |
138 | __asm_copy_to_user_10x_cont(to, from, ret, "", "", "") | |
139 | ||
140 | #define __asm_copy_to_user_11(to, from, ret) \ | |
141 | __asm_copy_to_user_10x_cont(to, from, ret, \ | |
142 | " GETB D1Ar1,[%1++]\n" \ | |
143 | " SETB [%0],D1Ar1\n" \ | |
144 | "8: SETB [%0++],D1Ar1\n", \ | |
145 | "9: ADD %2,%2,#1\n", \ | |
146 | " .long 8b,9b\n") | |
147 | ||
148 | #define __asm_copy_to_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
149 | __asm_copy_to_user_8x_cont(to, from, ret, \ | |
150 | " GETD D1Ar1,[%1++]\n" \ | |
151 | " SETD [%0],D1Ar1\n" \ | |
152 | "6: SETD [%0++],D1Ar1\n" COPY, \ | |
153 | "7: ADD %2,%2,#4\n" FIXUP, \ | |
154 | " .long 6b,7b\n" TENTRY) | |
155 | #define __asm_copy_to_user_12(to, from, ret) \ | |
156 | __asm_copy_to_user_12x_cont(to, from, ret, "", "", "") | |
157 | ||
158 | #define __asm_copy_to_user_13(to, from, ret) \ | |
159 | __asm_copy_to_user_12x_cont(to, from, ret, \ | |
160 | " GETB D1Ar1,[%1++]\n" \ | |
161 | " SETB [%0],D1Ar1\n" \ | |
162 | "8: SETB [%0++],D1Ar1\n", \ | |
163 | "9: ADD %2,%2,#1\n", \ | |
164 | " .long 8b,9b\n") | |
165 | ||
166 | #define __asm_copy_to_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
167 | __asm_copy_to_user_12x_cont(to, from, ret, \ | |
168 | " GETW D1Ar1,[%1++]\n" \ | |
169 | " SETW [%0],D1Ar1\n" \ | |
170 | "8: SETW [%0++],D1Ar1\n" COPY, \ | |
171 | "9: ADD %2,%2,#2\n" FIXUP, \ | |
172 | " .long 8b,9b\n" TENTRY) | |
173 | ||
174 | #define __asm_copy_to_user_14(to, from, ret) \ | |
175 | __asm_copy_to_user_14x_cont(to, from, ret, "", "", "") | |
176 | ||
177 | #define __asm_copy_to_user_15(to, from, ret) \ | |
178 | __asm_copy_to_user_14x_cont(to, from, ret, \ | |
179 | " GETB D1Ar1,[%1++]\n" \ | |
180 | " SETB [%0],D1Ar1\n" \ | |
181 | "10: SETB [%0++],D1Ar1\n", \ | |
182 | "11: ADD %2,%2,#1\n", \ | |
183 | " .long 10b,11b\n") | |
184 | ||
185 | #define __asm_copy_to_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
186 | __asm_copy_to_user_12x_cont(to, from, ret, \ | |
187 | " GETD D1Ar1,[%1++]\n" \ | |
188 | " SETD [%0],D1Ar1\n" \ | |
189 | "8: SETD [%0++],D1Ar1\n" COPY, \ | |
190 | "9: ADD %2,%2,#4\n" FIXUP, \ | |
191 | " .long 8b,9b\n" TENTRY) | |
192 | ||
193 | #define __asm_copy_to_user_16(to, from, ret) \ | |
194 | __asm_copy_to_user_16x_cont(to, from, ret, "", "", "") | |
195 | ||
196 | #define __asm_copy_to_user_8x64(to, from, ret) \ | |
197 | asm volatile ( \ | |
198 | " GETL D0Ar2,D1Ar1,[%1++]\n" \ | |
199 | " SETL [%0],D0Ar2,D1Ar1\n" \ | |
200 | "2: SETL [%0++],D0Ar2,D1Ar1\n" \ | |
201 | "1:\n" \ | |
202 | " .section .fixup,\"ax\"\n" \ | |
203 | "3: ADD %2,%2,#8\n" \ | |
204 | " MOVT D0Ar2,#HI(1b)\n" \ | |
205 | " JUMP D0Ar2,#LO(1b)\n" \ | |
206 | " .previous\n" \ | |
207 | " .section __ex_table,\"a\"\n" \ | |
208 | " .long 2b,3b\n" \ | |
209 | " .previous\n" \ | |
210 | : "=r" (to), "=r" (from), "=r" (ret) \ | |
211 | : "0" (to), "1" (from), "2" (ret) \ | |
212 | : "D1Ar1", "D0Ar2", "memory") | |
213 | ||
214 | /* | |
215 | * optimized copying loop using RAPF when 64 bit aligned | |
216 | * | |
217 | * n will be automatically decremented inside the loop | |
218 | * ret will be left intact. if error occurs we will rewind | |
219 | * so that the original non optimized code will fill up | |
220 | * this value correctly. | |
221 | * | |
222 | * on fault: | |
223 | * > n will hold total number of uncopied bytes | |
224 | * | |
225 | * > {'to','from'} will be rewind back so that | |
226 | * the non-optimized code will do the proper fix up | |
227 | * | |
228 | * DCACHE drops the cacheline which helps in reducing cache | |
229 | * pollution. | |
230 | * | |
231 | * We introduce an extra SETL at the end of the loop to | |
232 | * ensure we don't fall off the loop before we catch all | |
233 | * erros. | |
234 | * | |
235 | * NOTICE: | |
236 | * LSM_STEP in TXSTATUS must be cleared in fix up code. | |
237 | * since we're using M{S,G}ETL, a fault might happen at | |
238 | * any address in the middle of M{S,G}ETL causing | |
239 | * the value of LSM_STEP to be incorrect which can | |
240 | * cause subsequent use of M{S,G}ET{L,D} to go wrong. | |
241 | * ie: if LSM_STEP was 1 when a fault occurs, the | |
242 | * next call to M{S,G}ET{L,D} will skip the first | |
243 | * copy/getting as it think that the first 1 has already | |
244 | * been done. | |
245 | * | |
246 | */ | |
247 | #define __asm_copy_user_64bit_rapf_loop( \ | |
248 | to, from, ret, n, id, FIXUP) \ | |
249 | asm volatile ( \ | |
250 | ".balign 8\n" \ | |
251 | "MOV RAPF, %1\n" \ | |
252 | "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \ | |
253 | "MOV D0Ar6, #0\n" \ | |
254 | "LSR D1Ar5, %3, #6\n" \ | |
255 | "SUB TXRPT, D1Ar5, #2\n" \ | |
256 | "MOV RAPF, %1\n" \ | |
257 | "$Lloop"id":\n" \ | |
258 | "ADD RAPF, %1, #64\n" \ | |
259 | "21:\n" \ | |
260 | "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
261 | "22:\n" \ | |
262 | "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
263 | "SUB %3, %3, #32\n" \ | |
264 | "23:\n" \ | |
265 | "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
266 | "24:\n" \ | |
267 | "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
268 | "SUB %3, %3, #32\n" \ | |
269 | "DCACHE [%1+#-64], D0Ar6\n" \ | |
270 | "BR $Lloop"id"\n" \ | |
271 | \ | |
272 | "MOV RAPF, %1\n" \ | |
273 | "25:\n" \ | |
274 | "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
275 | "26:\n" \ | |
276 | "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
277 | "SUB %3, %3, #32\n" \ | |
278 | "27:\n" \ | |
279 | "MGETL D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
280 | "28:\n" \ | |
281 | "MSETL [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
282 | "SUB %0, %0, #8\n" \ | |
283 | "29:\n" \ | |
284 | "SETL [%0++], D0.7, D1.7\n" \ | |
285 | "SUB %3, %3, #32\n" \ | |
286 | "1:" \ | |
287 | "DCACHE [%1+#-64], D0Ar6\n" \ | |
288 | "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \ | |
289 | "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \ | |
290 | "GETL D0.5, D1.5, [A0StP+#-24]\n" \ | |
291 | "GETL D0.6, D1.6, [A0StP+#-16]\n" \ | |
292 | "GETL D0.7, D1.7, [A0StP+#-8]\n" \ | |
293 | "SUB A0StP, A0StP, #40\n" \ | |
294 | " .section .fixup,\"ax\"\n" \ | |
295 | "4:\n" \ | |
296 | " ADD %0, %0, #8\n" \ | |
297 | "3:\n" \ | |
298 | " MOV D0Ar2, TXSTATUS\n" \ | |
299 | " MOV D1Ar1, TXSTATUS\n" \ | |
300 | " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \ | |
301 | " MOV TXSTATUS, D1Ar1\n" \ | |
302 | FIXUP \ | |
303 | " MOVT D0Ar2,#HI(1b)\n" \ | |
304 | " JUMP D0Ar2,#LO(1b)\n" \ | |
305 | " .previous\n" \ | |
306 | " .section __ex_table,\"a\"\n" \ | |
307 | " .long 21b,3b\n" \ | |
308 | " .long 22b,3b\n" \ | |
309 | " .long 23b,3b\n" \ | |
310 | " .long 24b,3b\n" \ | |
311 | " .long 25b,3b\n" \ | |
312 | " .long 26b,3b\n" \ | |
313 | " .long 27b,3b\n" \ | |
314 | " .long 28b,3b\n" \ | |
315 | " .long 29b,4b\n" \ | |
316 | " .previous\n" \ | |
317 | : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \ | |
318 | : "0" (to), "1" (from), "2" (ret), "3" (n) \ | |
319 | : "D1Ar1", "D0Ar2", "memory") | |
320 | ||
321 | /* rewind 'to' and 'from' pointers when a fault occurs | |
322 | * | |
323 | * Rationale: | |
324 | * A fault always occurs on writing to user buffer. A fault | |
325 | * is at a single address, so we need to rewind by only 4 | |
326 | * bytes. | |
327 | * Since we do a complete read from kernel buffer before | |
328 | * writing, we need to rewind it also. The amount to be | |
329 | * rewind equals the number of faulty writes in MSETD | |
330 | * which is: [4 - (LSM_STEP-1)]*8 | |
331 | * LSM_STEP is bits 10:8 in TXSTATUS which is already read | |
332 | * and stored in D0Ar2 | |
333 | * | |
334 | * NOTE: If a fault occurs at the last operation in M{G,S}ETL | |
335 | * LSM_STEP will be 0. ie: we do 4 writes in our case, if | |
336 | * a fault happens at the 4th write, LSM_STEP will be 0 | |
337 | * instead of 4. The code copes with that. | |
338 | * | |
339 | * n is updated by the number of successful writes, which is: | |
340 | * n = n - (LSM_STEP-1)*8 | |
341 | */ | |
342 | #define __asm_copy_to_user_64bit_rapf_loop(to, from, ret, n, id)\ | |
343 | __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \ | |
344 | "LSR D0Ar2, D0Ar2, #8\n" \ | |
345 | "AND D0Ar2, D0Ar2, #0x7\n" \ | |
346 | "ADDZ D0Ar2, D0Ar2, #4\n" \ | |
347 | "SUB D0Ar2, D0Ar2, #1\n" \ | |
348 | "MOV D1Ar1, #4\n" \ | |
349 | "SUB D0Ar2, D1Ar1, D0Ar2\n" \ | |
350 | "LSL D0Ar2, D0Ar2, #3\n" \ | |
351 | "LSL D1Ar1, D1Ar1, #3\n" \ | |
352 | "SUB D1Ar1, D1Ar1, D0Ar2\n" \ | |
353 | "SUB %0, %0, #8\n" \ | |
354 | "SUB %1, %1,D0Ar2\n" \ | |
355 | "SUB %3, %3, D1Ar1\n") | |
356 | ||
357 | /* | |
358 | * optimized copying loop using RAPF when 32 bit aligned | |
359 | * | |
360 | * n will be automatically decremented inside the loop | |
361 | * ret will be left intact. if error occurs we will rewind | |
362 | * so that the original non optimized code will fill up | |
363 | * this value correctly. | |
364 | * | |
365 | * on fault: | |
366 | * > n will hold total number of uncopied bytes | |
367 | * | |
368 | * > {'to','from'} will be rewind back so that | |
369 | * the non-optimized code will do the proper fix up | |
370 | * | |
371 | * DCACHE drops the cacheline which helps in reducing cache | |
372 | * pollution. | |
373 | * | |
374 | * We introduce an extra SETD at the end of the loop to | |
375 | * ensure we don't fall off the loop before we catch all | |
376 | * erros. | |
377 | * | |
378 | * NOTICE: | |
379 | * LSM_STEP in TXSTATUS must be cleared in fix up code. | |
380 | * since we're using M{S,G}ETL, a fault might happen at | |
381 | * any address in the middle of M{S,G}ETL causing | |
382 | * the value of LSM_STEP to be incorrect which can | |
383 | * cause subsequent use of M{S,G}ET{L,D} to go wrong. | |
384 | * ie: if LSM_STEP was 1 when a fault occurs, the | |
385 | * next call to M{S,G}ET{L,D} will skip the first | |
386 | * copy/getting as it think that the first 1 has already | |
387 | * been done. | |
388 | * | |
389 | */ | |
390 | #define __asm_copy_user_32bit_rapf_loop( \ | |
391 | to, from, ret, n, id, FIXUP) \ | |
392 | asm volatile ( \ | |
393 | ".balign 8\n" \ | |
394 | "MOV RAPF, %1\n" \ | |
395 | "MSETL [A0StP++], D0Ar6, D0FrT, D0.5, D0.6, D0.7\n" \ | |
396 | "MOV D0Ar6, #0\n" \ | |
397 | "LSR D1Ar5, %3, #6\n" \ | |
398 | "SUB TXRPT, D1Ar5, #2\n" \ | |
399 | "MOV RAPF, %1\n" \ | |
400 | "$Lloop"id":\n" \ | |
401 | "ADD RAPF, %1, #64\n" \ | |
402 | "21:\n" \ | |
403 | "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
404 | "22:\n" \ | |
405 | "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
406 | "SUB %3, %3, #16\n" \ | |
407 | "23:\n" \ | |
408 | "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
409 | "24:\n" \ | |
410 | "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
411 | "SUB %3, %3, #16\n" \ | |
412 | "25:\n" \ | |
413 | "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
414 | "26:\n" \ | |
415 | "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
416 | "SUB %3, %3, #16\n" \ | |
417 | "27:\n" \ | |
418 | "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
419 | "28:\n" \ | |
420 | "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
421 | "SUB %3, %3, #16\n" \ | |
422 | "DCACHE [%1+#-64], D0Ar6\n" \ | |
423 | "BR $Lloop"id"\n" \ | |
424 | \ | |
425 | "MOV RAPF, %1\n" \ | |
426 | "29:\n" \ | |
427 | "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
428 | "30:\n" \ | |
429 | "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
430 | "SUB %3, %3, #16\n" \ | |
431 | "31:\n" \ | |
432 | "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
433 | "32:\n" \ | |
434 | "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
435 | "SUB %3, %3, #16\n" \ | |
436 | "33:\n" \ | |
437 | "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
438 | "34:\n" \ | |
439 | "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
440 | "SUB %3, %3, #16\n" \ | |
441 | "35:\n" \ | |
442 | "MGETD D0FrT, D0.5, D0.6, D0.7, [%1++]\n" \ | |
443 | "36:\n" \ | |
444 | "MSETD [%0++], D0FrT, D0.5, D0.6, D0.7\n" \ | |
445 | "SUB %0, %0, #4\n" \ | |
446 | "37:\n" \ | |
447 | "SETD [%0++], D0.7\n" \ | |
448 | "SUB %3, %3, #16\n" \ | |
449 | "1:" \ | |
450 | "DCACHE [%1+#-64], D0Ar6\n" \ | |
451 | "GETL D0Ar6, D1Ar5, [A0StP+#-40]\n" \ | |
452 | "GETL D0FrT, D1RtP, [A0StP+#-32]\n" \ | |
453 | "GETL D0.5, D1.5, [A0StP+#-24]\n" \ | |
454 | "GETL D0.6, D1.6, [A0StP+#-16]\n" \ | |
455 | "GETL D0.7, D1.7, [A0StP+#-8]\n" \ | |
456 | "SUB A0StP, A0StP, #40\n" \ | |
457 | " .section .fixup,\"ax\"\n" \ | |
458 | "4:\n" \ | |
459 | " ADD %0, %0, #4\n" \ | |
460 | "3:\n" \ | |
461 | " MOV D0Ar2, TXSTATUS\n" \ | |
462 | " MOV D1Ar1, TXSTATUS\n" \ | |
463 | " AND D1Ar1, D1Ar1, #0xFFFFF8FF\n" \ | |
464 | " MOV TXSTATUS, D1Ar1\n" \ | |
465 | FIXUP \ | |
466 | " MOVT D0Ar2,#HI(1b)\n" \ | |
467 | " JUMP D0Ar2,#LO(1b)\n" \ | |
468 | " .previous\n" \ | |
469 | " .section __ex_table,\"a\"\n" \ | |
470 | " .long 21b,3b\n" \ | |
471 | " .long 22b,3b\n" \ | |
472 | " .long 23b,3b\n" \ | |
473 | " .long 24b,3b\n" \ | |
474 | " .long 25b,3b\n" \ | |
475 | " .long 26b,3b\n" \ | |
476 | " .long 27b,3b\n" \ | |
477 | " .long 28b,3b\n" \ | |
478 | " .long 29b,3b\n" \ | |
479 | " .long 30b,3b\n" \ | |
480 | " .long 31b,3b\n" \ | |
481 | " .long 32b,3b\n" \ | |
482 | " .long 33b,3b\n" \ | |
483 | " .long 34b,3b\n" \ | |
484 | " .long 35b,3b\n" \ | |
485 | " .long 36b,3b\n" \ | |
486 | " .long 37b,4b\n" \ | |
487 | " .previous\n" \ | |
488 | : "=r" (to), "=r" (from), "=r" (ret), "=d" (n) \ | |
489 | : "0" (to), "1" (from), "2" (ret), "3" (n) \ | |
490 | : "D1Ar1", "D0Ar2", "memory") | |
491 | ||
492 | /* rewind 'to' and 'from' pointers when a fault occurs | |
493 | * | |
494 | * Rationale: | |
495 | * A fault always occurs on writing to user buffer. A fault | |
496 | * is at a single address, so we need to rewind by only 4 | |
497 | * bytes. | |
498 | * Since we do a complete read from kernel buffer before | |
499 | * writing, we need to rewind it also. The amount to be | |
500 | * rewind equals the number of faulty writes in MSETD | |
501 | * which is: [4 - (LSM_STEP-1)]*4 | |
502 | * LSM_STEP is bits 10:8 in TXSTATUS which is already read | |
503 | * and stored in D0Ar2 | |
504 | * | |
505 | * NOTE: If a fault occurs at the last operation in M{G,S}ETL | |
506 | * LSM_STEP will be 0. ie: we do 4 writes in our case, if | |
507 | * a fault happens at the 4th write, LSM_STEP will be 0 | |
508 | * instead of 4. The code copes with that. | |
509 | * | |
510 | * n is updated by the number of successful writes, which is: | |
511 | * n = n - (LSM_STEP-1)*4 | |
512 | */ | |
513 | #define __asm_copy_to_user_32bit_rapf_loop(to, from, ret, n, id)\ | |
514 | __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \ | |
515 | "LSR D0Ar2, D0Ar2, #8\n" \ | |
516 | "AND D0Ar2, D0Ar2, #0x7\n" \ | |
517 | "ADDZ D0Ar2, D0Ar2, #4\n" \ | |
518 | "SUB D0Ar2, D0Ar2, #1\n" \ | |
519 | "MOV D1Ar1, #4\n" \ | |
520 | "SUB D0Ar2, D1Ar1, D0Ar2\n" \ | |
521 | "LSL D0Ar2, D0Ar2, #2\n" \ | |
522 | "LSL D1Ar1, D1Ar1, #2\n" \ | |
523 | "SUB D1Ar1, D1Ar1, D0Ar2\n" \ | |
524 | "SUB %0, %0, #4\n" \ | |
525 | "SUB %1, %1, D0Ar2\n" \ | |
526 | "SUB %3, %3, D1Ar1\n") | |
527 | ||
528 | unsigned long __copy_user(void __user *pdst, const void *psrc, | |
529 | unsigned long n) | |
530 | { | |
531 | register char __user *dst asm ("A0.2") = pdst; | |
532 | register const char *src asm ("A1.2") = psrc; | |
533 | unsigned long retn = 0; | |
534 | ||
535 | if (n == 0) | |
536 | return 0; | |
537 | ||
538 | if ((unsigned long) src & 1) { | |
539 | __asm_copy_to_user_1(dst, src, retn); | |
540 | n--; | |
541 | } | |
542 | if ((unsigned long) dst & 1) { | |
543 | /* Worst case - byte copy */ | |
544 | while (n > 0) { | |
545 | __asm_copy_to_user_1(dst, src, retn); | |
546 | n--; | |
547 | } | |
548 | } | |
549 | if (((unsigned long) src & 2) && n >= 2) { | |
550 | __asm_copy_to_user_2(dst, src, retn); | |
551 | n -= 2; | |
552 | } | |
553 | if ((unsigned long) dst & 2) { | |
554 | /* Second worst case - word copy */ | |
555 | while (n >= 2) { | |
556 | __asm_copy_to_user_2(dst, src, retn); | |
557 | n -= 2; | |
558 | } | |
559 | } | |
560 | ||
561 | #ifdef USE_RAPF | |
562 | /* 64 bit copy loop */ | |
563 | if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) { | |
564 | if (n >= RAPF_MIN_BUF_SIZE) { | |
565 | /* copy user using 64 bit rapf copy */ | |
566 | __asm_copy_to_user_64bit_rapf_loop(dst, src, retn, | |
567 | n, "64cu"); | |
568 | } | |
569 | while (n >= 8) { | |
570 | __asm_copy_to_user_8x64(dst, src, retn); | |
571 | n -= 8; | |
572 | } | |
573 | } | |
574 | if (n >= RAPF_MIN_BUF_SIZE) { | |
575 | /* copy user using 32 bit rapf copy */ | |
576 | __asm_copy_to_user_32bit_rapf_loop(dst, src, retn, n, "32cu"); | |
577 | } | |
578 | #else | |
579 | /* 64 bit copy loop */ | |
580 | if (!(((unsigned long) src | (__force unsigned long) dst) & 7)) { | |
581 | while (n >= 8) { | |
582 | __asm_copy_to_user_8x64(dst, src, retn); | |
583 | n -= 8; | |
584 | } | |
585 | } | |
586 | #endif | |
587 | ||
588 | while (n >= 16) { | |
589 | __asm_copy_to_user_16(dst, src, retn); | |
590 | n -= 16; | |
591 | } | |
592 | ||
593 | while (n >= 4) { | |
594 | __asm_copy_to_user_4(dst, src, retn); | |
595 | n -= 4; | |
596 | } | |
597 | ||
598 | switch (n) { | |
599 | case 0: | |
600 | break; | |
601 | case 1: | |
602 | __asm_copy_to_user_1(dst, src, retn); | |
603 | break; | |
604 | case 2: | |
605 | __asm_copy_to_user_2(dst, src, retn); | |
606 | break; | |
607 | case 3: | |
608 | __asm_copy_to_user_3(dst, src, retn); | |
609 | break; | |
610 | } | |
611 | ||
612 | return retn; | |
613 | } | |
9da3ee9a | 614 | EXPORT_SYMBOL(__copy_user); |
373cd784 JH |
615 | |
616 | #define __asm_copy_from_user_1(to, from, ret) \ | |
617 | __asm_copy_user_cont(to, from, ret, \ | |
618 | " GETB D1Ar1,[%1++]\n" \ | |
619 | "2: SETB [%0++],D1Ar1\n", \ | |
620 | "3: ADD %2,%2,#1\n" \ | |
621 | " SETB [%0++],D1Ar1\n", \ | |
622 | " .long 2b,3b\n") | |
623 | ||
624 | #define __asm_copy_from_user_2x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
625 | __asm_copy_user_cont(to, from, ret, \ | |
626 | " GETW D1Ar1,[%1++]\n" \ | |
627 | "2: SETW [%0++],D1Ar1\n" COPY, \ | |
628 | "3: ADD %2,%2,#2\n" \ | |
629 | " SETW [%0++],D1Ar1\n" FIXUP, \ | |
630 | " .long 2b,3b\n" TENTRY) | |
631 | ||
632 | #define __asm_copy_from_user_2(to, from, ret) \ | |
633 | __asm_copy_from_user_2x_cont(to, from, ret, "", "", "") | |
634 | ||
635 | #define __asm_copy_from_user_3(to, from, ret) \ | |
636 | __asm_copy_from_user_2x_cont(to, from, ret, \ | |
637 | " GETB D1Ar1,[%1++]\n" \ | |
638 | "4: SETB [%0++],D1Ar1\n", \ | |
639 | "5: ADD %2,%2,#1\n" \ | |
640 | " SETB [%0++],D1Ar1\n", \ | |
641 | " .long 4b,5b\n") | |
642 | ||
643 | #define __asm_copy_from_user_4x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
644 | __asm_copy_user_cont(to, from, ret, \ | |
645 | " GETD D1Ar1,[%1++]\n" \ | |
646 | "2: SETD [%0++],D1Ar1\n" COPY, \ | |
647 | "3: ADD %2,%2,#4\n" \ | |
648 | " SETD [%0++],D1Ar1\n" FIXUP, \ | |
649 | " .long 2b,3b\n" TENTRY) | |
650 | ||
651 | #define __asm_copy_from_user_4(to, from, ret) \ | |
652 | __asm_copy_from_user_4x_cont(to, from, ret, "", "", "") | |
653 | ||
654 | #define __asm_copy_from_user_5(to, from, ret) \ | |
655 | __asm_copy_from_user_4x_cont(to, from, ret, \ | |
656 | " GETB D1Ar1,[%1++]\n" \ | |
657 | "4: SETB [%0++],D1Ar1\n", \ | |
658 | "5: ADD %2,%2,#1\n" \ | |
659 | " SETB [%0++],D1Ar1\n", \ | |
660 | " .long 4b,5b\n") | |
661 | ||
662 | #define __asm_copy_from_user_6x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
663 | __asm_copy_from_user_4x_cont(to, from, ret, \ | |
664 | " GETW D1Ar1,[%1++]\n" \ | |
665 | "4: SETW [%0++],D1Ar1\n" COPY, \ | |
666 | "5: ADD %2,%2,#2\n" \ | |
667 | " SETW [%0++],D1Ar1\n" FIXUP, \ | |
668 | " .long 4b,5b\n" TENTRY) | |
669 | ||
670 | #define __asm_copy_from_user_6(to, from, ret) \ | |
671 | __asm_copy_from_user_6x_cont(to, from, ret, "", "", "") | |
672 | ||
673 | #define __asm_copy_from_user_7(to, from, ret) \ | |
674 | __asm_copy_from_user_6x_cont(to, from, ret, \ | |
675 | " GETB D1Ar1,[%1++]\n" \ | |
676 | "6: SETB [%0++],D1Ar1\n", \ | |
677 | "7: ADD %2,%2,#1\n" \ | |
678 | " SETB [%0++],D1Ar1\n", \ | |
679 | " .long 6b,7b\n") | |
680 | ||
681 | #define __asm_copy_from_user_8x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
682 | __asm_copy_from_user_4x_cont(to, from, ret, \ | |
683 | " GETD D1Ar1,[%1++]\n" \ | |
684 | "4: SETD [%0++],D1Ar1\n" COPY, \ | |
685 | "5: ADD %2,%2,#4\n" \ | |
686 | " SETD [%0++],D1Ar1\n" FIXUP, \ | |
687 | " .long 4b,5b\n" TENTRY) | |
688 | ||
689 | #define __asm_copy_from_user_8(to, from, ret) \ | |
690 | __asm_copy_from_user_8x_cont(to, from, ret, "", "", "") | |
691 | ||
692 | #define __asm_copy_from_user_9(to, from, ret) \ | |
693 | __asm_copy_from_user_8x_cont(to, from, ret, \ | |
694 | " GETB D1Ar1,[%1++]\n" \ | |
695 | "6: SETB [%0++],D1Ar1\n", \ | |
696 | "7: ADD %2,%2,#1\n" \ | |
697 | " SETB [%0++],D1Ar1\n", \ | |
698 | " .long 6b,7b\n") | |
699 | ||
700 | #define __asm_copy_from_user_10x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
701 | __asm_copy_from_user_8x_cont(to, from, ret, \ | |
702 | " GETW D1Ar1,[%1++]\n" \ | |
703 | "6: SETW [%0++],D1Ar1\n" COPY, \ | |
704 | "7: ADD %2,%2,#2\n" \ | |
705 | " SETW [%0++],D1Ar1\n" FIXUP, \ | |
706 | " .long 6b,7b\n" TENTRY) | |
707 | ||
708 | #define __asm_copy_from_user_10(to, from, ret) \ | |
709 | __asm_copy_from_user_10x_cont(to, from, ret, "", "", "") | |
710 | ||
711 | #define __asm_copy_from_user_11(to, from, ret) \ | |
712 | __asm_copy_from_user_10x_cont(to, from, ret, \ | |
713 | " GETB D1Ar1,[%1++]\n" \ | |
714 | "8: SETB [%0++],D1Ar1\n", \ | |
715 | "9: ADD %2,%2,#1\n" \ | |
716 | " SETB [%0++],D1Ar1\n", \ | |
717 | " .long 8b,9b\n") | |
718 | ||
719 | #define __asm_copy_from_user_12x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
720 | __asm_copy_from_user_8x_cont(to, from, ret, \ | |
721 | " GETD D1Ar1,[%1++]\n" \ | |
722 | "6: SETD [%0++],D1Ar1\n" COPY, \ | |
723 | "7: ADD %2,%2,#4\n" \ | |
724 | " SETD [%0++],D1Ar1\n" FIXUP, \ | |
725 | " .long 6b,7b\n" TENTRY) | |
726 | ||
727 | #define __asm_copy_from_user_12(to, from, ret) \ | |
728 | __asm_copy_from_user_12x_cont(to, from, ret, "", "", "") | |
729 | ||
730 | #define __asm_copy_from_user_13(to, from, ret) \ | |
731 | __asm_copy_from_user_12x_cont(to, from, ret, \ | |
732 | " GETB D1Ar1,[%1++]\n" \ | |
733 | "8: SETB [%0++],D1Ar1\n", \ | |
734 | "9: ADD %2,%2,#1\n" \ | |
735 | " SETB [%0++],D1Ar1\n", \ | |
736 | " .long 8b,9b\n") | |
737 | ||
738 | #define __asm_copy_from_user_14x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
739 | __asm_copy_from_user_12x_cont(to, from, ret, \ | |
740 | " GETW D1Ar1,[%1++]\n" \ | |
741 | "8: SETW [%0++],D1Ar1\n" COPY, \ | |
742 | "9: ADD %2,%2,#2\n" \ | |
743 | " SETW [%0++],D1Ar1\n" FIXUP, \ | |
744 | " .long 8b,9b\n" TENTRY) | |
745 | ||
746 | #define __asm_copy_from_user_14(to, from, ret) \ | |
747 | __asm_copy_from_user_14x_cont(to, from, ret, "", "", "") | |
748 | ||
749 | #define __asm_copy_from_user_15(to, from, ret) \ | |
750 | __asm_copy_from_user_14x_cont(to, from, ret, \ | |
751 | " GETB D1Ar1,[%1++]\n" \ | |
752 | "10: SETB [%0++],D1Ar1\n", \ | |
753 | "11: ADD %2,%2,#1\n" \ | |
754 | " SETB [%0++],D1Ar1\n", \ | |
755 | " .long 10b,11b\n") | |
756 | ||
757 | #define __asm_copy_from_user_16x_cont(to, from, ret, COPY, FIXUP, TENTRY) \ | |
758 | __asm_copy_from_user_12x_cont(to, from, ret, \ | |
759 | " GETD D1Ar1,[%1++]\n" \ | |
760 | "8: SETD [%0++],D1Ar1\n" COPY, \ | |
761 | "9: ADD %2,%2,#4\n" \ | |
762 | " SETD [%0++],D1Ar1\n" FIXUP, \ | |
763 | " .long 8b,9b\n" TENTRY) | |
764 | ||
765 | #define __asm_copy_from_user_16(to, from, ret) \ | |
766 | __asm_copy_from_user_16x_cont(to, from, ret, "", "", "") | |
767 | ||
768 | #define __asm_copy_from_user_8x64(to, from, ret) \ | |
769 | asm volatile ( \ | |
770 | " GETL D0Ar2,D1Ar1,[%1++]\n" \ | |
771 | "2: SETL [%0++],D0Ar2,D1Ar1\n" \ | |
772 | "1:\n" \ | |
773 | " .section .fixup,\"ax\"\n" \ | |
774 | " MOV D1Ar1,#0\n" \ | |
775 | " MOV D0Ar2,#0\n" \ | |
776 | "3: ADD %2,%2,#8\n" \ | |
777 | " SETL [%0++],D0Ar2,D1Ar1\n" \ | |
778 | " MOVT D0Ar2,#HI(1b)\n" \ | |
779 | " JUMP D0Ar2,#LO(1b)\n" \ | |
780 | " .previous\n" \ | |
781 | " .section __ex_table,\"a\"\n" \ | |
782 | " .long 2b,3b\n" \ | |
783 | " .previous\n" \ | |
784 | : "=a" (to), "=r" (from), "=r" (ret) \ | |
785 | : "0" (to), "1" (from), "2" (ret) \ | |
786 | : "D1Ar1", "D0Ar2", "memory") | |
787 | ||
788 | /* rewind 'from' pointer when a fault occurs | |
789 | * | |
790 | * Rationale: | |
791 | * A fault occurs while reading from user buffer, which is the | |
792 | * source. Since the fault is at a single address, we only | |
793 | * need to rewind by 8 bytes. | |
794 | * Since we don't write to kernel buffer until we read first, | |
795 | * the kernel buffer is at the right state and needn't be | |
796 | * corrected. | |
797 | */ | |
798 | #define __asm_copy_from_user_64bit_rapf_loop(to, from, ret, n, id) \ | |
799 | __asm_copy_user_64bit_rapf_loop(to, from, ret, n, id, \ | |
800 | "SUB %1, %1, #8\n") | |
801 | ||
802 | /* rewind 'from' pointer when a fault occurs | |
803 | * | |
804 | * Rationale: | |
805 | * A fault occurs while reading from user buffer, which is the | |
806 | * source. Since the fault is at a single address, we only | |
807 | * need to rewind by 4 bytes. | |
808 | * Since we don't write to kernel buffer until we read first, | |
809 | * the kernel buffer is at the right state and needn't be | |
810 | * corrected. | |
811 | */ | |
812 | #define __asm_copy_from_user_32bit_rapf_loop(to, from, ret, n, id) \ | |
813 | __asm_copy_user_32bit_rapf_loop(to, from, ret, n, id, \ | |
814 | "SUB %1, %1, #4\n") | |
815 | ||
816 | ||
817 | /* Copy from user to kernel, zeroing the bytes that were inaccessible in | |
818 | userland. The return-value is the number of bytes that were | |
819 | inaccessible. */ | |
820 | unsigned long __copy_user_zeroing(void *pdst, const void __user *psrc, | |
821 | unsigned long n) | |
822 | { | |
823 | register char *dst asm ("A0.2") = pdst; | |
824 | register const char __user *src asm ("A1.2") = psrc; | |
825 | unsigned long retn = 0; | |
826 | ||
827 | if (n == 0) | |
828 | return 0; | |
829 | ||
830 | if ((unsigned long) src & 1) { | |
831 | __asm_copy_from_user_1(dst, src, retn); | |
832 | n--; | |
833 | } | |
834 | if ((unsigned long) dst & 1) { | |
835 | /* Worst case - byte copy */ | |
836 | while (n > 0) { | |
837 | __asm_copy_from_user_1(dst, src, retn); | |
838 | n--; | |
839 | if (retn) | |
840 | goto copy_exception_bytes; | |
841 | } | |
842 | } | |
843 | if (((unsigned long) src & 2) && n >= 2) { | |
844 | __asm_copy_from_user_2(dst, src, retn); | |
845 | n -= 2; | |
846 | } | |
847 | if ((unsigned long) dst & 2) { | |
848 | /* Second worst case - word copy */ | |
849 | while (n >= 2) { | |
850 | __asm_copy_from_user_2(dst, src, retn); | |
851 | n -= 2; | |
852 | if (retn) | |
853 | goto copy_exception_bytes; | |
854 | } | |
855 | } | |
856 | ||
857 | /* We only need one check after the unalignment-adjustments, | |
858 | because if both adjustments were done, either both or | |
859 | neither reference had an exception. */ | |
860 | if (retn != 0) | |
861 | goto copy_exception_bytes; | |
862 | ||
863 | #ifdef USE_RAPF | |
864 | /* 64 bit copy loop */ | |
865 | if (!(((unsigned long) src | (unsigned long) dst) & 7)) { | |
866 | if (n >= RAPF_MIN_BUF_SIZE) { | |
867 | /* Copy using fast 64bit rapf */ | |
868 | __asm_copy_from_user_64bit_rapf_loop(dst, src, retn, | |
869 | n, "64cuz"); | |
870 | } | |
871 | while (n >= 8) { | |
872 | __asm_copy_from_user_8x64(dst, src, retn); | |
873 | n -= 8; | |
874 | if (retn) | |
875 | goto copy_exception_bytes; | |
876 | } | |
877 | } | |
878 | ||
879 | if (n >= RAPF_MIN_BUF_SIZE) { | |
880 | /* Copy using fast 32bit rapf */ | |
881 | __asm_copy_from_user_32bit_rapf_loop(dst, src, retn, | |
882 | n, "32cuz"); | |
883 | } | |
884 | #else | |
885 | /* 64 bit copy loop */ | |
886 | if (!(((unsigned long) src | (unsigned long) dst) & 7)) { | |
887 | while (n >= 8) { | |
888 | __asm_copy_from_user_8x64(dst, src, retn); | |
889 | n -= 8; | |
890 | if (retn) | |
891 | goto copy_exception_bytes; | |
892 | } | |
893 | } | |
894 | #endif | |
895 | ||
896 | while (n >= 4) { | |
897 | __asm_copy_from_user_4(dst, src, retn); | |
898 | n -= 4; | |
899 | ||
900 | if (retn) | |
901 | goto copy_exception_bytes; | |
902 | } | |
903 | ||
904 | /* If we get here, there were no memory read faults. */ | |
905 | switch (n) { | |
906 | /* These copies are at least "naturally aligned" (so we don't | |
907 | have to check each byte), due to the src alignment code. | |
908 | The *_3 case *will* get the correct count for retn. */ | |
909 | case 0: | |
910 | /* This case deliberately left in (if you have doubts check the | |
911 | generated assembly code). */ | |
912 | break; | |
913 | case 1: | |
914 | __asm_copy_from_user_1(dst, src, retn); | |
915 | break; | |
916 | case 2: | |
917 | __asm_copy_from_user_2(dst, src, retn); | |
918 | break; | |
919 | case 3: | |
920 | __asm_copy_from_user_3(dst, src, retn); | |
921 | break; | |
922 | } | |
923 | ||
924 | /* If we get here, retn correctly reflects the number of failing | |
925 | bytes. */ | |
926 | return retn; | |
927 | ||
928 | copy_exception_bytes: | |
929 | /* We already have "retn" bytes cleared, and need to clear the | |
930 | remaining "n" bytes. A non-optimized simple byte-for-byte in-line | |
931 | memset is preferred here, since this isn't speed-critical code and | |
932 | we'd rather have this a leaf-function than calling memset. */ | |
933 | { | |
934 | char *endp; | |
935 | for (endp = dst + n; dst < endp; dst++) | |
936 | *dst = 0; | |
937 | } | |
938 | ||
939 | return retn + n; | |
940 | } | |
9da3ee9a | 941 | EXPORT_SYMBOL(__copy_user_zeroing); |
373cd784 JH |
942 | |
943 | #define __asm_clear_8x64(to, ret) \ | |
944 | asm volatile ( \ | |
945 | " MOV D0Ar2,#0\n" \ | |
946 | " MOV D1Ar1,#0\n" \ | |
947 | " SETL [%0],D0Ar2,D1Ar1\n" \ | |
948 | "2: SETL [%0++],D0Ar2,D1Ar1\n" \ | |
949 | "1:\n" \ | |
950 | " .section .fixup,\"ax\"\n" \ | |
951 | "3: ADD %1,%1,#8\n" \ | |
952 | " MOVT D0Ar2,#HI(1b)\n" \ | |
953 | " JUMP D0Ar2,#LO(1b)\n" \ | |
954 | " .previous\n" \ | |
955 | " .section __ex_table,\"a\"\n" \ | |
956 | " .long 2b,3b\n" \ | |
957 | " .previous\n" \ | |
958 | : "=r" (to), "=r" (ret) \ | |
959 | : "0" (to), "1" (ret) \ | |
960 | : "D1Ar1", "D0Ar2", "memory") | |
961 | ||
962 | /* Zero userspace. */ | |
963 | ||
964 | #define __asm_clear(to, ret, CLEAR, FIXUP, TENTRY) \ | |
965 | asm volatile ( \ | |
966 | " MOV D1Ar1,#0\n" \ | |
967 | CLEAR \ | |
968 | "1:\n" \ | |
969 | " .section .fixup,\"ax\"\n" \ | |
970 | FIXUP \ | |
971 | " MOVT D1Ar1,#HI(1b)\n" \ | |
972 | " JUMP D1Ar1,#LO(1b)\n" \ | |
973 | " .previous\n" \ | |
974 | " .section __ex_table,\"a\"\n" \ | |
975 | TENTRY \ | |
976 | " .previous" \ | |
977 | : "=r" (to), "=r" (ret) \ | |
978 | : "0" (to), "1" (ret) \ | |
979 | : "D1Ar1", "memory") | |
980 | ||
981 | #define __asm_clear_1(to, ret) \ | |
982 | __asm_clear(to, ret, \ | |
983 | " SETB [%0],D1Ar1\n" \ | |
984 | "2: SETB [%0++],D1Ar1\n", \ | |
985 | "3: ADD %1,%1,#1\n", \ | |
986 | " .long 2b,3b\n") | |
987 | ||
988 | #define __asm_clear_2(to, ret) \ | |
989 | __asm_clear(to, ret, \ | |
990 | " SETW [%0],D1Ar1\n" \ | |
991 | "2: SETW [%0++],D1Ar1\n", \ | |
992 | "3: ADD %1,%1,#2\n", \ | |
993 | " .long 2b,3b\n") | |
994 | ||
995 | #define __asm_clear_3(to, ret) \ | |
996 | __asm_clear(to, ret, \ | |
997 | "2: SETW [%0++],D1Ar1\n" \ | |
998 | " SETB [%0],D1Ar1\n" \ | |
999 | "3: SETB [%0++],D1Ar1\n", \ | |
1000 | "4: ADD %1,%1,#2\n" \ | |
1001 | "5: ADD %1,%1,#1\n", \ | |
1002 | " .long 2b,4b\n" \ | |
1003 | " .long 3b,5b\n") | |
1004 | ||
1005 | #define __asm_clear_4x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ | |
1006 | __asm_clear(to, ret, \ | |
1007 | " SETD [%0],D1Ar1\n" \ | |
1008 | "2: SETD [%0++],D1Ar1\n" CLEAR, \ | |
1009 | "3: ADD %1,%1,#4\n" FIXUP, \ | |
1010 | " .long 2b,3b\n" TENTRY) | |
1011 | ||
1012 | #define __asm_clear_4(to, ret) \ | |
1013 | __asm_clear_4x_cont(to, ret, "", "", "") | |
1014 | ||
1015 | #define __asm_clear_8x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ | |
1016 | __asm_clear_4x_cont(to, ret, \ | |
1017 | " SETD [%0],D1Ar1\n" \ | |
1018 | "4: SETD [%0++],D1Ar1\n" CLEAR, \ | |
1019 | "5: ADD %1,%1,#4\n" FIXUP, \ | |
1020 | " .long 4b,5b\n" TENTRY) | |
1021 | ||
1022 | #define __asm_clear_8(to, ret) \ | |
1023 | __asm_clear_8x_cont(to, ret, "", "", "") | |
1024 | ||
1025 | #define __asm_clear_12x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ | |
1026 | __asm_clear_8x_cont(to, ret, \ | |
1027 | " SETD [%0],D1Ar1\n" \ | |
1028 | "6: SETD [%0++],D1Ar1\n" CLEAR, \ | |
1029 | "7: ADD %1,%1,#4\n" FIXUP, \ | |
1030 | " .long 6b,7b\n" TENTRY) | |
1031 | ||
1032 | #define __asm_clear_12(to, ret) \ | |
1033 | __asm_clear_12x_cont(to, ret, "", "", "") | |
1034 | ||
1035 | #define __asm_clear_16x_cont(to, ret, CLEAR, FIXUP, TENTRY) \ | |
1036 | __asm_clear_12x_cont(to, ret, \ | |
1037 | " SETD [%0],D1Ar1\n" \ | |
1038 | "8: SETD [%0++],D1Ar1\n" CLEAR, \ | |
1039 | "9: ADD %1,%1,#4\n" FIXUP, \ | |
1040 | " .long 8b,9b\n" TENTRY) | |
1041 | ||
1042 | #define __asm_clear_16(to, ret) \ | |
1043 | __asm_clear_16x_cont(to, ret, "", "", "") | |
1044 | ||
1045 | unsigned long __do_clear_user(void __user *pto, unsigned long pn) | |
1046 | { | |
1047 | register char __user *dst asm ("D0Re0") = pto; | |
1048 | register unsigned long n asm ("D1Re0") = pn; | |
1049 | register unsigned long retn asm ("D0Ar6") = 0; | |
1050 | ||
1051 | if ((unsigned long) dst & 1) { | |
1052 | __asm_clear_1(dst, retn); | |
1053 | n--; | |
1054 | } | |
1055 | ||
1056 | if ((unsigned long) dst & 2) { | |
1057 | __asm_clear_2(dst, retn); | |
1058 | n -= 2; | |
1059 | } | |
1060 | ||
1061 | /* 64 bit copy loop */ | |
1062 | if (!((__force unsigned long) dst & 7)) { | |
1063 | while (n >= 8) { | |
1064 | __asm_clear_8x64(dst, retn); | |
1065 | n -= 8; | |
1066 | } | |
1067 | } | |
1068 | ||
1069 | while (n >= 16) { | |
1070 | __asm_clear_16(dst, retn); | |
1071 | n -= 16; | |
1072 | } | |
1073 | ||
1074 | while (n >= 4) { | |
1075 | __asm_clear_4(dst, retn); | |
1076 | n -= 4; | |
1077 | } | |
1078 | ||
1079 | switch (n) { | |
1080 | case 0: | |
1081 | break; | |
1082 | case 1: | |
1083 | __asm_clear_1(dst, retn); | |
1084 | break; | |
1085 | case 2: | |
1086 | __asm_clear_2(dst, retn); | |
1087 | break; | |
1088 | case 3: | |
1089 | __asm_clear_3(dst, retn); | |
1090 | break; | |
1091 | } | |
1092 | ||
1093 | return retn; | |
1094 | } | |
9da3ee9a | 1095 | EXPORT_SYMBOL(__do_clear_user); |
373cd784 JH |
1096 | |
1097 | unsigned char __get_user_asm_b(const void __user *addr, long *err) | |
1098 | { | |
1099 | register unsigned char x asm ("D0Re0") = 0; | |
1100 | asm volatile ( | |
1101 | " GETB %0,[%2]\n" | |
1102 | "1:\n" | |
1103 | " GETB %0,[%2]\n" | |
1104 | "2:\n" | |
1105 | " .section .fixup,\"ax\"\n" | |
1106 | "3: MOV D0FrT,%3\n" | |
1107 | " SETD [%1],D0FrT\n" | |
1108 | " MOVT D0FrT,#HI(2b)\n" | |
1109 | " JUMP D0FrT,#LO(2b)\n" | |
1110 | " .previous\n" | |
1111 | " .section __ex_table,\"a\"\n" | |
1112 | " .long 1b,3b\n" | |
1113 | " .previous\n" | |
1114 | : "=r" (x) | |
1115 | : "r" (err), "r" (addr), "P" (-EFAULT) | |
1116 | : "D0FrT"); | |
1117 | return x; | |
1118 | } | |
9da3ee9a | 1119 | EXPORT_SYMBOL(__get_user_asm_b); |
373cd784 JH |
1120 | |
1121 | unsigned short __get_user_asm_w(const void __user *addr, long *err) | |
1122 | { | |
1123 | register unsigned short x asm ("D0Re0") = 0; | |
1124 | asm volatile ( | |
1125 | " GETW %0,[%2]\n" | |
1126 | "1:\n" | |
1127 | " GETW %0,[%2]\n" | |
1128 | "2:\n" | |
1129 | " .section .fixup,\"ax\"\n" | |
1130 | "3: MOV D0FrT,%3\n" | |
1131 | " SETD [%1],D0FrT\n" | |
1132 | " MOVT D0FrT,#HI(2b)\n" | |
1133 | " JUMP D0FrT,#LO(2b)\n" | |
1134 | " .previous\n" | |
1135 | " .section __ex_table,\"a\"\n" | |
1136 | " .long 1b,3b\n" | |
1137 | " .previous\n" | |
1138 | : "=r" (x) | |
1139 | : "r" (err), "r" (addr), "P" (-EFAULT) | |
1140 | : "D0FrT"); | |
1141 | return x; | |
1142 | } | |
9da3ee9a | 1143 | EXPORT_SYMBOL(__get_user_asm_w); |
373cd784 JH |
1144 | |
1145 | unsigned int __get_user_asm_d(const void __user *addr, long *err) | |
1146 | { | |
1147 | register unsigned int x asm ("D0Re0") = 0; | |
1148 | asm volatile ( | |
1149 | " GETD %0,[%2]\n" | |
1150 | "1:\n" | |
1151 | " GETD %0,[%2]\n" | |
1152 | "2:\n" | |
1153 | " .section .fixup,\"ax\"\n" | |
1154 | "3: MOV D0FrT,%3\n" | |
1155 | " SETD [%1],D0FrT\n" | |
1156 | " MOVT D0FrT,#HI(2b)\n" | |
1157 | " JUMP D0FrT,#LO(2b)\n" | |
1158 | " .previous\n" | |
1159 | " .section __ex_table,\"a\"\n" | |
1160 | " .long 1b,3b\n" | |
1161 | " .previous\n" | |
1162 | : "=r" (x) | |
1163 | : "r" (err), "r" (addr), "P" (-EFAULT) | |
1164 | : "D0FrT"); | |
1165 | return x; | |
1166 | } | |
9da3ee9a | 1167 | EXPORT_SYMBOL(__get_user_asm_d); |
373cd784 JH |
1168 | |
1169 | long __put_user_asm_b(unsigned int x, void __user *addr) | |
1170 | { | |
1171 | register unsigned int err asm ("D0Re0") = 0; | |
1172 | asm volatile ( | |
1173 | " MOV %0,#0\n" | |
1174 | " SETB [%2],%1\n" | |
1175 | "1:\n" | |
1176 | " SETB [%2],%1\n" | |
1177 | "2:\n" | |
1178 | ".section .fixup,\"ax\"\n" | |
1179 | "3: MOV %0,%3\n" | |
1180 | " MOVT D0FrT,#HI(2b)\n" | |
1181 | " JUMP D0FrT,#LO(2b)\n" | |
1182 | ".previous\n" | |
1183 | ".section __ex_table,\"a\"\n" | |
1184 | " .long 1b,3b\n" | |
1185 | ".previous" | |
1186 | : "=r"(err) | |
1187 | : "d" (x), "a" (addr), "P"(-EFAULT) | |
1188 | : "D0FrT"); | |
1189 | return err; | |
1190 | } | |
9da3ee9a | 1191 | EXPORT_SYMBOL(__put_user_asm_b); |
373cd784 JH |
1192 | |
1193 | long __put_user_asm_w(unsigned int x, void __user *addr) | |
1194 | { | |
1195 | register unsigned int err asm ("D0Re0") = 0; | |
1196 | asm volatile ( | |
1197 | " MOV %0,#0\n" | |
1198 | " SETW [%2],%1\n" | |
1199 | "1:\n" | |
1200 | " SETW [%2],%1\n" | |
1201 | "2:\n" | |
1202 | ".section .fixup,\"ax\"\n" | |
1203 | "3: MOV %0,%3\n" | |
1204 | " MOVT D0FrT,#HI(2b)\n" | |
1205 | " JUMP D0FrT,#LO(2b)\n" | |
1206 | ".previous\n" | |
1207 | ".section __ex_table,\"a\"\n" | |
1208 | " .long 1b,3b\n" | |
1209 | ".previous" | |
1210 | : "=r"(err) | |
1211 | : "d" (x), "a" (addr), "P"(-EFAULT) | |
1212 | : "D0FrT"); | |
1213 | return err; | |
1214 | } | |
9da3ee9a | 1215 | EXPORT_SYMBOL(__put_user_asm_w); |
373cd784 JH |
1216 | |
1217 | long __put_user_asm_d(unsigned int x, void __user *addr) | |
1218 | { | |
1219 | register unsigned int err asm ("D0Re0") = 0; | |
1220 | asm volatile ( | |
1221 | " MOV %0,#0\n" | |
1222 | " SETD [%2],%1\n" | |
1223 | "1:\n" | |
1224 | " SETD [%2],%1\n" | |
1225 | "2:\n" | |
1226 | ".section .fixup,\"ax\"\n" | |
1227 | "3: MOV %0,%3\n" | |
1228 | " MOVT D0FrT,#HI(2b)\n" | |
1229 | " JUMP D0FrT,#LO(2b)\n" | |
1230 | ".previous\n" | |
1231 | ".section __ex_table,\"a\"\n" | |
1232 | " .long 1b,3b\n" | |
1233 | ".previous" | |
1234 | : "=r"(err) | |
1235 | : "d" (x), "a" (addr), "P"(-EFAULT) | |
1236 | : "D0FrT"); | |
1237 | return err; | |
1238 | } | |
9da3ee9a | 1239 | EXPORT_SYMBOL(__put_user_asm_d); |
373cd784 JH |
1240 | |
1241 | long __put_user_asm_l(unsigned long long x, void __user *addr) | |
1242 | { | |
1243 | register unsigned int err asm ("D0Re0") = 0; | |
1244 | asm volatile ( | |
1245 | " MOV %0,#0\n" | |
1246 | " SETL [%2],%1,%t1\n" | |
1247 | "1:\n" | |
1248 | " SETL [%2],%1,%t1\n" | |
1249 | "2:\n" | |
1250 | ".section .fixup,\"ax\"\n" | |
1251 | "3: MOV %0,%3\n" | |
1252 | " MOVT D0FrT,#HI(2b)\n" | |
1253 | " JUMP D0FrT,#LO(2b)\n" | |
1254 | ".previous\n" | |
1255 | ".section __ex_table,\"a\"\n" | |
1256 | " .long 1b,3b\n" | |
1257 | ".previous" | |
1258 | : "=r"(err) | |
1259 | : "d" (x), "a" (addr), "P"(-EFAULT) | |
1260 | : "D0FrT"); | |
1261 | return err; | |
1262 | } | |
9da3ee9a | 1263 | EXPORT_SYMBOL(__put_user_asm_l); |
373cd784 JH |
1264 | |
1265 | long strnlen_user(const char __user *src, long count) | |
1266 | { | |
1267 | long res; | |
1268 | ||
1269 | if (!access_ok(VERIFY_READ, src, 0)) | |
1270 | return 0; | |
1271 | ||
1272 | asm volatile (" MOV D0Ar4, %1\n" | |
1273 | " MOV D0Ar6, %2\n" | |
1274 | "0:\n" | |
1275 | " SUBS D0FrT, D0Ar6, #0\n" | |
1276 | " SUB D0Ar6, D0Ar6, #1\n" | |
1277 | " BLE 2f\n" | |
1278 | " GETB D0FrT, [D0Ar4+#1++]\n" | |
1279 | "1:\n" | |
1280 | " TST D0FrT, #255\n" | |
1281 | " BNE 0b\n" | |
1282 | "2:\n" | |
1283 | " SUB %0, %2, D0Ar6\n" | |
1284 | "3:\n" | |
1285 | " .section .fixup,\"ax\"\n" | |
1286 | "4:\n" | |
1287 | " MOV %0, #0\n" | |
1288 | " MOVT D0FrT,#HI(3b)\n" | |
1289 | " JUMP D0FrT,#LO(3b)\n" | |
1290 | " .previous\n" | |
1291 | " .section __ex_table,\"a\"\n" | |
1292 | " .long 1b,4b\n" | |
1293 | " .previous\n" | |
1294 | : "=r" (res) | |
1295 | : "r" (src), "r" (count) | |
1296 | : "D0FrT", "D0Ar4", "D0Ar6", "cc"); | |
1297 | ||
1298 | return res; | |
1299 | } | |
9da3ee9a | 1300 | EXPORT_SYMBOL(strnlen_user); |
373cd784 JH |
1301 | |
1302 | long __strncpy_from_user(char *dst, const char __user *src, long count) | |
1303 | { | |
1304 | long res; | |
1305 | ||
1306 | if (count == 0) | |
1307 | return 0; | |
1308 | ||
1309 | /* | |
1310 | * Currently, in 2.4.0-test9, most ports use a simple byte-copy loop. | |
1311 | * So do we. | |
1312 | * | |
1313 | * This code is deduced from: | |
1314 | * | |
1315 | * char tmp2; | |
1316 | * long tmp1, tmp3; | |
1317 | * tmp1 = count; | |
1318 | * while ((*dst++ = (tmp2 = *src++)) != 0 | |
1319 | * && --tmp1) | |
1320 | * ; | |
1321 | * | |
1322 | * res = count - tmp1; | |
1323 | * | |
1324 | * with tweaks. | |
1325 | */ | |
1326 | ||
1327 | asm volatile (" MOV %0,%3\n" | |
1328 | "1:\n" | |
1329 | " GETB D0FrT,[%2++]\n" | |
1330 | "2:\n" | |
1331 | " CMP D0FrT,#0\n" | |
1332 | " SETB [%1++],D0FrT\n" | |
1333 | " BEQ 3f\n" | |
1334 | " SUBS %0,%0,#1\n" | |
1335 | " BNZ 1b\n" | |
1336 | "3:\n" | |
1337 | " SUB %0,%3,%0\n" | |
1338 | "4:\n" | |
1339 | " .section .fixup,\"ax\"\n" | |
1340 | "5:\n" | |
1341 | " MOV %0,%7\n" | |
1342 | " MOVT D0FrT,#HI(4b)\n" | |
1343 | " JUMP D0FrT,#LO(4b)\n" | |
1344 | " .previous\n" | |
1345 | " .section __ex_table,\"a\"\n" | |
1346 | " .long 2b,5b\n" | |
1347 | " .previous" | |
1348 | : "=r" (res), "=r" (dst), "=r" (src), "=r" (count) | |
1349 | : "3" (count), "1" (dst), "2" (src), "P" (-EFAULT) | |
1350 | : "D0FrT", "memory", "cc"); | |
1351 | ||
1352 | return res; | |
1353 | } | |
9da3ee9a | 1354 | EXPORT_SYMBOL(__strncpy_from_user); |