Commit | Line | Data |
---|---|---|
867e359b CM |
1 | /* |
2 | * Copyright 2010 Tilera Corporation. All Rights Reserved. | |
3 | * | |
4 | * This program is free software; you can redistribute it and/or | |
5 | * modify it under the terms of the GNU General Public License | |
6 | * as published by the Free Software Foundation, version 2. | |
7 | * | |
8 | * This program is distributed in the hope that it will be useful, but | |
9 | * WITHOUT ANY WARRANTY; without even the implied warranty of | |
10 | * MERCHANTABILITY OR FITNESS FOR A PARTICULAR PURPOSE, GOOD TITLE or | |
11 | * NON INFRINGEMENT. See the GNU General Public License for | |
12 | * more details. | |
867e359b CM |
13 | */ |
14 | ||
15 | #include <arch/chip.h> | |
16 | ||
867e359b | 17 | |
29507663 CM |
18 | /* |
19 | * This file shares the implementation of the userspace memcpy and | |
20 | * the kernel's memcpy, copy_to_user and copy_from_user. | |
21 | */ | |
22 | ||
867e359b CM |
23 | #include <linux/linkage.h> |
24 | ||
867e359b CM |
25 | #define IS_MEMCPY 0 |
26 | #define IS_COPY_FROM_USER 1 | |
27 | #define IS_COPY_FROM_USER_ZEROING 2 | |
28 | #define IS_COPY_TO_USER -1 | |
29 | ||
30 | .section .text.memcpy_common, "ax" | |
31 | .align 64 | |
32 | ||
33 | /* Use this to preface each bundle that can cause an exception so | |
34 | * the kernel can clean up properly. The special cleanup code should | |
35 | * not use these, since it knows what it is doing. | |
36 | */ | |
37 | #define EX \ | |
38 | .pushsection __ex_table, "a"; \ | |
d4d9eab4 | 39 | .align 4; \ |
867e359b CM |
40 | .word 9f, memcpy_common_fixup; \ |
41 | .popsection; \ | |
42 | 9 | |
43 | ||
44 | ||
45 | /* __copy_from_user_inatomic takes the kernel target address in r0, | |
46 | * the user source in r1, and the bytes to copy in r2. | |
47 | * It returns the number of uncopiable bytes (hopefully zero) in r0. | |
48 | */ | |
49 | ENTRY(__copy_from_user_inatomic) | |
50 | .type __copy_from_user_inatomic, @function | |
29507663 | 51 | FEEDBACK_ENTER_EXPLICIT(__copy_from_user_inatomic, \ |
867e359b | 52 | .text.memcpy_common, \ |
29507663 | 53 | .Lend_memcpy_common - __copy_from_user_inatomic) |
867e359b CM |
54 | { movei r29, IS_COPY_FROM_USER; j memcpy_common } |
55 | .size __copy_from_user_inatomic, . - __copy_from_user_inatomic | |
56 | ||
57 | /* __copy_from_user_zeroing is like __copy_from_user_inatomic, but | |
58 | * any uncopiable bytes are zeroed in the target. | |
59 | */ | |
60 | ENTRY(__copy_from_user_zeroing) | |
61 | .type __copy_from_user_zeroing, @function | |
29507663 | 62 | FEEDBACK_REENTER(__copy_from_user_inatomic) |
867e359b CM |
63 | { movei r29, IS_COPY_FROM_USER_ZEROING; j memcpy_common } |
64 | .size __copy_from_user_zeroing, . - __copy_from_user_zeroing | |
65 | ||
66 | /* __copy_to_user_inatomic takes the user target address in r0, | |
67 | * the kernel source in r1, and the bytes to copy in r2. | |
68 | * It returns the number of uncopiable bytes (hopefully zero) in r0. | |
69 | */ | |
70 | ENTRY(__copy_to_user_inatomic) | |
71 | .type __copy_to_user_inatomic, @function | |
29507663 | 72 | FEEDBACK_REENTER(__copy_from_user_inatomic) |
867e359b CM |
73 | { movei r29, IS_COPY_TO_USER; j memcpy_common } |
74 | .size __copy_to_user_inatomic, . - __copy_to_user_inatomic | |
75 | ||
76 | ENTRY(memcpy) | |
77 | .type memcpy, @function | |
29507663 | 78 | FEEDBACK_REENTER(__copy_from_user_inatomic) |
867e359b CM |
79 | { movei r29, IS_MEMCPY } |
80 | .size memcpy, . - memcpy | |
81 | /* Fall through */ | |
82 | ||
83 | .type memcpy_common, @function | |
84 | memcpy_common: | |
85 | /* On entry, r29 holds one of the IS_* macro values from above. */ | |
86 | ||
87 | ||
88 | /* r0 is the dest, r1 is the source, r2 is the size. */ | |
89 | ||
90 | /* Save aside original dest so we can return it at the end. */ | |
91 | { sw sp, lr; move r23, r0; or r4, r0, r1 } | |
92 | ||
93 | /* Check for an empty size. */ | |
94 | { bz r2, .Ldone; andi r4, r4, 3 } | |
95 | ||
96 | /* Save aside original values in case of a fault. */ | |
97 | { move r24, r1; move r25, r2 } | |
98 | move r27, lr | |
99 | ||
100 | /* Check for an unaligned source or dest. */ | |
101 | { bnz r4, .Lcopy_unaligned_maybe_many; addli r4, r2, -256 } | |
102 | ||
103 | .Lcheck_aligned_copy_size: | |
104 | /* If we are copying < 256 bytes, branch to simple case. */ | |
105 | { blzt r4, .Lcopy_8_check; slti_u r8, r2, 8 } | |
106 | ||
107 | /* Copying >= 256 bytes, so jump to complex prefetching loop. */ | |
108 | { andi r6, r1, 63; j .Lcopy_many } | |
109 | ||
110 | /* | |
111 | * | |
112 | * Aligned 4 byte at a time copy loop | |
113 | * | |
114 | */ | |
115 | ||
116 | .Lcopy_8_loop: | |
117 | /* Copy two words at a time to hide load latency. */ | |
118 | EX: { lw r3, r1; addi r1, r1, 4; slti_u r8, r2, 16 } | |
119 | EX: { lw r4, r1; addi r1, r1, 4 } | |
120 | EX: { sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 } | |
121 | EX: { sw r0, r4; addi r0, r0, 4; addi r2, r2, -4 } | |
122 | .Lcopy_8_check: | |
123 | { bzt r8, .Lcopy_8_loop; slti_u r4, r2, 4 } | |
124 | ||
125 | /* Copy odd leftover word, if any. */ | |
126 | { bnzt r4, .Lcheck_odd_stragglers } | |
127 | EX: { lw r3, r1; addi r1, r1, 4 } | |
128 | EX: { sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 } | |
129 | ||
130 | .Lcheck_odd_stragglers: | |
131 | { bnz r2, .Lcopy_unaligned_few } | |
132 | ||
133 | .Ldone: | |
134 | /* For memcpy return original dest address, else zero. */ | |
135 | { mz r0, r29, r23; jrp lr } | |
136 | ||
137 | ||
138 | /* | |
139 | * | |
140 | * Prefetching multiple cache line copy handler (for large transfers). | |
141 | * | |
142 | */ | |
143 | ||
144 | /* Copy words until r1 is cache-line-aligned. */ | |
145 | .Lalign_loop: | |
146 | EX: { lw r3, r1; addi r1, r1, 4 } | |
147 | { andi r6, r1, 63 } | |
148 | EX: { sw r0, r3; addi r0, r0, 4; addi r2, r2, -4 } | |
149 | .Lcopy_many: | |
150 | { bnzt r6, .Lalign_loop; addi r9, r0, 63 } | |
151 | ||
152 | { addi r3, r1, 60; andi r9, r9, -64 } | |
153 | ||
29507663 CM |
154 | /* No need to prefetch dst, we'll just do the wh64 |
155 | * right before we copy a line. | |
867e359b | 156 | */ |
867e359b | 157 | EX: { lw r5, r3; addi r3, r3, 64; movei r4, 1 } |
29507663 CM |
158 | /* Intentionally stall for a few cycles to leave L2 cache alone. */ |
159 | { bnzt zero, .; move r27, lr } | |
867e359b | 160 | EX: { lw r6, r3; addi r3, r3, 64 } |
29507663 CM |
161 | /* Intentionally stall for a few cycles to leave L2 cache alone. */ |
162 | { bnzt zero, . } | |
867e359b | 163 | EX: { lw r7, r3; addi r3, r3, 64 } |
29507663 CM |
164 | /* Intentionally stall for a few cycles to leave L2 cache alone. */ |
165 | { bz zero, .Lbig_loop2 } | |
867e359b CM |
166 | |
167 | /* On entry to this loop: | |
168 | * - r0 points to the start of dst line 0 | |
169 | * - r1 points to start of src line 0 | |
170 | * - r2 >= (256 - 60), only the first time the loop trips. | |
171 | * - r3 contains r1 + 128 + 60 [pointer to end of source line 2] | |
172 | * This is our prefetch address. When we get near the end | |
173 | * rather than prefetching off the end this is changed to point | |
174 | * to some "safe" recently loaded address. | |
175 | * - r5 contains *(r1 + 60) [i.e. last word of source line 0] | |
176 | * - r6 contains *(r1 + 64 + 60) [i.e. last word of source line 1] | |
29507663 | 177 | * - r9 contains ((r0 + 63) & -64) |
867e359b CM |
178 | * [start of next dst cache line.] |
179 | */ | |
180 | ||
181 | .Lbig_loop: | |
182 | { jal .Lcopy_line2; add r15, r1, r2 } | |
183 | ||
184 | .Lbig_loop2: | |
185 | /* Copy line 0, first stalling until r5 is ready. */ | |
186 | EX: { move r12, r5; lw r16, r1 } | |
187 | { bz r4, .Lcopy_8_check; slti_u r8, r2, 8 } | |
29507663 | 188 | /* Prefetch several lines ahead. */ |
867e359b | 189 | EX: { lw r5, r3; addi r3, r3, 64 } |
29507663 | 190 | { jal .Lcopy_line } |
867e359b CM |
191 | |
192 | /* Copy line 1, first stalling until r6 is ready. */ | |
193 | EX: { move r12, r6; lw r16, r1 } | |
194 | { bz r4, .Lcopy_8_check; slti_u r8, r2, 8 } | |
29507663 | 195 | /* Prefetch several lines ahead. */ |
867e359b CM |
196 | EX: { lw r6, r3; addi r3, r3, 64 } |
197 | { jal .Lcopy_line } | |
198 | ||
199 | /* Copy line 2, first stalling until r7 is ready. */ | |
200 | EX: { move r12, r7; lw r16, r1 } | |
201 | { bz r4, .Lcopy_8_check; slti_u r8, r2, 8 } | |
29507663 | 202 | /* Prefetch several lines ahead. */ |
867e359b | 203 | EX: { lw r7, r3; addi r3, r3, 64 } |
29507663 CM |
204 | /* Use up a caches-busy cycle by jumping back to the top of the |
205 | * loop. Might as well get it out of the way now. | |
206 | */ | |
207 | { j .Lbig_loop } | |
867e359b CM |
208 | |
209 | ||
210 | /* On entry: | |
211 | * - r0 points to the destination line. | |
212 | * - r1 points to the source line. | |
29507663 | 213 | * - r3 is the next prefetch address. |
867e359b CM |
214 | * - r9 holds the last address used for wh64. |
215 | * - r12 = WORD_15 | |
29507663 CM |
216 | * - r16 = WORD_0. |
217 | * - r17 == r1 + 16. | |
218 | * - r27 holds saved lr to restore. | |
867e359b CM |
219 | * |
220 | * On exit: | |
221 | * - r0 is incremented by 64. | |
222 | * - r1 is incremented by 64, unless that would point to a word | |
29507663 CM |
223 | * beyond the end of the source array, in which case it is redirected |
224 | * to point to an arbitrary word already in the cache. | |
867e359b | 225 | * - r2 is decremented by 64. |
29507663 CM |
226 | * - r3 is unchanged, unless it points to a word beyond the |
227 | * end of the source array, in which case it is redirected | |
228 | * to point to an arbitrary word already in the cache. | |
229 | * Redirecting is OK since if we are that close to the end | |
230 | * of the array we will not come back to this subroutine | |
231 | * and use the contents of the prefetched address. | |
867e359b | 232 | * - r4 is nonzero iff r2 >= 64. |
29507663 CM |
233 | * - r9 is incremented by 64, unless it points beyond the |
234 | * end of the last full destination cache line, in which | |
235 | * case it is redirected to a "safe address" that can be | |
236 | * clobbered (sp - 64) | |
867e359b CM |
237 | * - lr contains the value in r27. |
238 | */ | |
239 | ||
240 | /* r26 unused */ | |
241 | ||
242 | .Lcopy_line: | |
29507663 CM |
243 | /* TODO: when r3 goes past the end, we would like to redirect it |
244 | * to prefetch the last partial cache line (if any) just once, for the | |
245 | * benefit of the final cleanup loop. But we don't want to | |
246 | * prefetch that line more than once, or subsequent prefetches | |
247 | * will go into the RTF. But then .Lbig_loop should unconditionally | |
248 | * branch to top of loop to execute final prefetch, and its | |
249 | * nop should become a conditional branch. | |
250 | */ | |
251 | ||
252 | /* We need two non-memory cycles here to cover the resources | |
253 | * used by the loads initiated by the caller. | |
254 | */ | |
255 | { add r15, r1, r2 } | |
867e359b | 256 | .Lcopy_line2: |
29507663 | 257 | { slt_u r13, r3, r15; addi r17, r1, 16 } |
867e359b | 258 | |
29507663 | 259 | /* NOTE: this will stall for one cycle as L1 is busy. */ |
867e359b | 260 | |
29507663 | 261 | /* Fill second L1D line. */ |
867e359b CM |
262 | EX: { lw r17, r17; addi r1, r1, 48; mvz r3, r13, r1 } /* r17 = WORD_4 */ |
263 | ||
29507663 | 264 | /* Prepare destination line for writing. */ |
867e359b | 265 | EX: { wh64 r9; addi r9, r9, 64 } |
29507663 | 266 | /* Load seven words that are L1D hits to cover wh64 L2 usage. */ |
867e359b | 267 | |
29507663 CM |
268 | /* Load the three remaining words from the last L1D line, which |
269 | * we know has already filled the L1D. | |
270 | */ | |
867e359b CM |
271 | EX: { lw r4, r1; addi r1, r1, 4; addi r20, r1, 16 } /* r4 = WORD_12 */ |
272 | EX: { lw r8, r1; addi r1, r1, 4; slt_u r13, r20, r15 }/* r8 = WORD_13 */ | |
273 | EX: { lw r11, r1; addi r1, r1, -52; mvz r20, r13, r1 } /* r11 = WORD_14 */ | |
274 | ||
29507663 CM |
275 | /* Load the three remaining words from the first L1D line, first |
276 | * stalling until it has filled by "looking at" r16. | |
277 | */ | |
867e359b CM |
278 | EX: { lw r13, r1; addi r1, r1, 4; move zero, r16 } /* r13 = WORD_1 */ |
279 | EX: { lw r14, r1; addi r1, r1, 4 } /* r14 = WORD_2 */ | |
280 | EX: { lw r15, r1; addi r1, r1, 8; addi r10, r0, 60 } /* r15 = WORD_3 */ | |
281 | ||
29507663 CM |
282 | /* Load second word from the second L1D line, first |
283 | * stalling until it has filled by "looking at" r17. | |
284 | */ | |
867e359b CM |
285 | EX: { lw r19, r1; addi r1, r1, 4; move zero, r17 } /* r19 = WORD_5 */ |
286 | ||
29507663 CM |
287 | /* Store last word to the destination line, potentially dirtying it |
288 | * for the first time, which keeps the L2 busy for two cycles. | |
289 | */ | |
867e359b CM |
290 | EX: { sw r10, r12 } /* store(WORD_15) */ |
291 | ||
29507663 | 292 | /* Use two L1D hits to cover the sw L2 access above. */ |
867e359b CM |
293 | EX: { lw r10, r1; addi r1, r1, 4 } /* r10 = WORD_6 */ |
294 | EX: { lw r12, r1; addi r1, r1, 4 } /* r12 = WORD_7 */ | |
295 | ||
29507663 | 296 | /* Fill third L1D line. */ |
867e359b CM |
297 | EX: { lw r18, r1; addi r1, r1, 4 } /* r18 = WORD_8 */ |
298 | ||
29507663 | 299 | /* Store first L1D line. */ |
867e359b CM |
300 | EX: { sw r0, r16; addi r0, r0, 4; add r16, r0, r2 } /* store(WORD_0) */ |
301 | EX: { sw r0, r13; addi r0, r0, 4; andi r16, r16, -64 } /* store(WORD_1) */ | |
302 | EX: { sw r0, r14; addi r0, r0, 4; slt_u r16, r9, r16 } /* store(WORD_2) */ | |
867e359b | 303 | EX: { sw r0, r15; addi r0, r0, 4; addi r13, sp, -64 } /* store(WORD_3) */ |
29507663 | 304 | /* Store second L1D line. */ |
867e359b CM |
305 | EX: { sw r0, r17; addi r0, r0, 4; mvz r9, r16, r13 }/* store(WORD_4) */ |
306 | EX: { sw r0, r19; addi r0, r0, 4 } /* store(WORD_5) */ | |
307 | EX: { sw r0, r10; addi r0, r0, 4 } /* store(WORD_6) */ | |
308 | EX: { sw r0, r12; addi r0, r0, 4 } /* store(WORD_7) */ | |
309 | ||
310 | EX: { lw r13, r1; addi r1, r1, 4; move zero, r18 } /* r13 = WORD_9 */ | |
311 | EX: { lw r14, r1; addi r1, r1, 4 } /* r14 = WORD_10 */ | |
312 | EX: { lw r15, r1; move r1, r20 } /* r15 = WORD_11 */ | |
313 | ||
29507663 | 314 | /* Store third L1D line. */ |
867e359b CM |
315 | EX: { sw r0, r18; addi r0, r0, 4 } /* store(WORD_8) */ |
316 | EX: { sw r0, r13; addi r0, r0, 4 } /* store(WORD_9) */ | |
317 | EX: { sw r0, r14; addi r0, r0, 4 } /* store(WORD_10) */ | |
318 | EX: { sw r0, r15; addi r0, r0, 4 } /* store(WORD_11) */ | |
319 | ||
29507663 | 320 | /* Store rest of fourth L1D line. */ |
867e359b | 321 | EX: { sw r0, r4; addi r0, r0, 4 } /* store(WORD_12) */ |
29507663 | 322 | { |
867e359b | 323 | EX: sw r0, r8 /* store(WORD_13) */ |
29507663 | 324 | addi r0, r0, 4 |
867e359b | 325 | /* Will r2 be > 64 after we subtract 64 below? */ |
29507663 CM |
326 | shri r4, r2, 7 |
327 | } | |
328 | { | |
867e359b | 329 | EX: sw r0, r11 /* store(WORD_14) */ |
29507663 CM |
330 | addi r0, r0, 8 |
331 | /* Record 64 bytes successfully copied. */ | |
332 | addi r2, r2, -64 | |
333 | } | |
867e359b CM |
334 | |
335 | { jrp lr; move lr, r27 } | |
336 | ||
29507663 | 337 | /* Convey to the backtrace library that the stack frame is size |
867e359b CM |
338 | * zero, and the real return address is on the stack rather than |
339 | * in 'lr'. | |
340 | */ | |
341 | { info 8 } | |
342 | ||
343 | .align 64 | |
344 | .Lcopy_unaligned_maybe_many: | |
345 | /* Skip the setup overhead if we aren't copying many bytes. */ | |
346 | { slti_u r8, r2, 20; sub r4, zero, r0 } | |
347 | { bnzt r8, .Lcopy_unaligned_few; andi r4, r4, 3 } | |
348 | { bz r4, .Ldest_is_word_aligned; add r18, r1, r2 } | |
349 | ||
350 | /* | |
351 | * | |
352 | * unaligned 4 byte at a time copy handler. | |
353 | * | |
354 | */ | |
355 | ||
356 | /* Copy single bytes until r0 == 0 mod 4, so we can store words. */ | |
357 | .Lalign_dest_loop: | |
358 | EX: { lb_u r3, r1; addi r1, r1, 1; addi r4, r4, -1 } | |
359 | EX: { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } | |
360 | { bnzt r4, .Lalign_dest_loop; andi r3, r1, 3 } | |
361 | ||
362 | /* If source and dest are now *both* aligned, do an aligned copy. */ | |
363 | { bz r3, .Lcheck_aligned_copy_size; addli r4, r2, -256 } | |
364 | ||
365 | .Ldest_is_word_aligned: | |
366 | ||
867e359b CM |
367 | EX: { andi r8, r0, 63; lwadd_na r6, r1, 4} |
368 | { slti_u r9, r2, 64; bz r8, .Ldest_is_L2_line_aligned } | |
369 | ||
370 | /* This copies unaligned words until either there are fewer | |
371 | * than 4 bytes left to copy, or until the destination pointer | |
372 | * is cache-aligned, whichever comes first. | |
373 | * | |
374 | * On entry: | |
375 | * - r0 is the next store address. | |
376 | * - r1 points 4 bytes past the load address corresponding to r0. | |
377 | * - r2 >= 4 | |
378 | * - r6 is the next aligned word loaded. | |
379 | */ | |
380 | .Lcopy_unaligned_src_words: | |
381 | EX: { lwadd_na r7, r1, 4; slti_u r8, r2, 4 + 4 } | |
382 | /* stall */ | |
383 | { dword_align r6, r7, r1; slti_u r9, r2, 64 + 4 } | |
384 | EX: { swadd r0, r6, 4; addi r2, r2, -4 } | |
385 | { bnz r8, .Lcleanup_unaligned_words; andi r8, r0, 63 } | |
386 | { bnzt r8, .Lcopy_unaligned_src_words; move r6, r7 } | |
387 | ||
388 | /* On entry: | |
389 | * - r0 is the next store address. | |
390 | * - r1 points 4 bytes past the load address corresponding to r0. | |
391 | * - r2 >= 4 (# of bytes left to store). | |
392 | * - r6 is the next aligned src word value. | |
393 | * - r9 = (r2 < 64U). | |
394 | * - r18 points one byte past the end of source memory. | |
395 | */ | |
396 | .Ldest_is_L2_line_aligned: | |
397 | ||
398 | { | |
399 | /* Not a full cache line remains. */ | |
400 | bnz r9, .Lcleanup_unaligned_words | |
401 | move r7, r6 | |
402 | } | |
403 | ||
404 | /* r2 >= 64 */ | |
405 | ||
406 | /* Kick off two prefetches, but don't go past the end. */ | |
407 | { addi r3, r1, 63 - 4; addi r8, r1, 64 + 63 - 4 } | |
408 | { prefetch r3; move r3, r8; slt_u r8, r8, r18 } | |
409 | { mvz r3, r8, r1; addi r8, r3, 64 } | |
410 | { prefetch r3; move r3, r8; slt_u r8, r8, r18 } | |
411 | { mvz r3, r8, r1; movei r17, 0 } | |
412 | ||
413 | .Lcopy_unaligned_line: | |
414 | /* Prefetch another line. */ | |
415 | { prefetch r3; addi r15, r1, 60; addi r3, r3, 64 } | |
416 | /* Fire off a load of the last word we are about to copy. */ | |
417 | EX: { lw_na r15, r15; slt_u r8, r3, r18 } | |
418 | ||
419 | EX: { mvz r3, r8, r1; wh64 r0 } | |
420 | ||
421 | /* This loop runs twice. | |
422 | * | |
423 | * On entry: | |
424 | * - r17 is even before the first iteration, and odd before | |
425 | * the second. It is incremented inside the loop. Encountering | |
426 | * an even value at the end of the loop makes it stop. | |
427 | */ | |
428 | .Lcopy_half_an_unaligned_line: | |
429 | EX: { | |
430 | /* Stall until the last byte is ready. In the steady state this | |
431 | * guarantees all words to load below will be in the L2 cache, which | |
432 | * avoids shunting the loads to the RTF. | |
433 | */ | |
434 | move zero, r15 | |
435 | lwadd_na r7, r1, 16 | |
436 | } | |
437 | EX: { lwadd_na r11, r1, 12 } | |
438 | EX: { lwadd_na r14, r1, -24 } | |
439 | EX: { lwadd_na r8, r1, 4 } | |
440 | EX: { lwadd_na r9, r1, 4 } | |
441 | EX: { | |
442 | lwadd_na r10, r1, 8 | |
443 | /* r16 = (r2 < 64), after we subtract 32 from r2 below. */ | |
444 | slti_u r16, r2, 64 + 32 | |
445 | } | |
446 | EX: { lwadd_na r12, r1, 4; addi r17, r17, 1 } | |
447 | EX: { lwadd_na r13, r1, 8; dword_align r6, r7, r1 } | |
448 | EX: { swadd r0, r6, 4; dword_align r7, r8, r1 } | |
449 | EX: { swadd r0, r7, 4; dword_align r8, r9, r1 } | |
450 | EX: { swadd r0, r8, 4; dword_align r9, r10, r1 } | |
451 | EX: { swadd r0, r9, 4; dword_align r10, r11, r1 } | |
452 | EX: { swadd r0, r10, 4; dword_align r11, r12, r1 } | |
453 | EX: { swadd r0, r11, 4; dword_align r12, r13, r1 } | |
454 | EX: { swadd r0, r12, 4; dword_align r13, r14, r1 } | |
455 | EX: { swadd r0, r13, 4; addi r2, r2, -32 } | |
456 | { move r6, r14; bbst r17, .Lcopy_half_an_unaligned_line } | |
457 | ||
458 | { bzt r16, .Lcopy_unaligned_line; move r7, r6 } | |
459 | ||
460 | /* On entry: | |
461 | * - r0 is the next store address. | |
462 | * - r1 points 4 bytes past the load address corresponding to r0. | |
463 | * - r2 >= 0 (# of bytes left to store). | |
464 | * - r7 is the next aligned src word value. | |
465 | */ | |
466 | .Lcleanup_unaligned_words: | |
467 | /* Handle any trailing bytes. */ | |
468 | { bz r2, .Lcopy_unaligned_done; slti_u r8, r2, 4 } | |
469 | { bzt r8, .Lcopy_unaligned_src_words; move r6, r7 } | |
470 | ||
471 | /* Move r1 back to the point where it corresponds to r0. */ | |
472 | { addi r1, r1, -4 } | |
473 | ||
867e359b CM |
474 | /* Fall through */ |
475 | ||
476 | /* | |
477 | * | |
478 | * 1 byte at a time copy handler. | |
479 | * | |
480 | */ | |
481 | ||
482 | .Lcopy_unaligned_few: | |
483 | EX: { lb_u r3, r1; addi r1, r1, 1 } | |
484 | EX: { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } | |
485 | { bnzt r2, .Lcopy_unaligned_few } | |
486 | ||
487 | .Lcopy_unaligned_done: | |
488 | ||
489 | /* For memcpy return original dest address, else zero. */ | |
490 | { mz r0, r29, r23; jrp lr } | |
491 | ||
492 | .Lend_memcpy_common: | |
493 | .size memcpy_common, .Lend_memcpy_common - memcpy_common | |
494 | ||
495 | .section .fixup,"ax" | |
496 | memcpy_common_fixup: | |
497 | .type memcpy_common_fixup, @function | |
498 | ||
499 | /* Skip any bytes we already successfully copied. | |
500 | * r2 (num remaining) is correct, but r0 (dst) and r1 (src) | |
501 | * may not be quite right because of unrolling and prefetching. | |
502 | * So we need to recompute their values as the address just | |
503 | * after the last byte we are sure was successfully loaded and | |
504 | * then stored. | |
505 | */ | |
506 | ||
507 | /* Determine how many bytes we successfully copied. */ | |
508 | { sub r3, r25, r2 } | |
509 | ||
510 | /* Add this to the original r0 and r1 to get their new values. */ | |
511 | { add r0, r23, r3; add r1, r24, r3 } | |
512 | ||
513 | { bzt r29, memcpy_fixup_loop } | |
514 | { blzt r29, copy_to_user_fixup_loop } | |
515 | ||
516 | copy_from_user_fixup_loop: | |
517 | /* Try copying the rest one byte at a time, expecting a load fault. */ | |
518 | .Lcfu: { lb_u r3, r1; addi r1, r1, 1 } | |
519 | { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } | |
520 | { bnzt r2, copy_from_user_fixup_loop } | |
521 | ||
522 | .Lcopy_from_user_fixup_zero_remainder: | |
523 | { bbs r29, 2f } /* low bit set means IS_COPY_FROM_USER */ | |
524 | /* byte-at-a-time loop faulted, so zero the rest. */ | |
525 | { move r3, r2; bz r2, 2f /* should be impossible, but handle it. */ } | |
526 | 1: { sb r0, zero; addi r0, r0, 1; addi r3, r3, -1 } | |
527 | { bnzt r3, 1b } | |
528 | 2: move lr, r27 | |
529 | { move r0, r2; jrp lr } | |
530 | ||
531 | copy_to_user_fixup_loop: | |
532 | /* Try copying the rest one byte at a time, expecting a store fault. */ | |
533 | { lb_u r3, r1; addi r1, r1, 1 } | |
534 | .Lctu: { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } | |
535 | { bnzt r2, copy_to_user_fixup_loop } | |
536 | .Lcopy_to_user_fixup_done: | |
537 | move lr, r27 | |
538 | { move r0, r2; jrp lr } | |
539 | ||
540 | memcpy_fixup_loop: | |
541 | /* Try copying the rest one byte at a time. We expect a disastrous | |
542 | * fault to happen since we are in fixup code, but let it happen. | |
543 | */ | |
544 | { lb_u r3, r1; addi r1, r1, 1 } | |
545 | { sb r0, r3; addi r0, r0, 1; addi r2, r2, -1 } | |
546 | { bnzt r2, memcpy_fixup_loop } | |
547 | /* This should be unreachable, we should have faulted again. | |
548 | * But be paranoid and handle it in case some interrupt changed | |
549 | * the TLB or something. | |
550 | */ | |
551 | move lr, r27 | |
552 | { move r0, r23; jrp lr } | |
553 | ||
554 | .size memcpy_common_fixup, . - memcpy_common_fixup | |
555 | ||
556 | .section __ex_table,"a" | |
d4d9eab4 | 557 | .align 4 |
867e359b CM |
558 | .word .Lcfu, .Lcopy_from_user_fixup_zero_remainder |
559 | .word .Lctu, .Lcopy_to_user_fixup_done |