1 /* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy.
3 * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com)
4 * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz)
8 #include <asm/visasm.h>
10 #include <asm/export.h>
11 #define GLOBAL_SPARE g7
13 #define GLOBAL_SPARE g5
14 #define ASI_BLK_P 0xf0
17 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \
18 clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0;
19 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
21 #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs
22 #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs
41 #define EX_RETVAL(x) x
45 #define LOAD(type,addr,dest) type [addr], dest
49 #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest
53 #define STORE(type,src,addr) type src, [addr]
57 #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P
61 #define FUNC_NAME memcpy
72 #define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9) \
73 faligndata %f1, %f2, %f48; \
74 faligndata %f2, %f3, %f50; \
75 faligndata %f3, %f4, %f52; \
76 faligndata %f4, %f5, %f54; \
77 faligndata %f5, %f6, %f56; \
78 faligndata %f6, %f7, %f58; \
79 faligndata %f7, %f8, %f60; \
80 faligndata %f8, %f9, %f62;
82 #define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \
83 EX_LD_FP(LOAD_BLK(%src, %fdest)); \
84 EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
85 add %src, 0x40, %src; \
86 subcc %len, 0x40, %len; \
88 add %dest, 0x40, %dest; \
90 #define LOOP_CHUNK1(src, dest, len, branch_dest) \
91 MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest)
92 #define LOOP_CHUNK2(src, dest, len, branch_dest) \
93 MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest)
94 #define LOOP_CHUNK3(src, dest, len, branch_dest) \
95 MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest)
97 #define DO_SYNC membar #Sync;
98 #define STORE_SYNC(dest, fsrc) \
99 EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
100 add %dest, 0x40, %dest; \
103 #define STORE_JUMP(dest, fsrc, target) \
104 EX_ST_FP(STORE_BLK(%fsrc, %dest)); \
105 add %dest, 0x40, %dest; \
106 ba,pt %xcc, target; \
109 #define FINISH_VISCHUNK(dest, f0, f1, left) \
110 subcc %left, 8, %left;\
112 faligndata %f0, %f1, %f48; \
113 EX_ST_FP(STORE(std, %f48, %dest)); \
116 #define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
117 subcc %left, 8, %left; \
121 #define UNEVEN_VISCHUNK(dest, f0, f1, left) \
122 UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \
125 .register %g2,#scratch
126 .register %g3,#scratch
132 .type FUNC_NAME,#function
133 FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */
150 /* Clobbers o5/g1/g2/g3/g7/icc/xcc. */
153 /* Is 'dst' already aligned on an 64-byte boundary? */
157 /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number
158 * of bytes to copy to make 'dst' 64-byte aligned. We pre-
159 * subtract this from 'len'.
161 sub %o0, %o1, %GLOBAL_SPARE
169 1: subcc %g1, 0x1, %g1
170 EX_LD_FP(LOAD(ldub, %o1 + 0x00, %o3))
171 EX_ST_FP(STORE(stb, %o3, %o1 + %GLOBAL_SPARE))
175 add %o1, %GLOBAL_SPARE, %o0
180 alignaddr %o1, %g0, %o1
182 EX_LD_FP(LOAD(ldd, %o1, %f4))
183 1: EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f6))
186 faligndata %f4, %f6, %f0
187 EX_ST_FP(STORE(std, %f0, %o0))
191 EX_LD_FP(LOAD(ldd, %o1 + 0x8, %f4))
194 faligndata %f6, %f4, %f0
195 EX_ST_FP(STORE(std, %f0, %o0))
199 /* Destination is 64-byte aligned. */
201 membar #LoadStore | #StoreStore | #StoreLoad
203 subcc %o2, 0x40, %GLOBAL_SPARE
205 andncc %GLOBAL_SPARE, (0x40 - 1), %GLOBAL_SPARE
207 sub %o2, %GLOBAL_SPARE, %g3
208 andn %o1, (0x40 - 1), %o1
213 sub %o2, %GLOBAL_SPARE, %o2
215 add %g1, %GLOBAL_SPARE, %g1
218 EX_LD_FP(LOAD_BLK(%o1, %f0))
221 EX_LD_FP(LOAD_BLK(%o1, %f16))
223 sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE
224 EX_LD_FP(LOAD_BLK(%o1, %f32))
227 /* There are 8 instances of the unrolled loop,
228 * one for each possible alignment of the
229 * source buffer. Each loop instance is 452
238 add %o3, %lo(1f - 1b), %o3
243 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
244 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
245 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
246 LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
247 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
248 LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
250 faligndata %f0, %f2, %f48
251 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
253 FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
254 STORE_JUMP(o0, f48, 40f)
255 2: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0)
257 FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
258 STORE_JUMP(o0, f48, 48f)
259 3: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16)
261 FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32)
262 STORE_JUMP(o0, f48, 56f)
264 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
265 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
266 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
267 LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
268 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
269 LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
271 faligndata %f2, %f4, %f48
272 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
274 FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
275 STORE_JUMP(o0, f48, 41f)
276 2: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2)
278 FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
279 STORE_JUMP(o0, f48, 49f)
280 3: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18)
282 FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34)
283 STORE_JUMP(o0, f48, 57f)
285 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
286 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
287 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
288 LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
289 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
290 LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
292 faligndata %f4, %f6, %f48
293 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
295 FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
296 STORE_JUMP(o0, f48, 42f)
297 2: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4)
299 FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
300 STORE_JUMP(o0, f48, 50f)
301 3: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20)
303 FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36)
304 STORE_JUMP(o0, f48, 58f)
306 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
307 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
308 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
309 LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
310 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
311 LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
313 faligndata %f6, %f8, %f48
314 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
316 FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
317 STORE_JUMP(o0, f48, 43f)
318 2: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6)
320 FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
321 STORE_JUMP(o0, f48, 51f)
322 3: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22)
324 FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38)
325 STORE_JUMP(o0, f48, 59f)
327 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
328 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
329 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
330 LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
331 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
332 LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
334 faligndata %f8, %f10, %f48
335 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
337 FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
338 STORE_JUMP(o0, f48, 44f)
339 2: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8)
341 FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
342 STORE_JUMP(o0, f48, 52f)
343 3: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24)
345 FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40)
346 STORE_JUMP(o0, f48, 60f)
348 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
349 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
350 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
351 LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
352 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
353 LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
355 faligndata %f10, %f12, %f48
356 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
358 FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
359 STORE_JUMP(o0, f48, 45f)
360 2: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10)
362 FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
363 STORE_JUMP(o0, f48, 53f)
364 3: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26)
366 FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42)
367 STORE_JUMP(o0, f48, 61f)
369 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
370 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
371 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
372 LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
373 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
374 LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
376 faligndata %f12, %f14, %f48
377 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
379 FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
380 STORE_JUMP(o0, f48, 46f)
381 2: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12)
383 FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
384 STORE_JUMP(o0, f48, 54f)
385 3: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28)
387 FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44)
388 STORE_JUMP(o0, f48, 62f)
390 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
391 LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f)
392 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
393 LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f)
394 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
395 LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f)
397 faligndata %f14, %f16, %f48
398 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
400 FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
401 STORE_JUMP(o0, f48, 47f)
402 2: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14)
404 FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
405 STORE_JUMP(o0, f48, 55f)
406 3: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30)
408 FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46)
409 STORE_JUMP(o0, f48, 63f)
411 40: FINISH_VISCHUNK(o0, f0, f2, g3)
412 41: FINISH_VISCHUNK(o0, f2, f4, g3)
413 42: FINISH_VISCHUNK(o0, f4, f6, g3)
414 43: FINISH_VISCHUNK(o0, f6, f8, g3)
415 44: FINISH_VISCHUNK(o0, f8, f10, g3)
416 45: FINISH_VISCHUNK(o0, f10, f12, g3)
417 46: FINISH_VISCHUNK(o0, f12, f14, g3)
418 47: UNEVEN_VISCHUNK(o0, f14, f0, g3)
419 48: FINISH_VISCHUNK(o0, f16, f18, g3)
420 49: FINISH_VISCHUNK(o0, f18, f20, g3)
421 50: FINISH_VISCHUNK(o0, f20, f22, g3)
422 51: FINISH_VISCHUNK(o0, f22, f24, g3)
423 52: FINISH_VISCHUNK(o0, f24, f26, g3)
424 53: FINISH_VISCHUNK(o0, f26, f28, g3)
425 54: FINISH_VISCHUNK(o0, f28, f30, g3)
426 55: UNEVEN_VISCHUNK(o0, f30, f0, g3)
427 56: FINISH_VISCHUNK(o0, f32, f34, g3)
428 57: FINISH_VISCHUNK(o0, f34, f36, g3)
429 58: FINISH_VISCHUNK(o0, f36, f38, g3)
430 59: FINISH_VISCHUNK(o0, f38, f40, g3)
431 60: FINISH_VISCHUNK(o0, f40, f42, g3)
432 61: FINISH_VISCHUNK(o0, f42, f44, g3)
433 62: FINISH_VISCHUNK(o0, f44, f46, g3)
434 63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3)
436 93: EX_LD_FP(LOAD(ldd, %o1, %f2))
439 faligndata %f0, %f2, %f8
440 EX_ST_FP(STORE(std, %f8, %o0))
443 EX_LD_FP(LOAD(ldd, %o1, %f0))
446 faligndata %f2, %f0, %f8
447 EX_ST_FP(STORE(std, %f8, %o0))
454 1: EX_LD_FP(LOAD(ldub, %o1, %o3))
457 EX_ST_FP(STORE(stb, %o3, %o0))
461 2: membar #StoreLoad | #StoreStore
464 mov EX_RETVAL(%o4), %o0
467 70: /* 16 < len <= (5 * 64) */
471 72: andn %o2, 0xf, %GLOBAL_SPARE
473 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5))
474 EX_LD(LOAD(ldx, %o1 + 0x08, %g1))
475 subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE
476 EX_ST(STORE(stx, %o5, %o1 + %o3))
478 EX_ST(STORE(stx, %g1, %o1 + %o3))
481 73: andcc %o2, 0x8, %g0
484 EX_LD(LOAD(ldx, %o1, %o5))
486 EX_ST(STORE(stx, %o5, %o1 + %o3))
488 1: andcc %o2, 0x4, %g0
491 EX_LD(LOAD(lduw, %o1, %o5))
493 EX_ST(STORE(stw, %o5, %o1 + %o3))
501 75: andcc %o0, 0x7, %g1
507 1: EX_LD(LOAD(ldub, %o1, %o5))
509 EX_ST(STORE(stb, %o5, %o1 + %o3))
525 EX_LD(LOAD(ldx, %o1, %g2))
527 andn %o2, 0x7, %GLOBAL_SPARE
529 1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3))
530 subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE
534 EX_ST(STORE(stx, %o5, %o0))
547 80: /* 0 < len <= 16 */
552 1: EX_LD(LOAD(lduw, %o1, %g1))
554 EX_ST(STORE(stw, %g1, %o1 + %o3))
559 mov EX_RETVAL(%o4), %o0
562 90: EX_LD(LOAD(ldub, %o1, %g1))
564 EX_ST(STORE(stb, %g1, %o1 + %o3))
568 mov EX_RETVAL(%o4), %o0
570 .size FUNC_NAME, .-FUNC_NAME
571 EXPORT_SYMBOL(FUNC_NAME)