Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* U1memcpy.S: UltraSPARC-I/II/IIi/IIe optimized memcpy. |
2 | * | |
3 | * Copyright (C) 1997, 2004 David S. Miller (davem@redhat.com) | |
4 | * Copyright (C) 1996, 1997, 1998, 1999 Jakub Jelinek (jj@ultra.linux.cz) | |
5 | */ | |
6 | ||
7 | #ifdef __KERNEL__ | |
8 | #include <asm/visasm.h> | |
9 | #include <asm/asi.h> | |
10 | #define GLOBAL_SPARE g7 | |
11 | #else | |
12 | #define GLOBAL_SPARE g5 | |
13 | #define ASI_BLK_P 0xf0 | |
14 | #define FPRS_FEF 0x04 | |
15 | #ifdef MEMCPY_DEBUG | |
16 | #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs; \ | |
17 | clr %g1; clr %g2; clr %g3; subcc %g0, %g0, %g0; | |
18 | #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs | |
19 | #else | |
20 | #define VISEntry rd %fprs, %o5; wr %g0, FPRS_FEF, %fprs | |
21 | #define VISExit and %o5, FPRS_FEF, %o5; wr %o5, 0x0, %fprs | |
22 | #endif | |
23 | #endif | |
24 | ||
25 | #ifndef EX_LD | |
26 | #define EX_LD(x) x | |
27 | #endif | |
28 | ||
29 | #ifndef EX_ST | |
30 | #define EX_ST(x) x | |
31 | #endif | |
32 | ||
33 | #ifndef EX_RETVAL | |
34 | #define EX_RETVAL(x) x | |
35 | #endif | |
36 | ||
37 | #ifndef LOAD | |
38 | #define LOAD(type,addr,dest) type [addr], dest | |
39 | #endif | |
40 | ||
41 | #ifndef LOAD_BLK | |
42 | #define LOAD_BLK(addr,dest) ldda [addr] ASI_BLK_P, dest | |
43 | #endif | |
44 | ||
45 | #ifndef STORE | |
46 | #define STORE(type,src,addr) type src, [addr] | |
47 | #endif | |
48 | ||
49 | #ifndef STORE_BLK | |
50 | #define STORE_BLK(src,addr) stda src, [addr] ASI_BLK_P | |
51 | #endif | |
52 | ||
53 | #ifndef FUNC_NAME | |
54 | #define FUNC_NAME memcpy | |
55 | #endif | |
56 | ||
57 | #ifndef PREAMBLE | |
58 | #define PREAMBLE | |
59 | #endif | |
60 | ||
61 | #ifndef XCC | |
62 | #define XCC xcc | |
63 | #endif | |
64 | ||
65 | #define FREG_FROB(f1, f2, f3, f4, f5, f6, f7, f8, f9) \ | |
66 | faligndata %f1, %f2, %f48; \ | |
67 | faligndata %f2, %f3, %f50; \ | |
68 | faligndata %f3, %f4, %f52; \ | |
69 | faligndata %f4, %f5, %f54; \ | |
70 | faligndata %f5, %f6, %f56; \ | |
71 | faligndata %f6, %f7, %f58; \ | |
72 | faligndata %f7, %f8, %f60; \ | |
73 | faligndata %f8, %f9, %f62; | |
74 | ||
75 | #define MAIN_LOOP_CHUNK(src, dest, fdest, fsrc, len, jmptgt) \ | |
76 | EX_LD(LOAD_BLK(%src, %fdest)); \ | |
77 | EX_ST(STORE_BLK(%fsrc, %dest)); \ | |
78 | add %src, 0x40, %src; \ | |
79 | subcc %len, 0x40, %len; \ | |
80 | be,pn %xcc, jmptgt; \ | |
81 | add %dest, 0x40, %dest; \ | |
82 | ||
83 | #define LOOP_CHUNK1(src, dest, len, branch_dest) \ | |
84 | MAIN_LOOP_CHUNK(src, dest, f0, f48, len, branch_dest) | |
85 | #define LOOP_CHUNK2(src, dest, len, branch_dest) \ | |
86 | MAIN_LOOP_CHUNK(src, dest, f16, f48, len, branch_dest) | |
87 | #define LOOP_CHUNK3(src, dest, len, branch_dest) \ | |
88 | MAIN_LOOP_CHUNK(src, dest, f32, f48, len, branch_dest) | |
89 | ||
90 | #define STORE_SYNC(dest, fsrc) \ | |
91 | EX_ST(STORE_BLK(%fsrc, %dest)); \ | |
92 | add %dest, 0x40, %dest; | |
93 | ||
94 | #define STORE_JUMP(dest, fsrc, target) \ | |
95 | EX_ST(STORE_BLK(%fsrc, %dest)); \ | |
96 | add %dest, 0x40, %dest; \ | |
97 | ba,pt %xcc, target; | |
98 | ||
99 | #define FINISH_VISCHUNK(dest, f0, f1, left) \ | |
100 | subcc %left, 8, %left;\ | |
101 | bl,pn %xcc, 95f; \ | |
102 | faligndata %f0, %f1, %f48; \ | |
103 | EX_ST(STORE(std, %f48, %dest)); \ | |
104 | add %dest, 8, %dest; | |
105 | ||
106 | #define UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ | |
107 | subcc %left, 8, %left; \ | |
108 | bl,pn %xcc, 95f; \ | |
109 | fsrc1 %f0, %f1; | |
110 | ||
111 | #define UNEVEN_VISCHUNK(dest, f0, f1, left) \ | |
112 | UNEVEN_VISCHUNK_LAST(dest, f0, f1, left) \ | |
113 | ba,a,pt %xcc, 93f; | |
114 | ||
115 | .register %g2,#scratch | |
116 | .register %g3,#scratch | |
117 | ||
118 | .text | |
119 | .align 64 | |
120 | ||
121 | .globl FUNC_NAME | |
122 | .type FUNC_NAME,#function | |
123 | FUNC_NAME: /* %o0=dst, %o1=src, %o2=len */ | |
124 | srlx %o2, 31, %g2 | |
125 | cmp %g2, 0 | |
126 | tne %xcc, 5 | |
127 | PREAMBLE | |
128 | mov %o0, %o4 | |
129 | cmp %o2, 0 | |
130 | be,pn %XCC, 85f | |
131 | or %o0, %o1, %o3 | |
132 | cmp %o2, 16 | |
133 | blu,a,pn %XCC, 80f | |
134 | or %o3, %o2, %o3 | |
135 | ||
136 | cmp %o2, (5 * 64) | |
137 | blu,pt %XCC, 70f | |
138 | andcc %o3, 0x7, %g0 | |
139 | ||
140 | /* Clobbers o5/g1/g2/g3/g7/icc/xcc. */ | |
141 | VISEntry | |
142 | ||
143 | /* Is 'dst' already aligned on an 64-byte boundary? */ | |
144 | andcc %o0, 0x3f, %g2 | |
145 | be,pt %XCC, 2f | |
146 | ||
147 | /* Compute abs((dst & 0x3f) - 0x40) into %g2. This is the number | |
148 | * of bytes to copy to make 'dst' 64-byte aligned. We pre- | |
149 | * subtract this from 'len'. | |
150 | */ | |
151 | sub %o0, %o1, %GLOBAL_SPARE | |
152 | sub %g2, 0x40, %g2 | |
153 | sub %g0, %g2, %g2 | |
154 | sub %o2, %g2, %o2 | |
155 | andcc %g2, 0x7, %g1 | |
156 | be,pt %icc, 2f | |
157 | and %g2, 0x38, %g2 | |
158 | ||
159 | 1: subcc %g1, 0x1, %g1 | |
160 | EX_LD(LOAD(ldub, %o1 + 0x00, %o3)) | |
161 | EX_ST(STORE(stb, %o3, %o1 + %GLOBAL_SPARE)) | |
162 | bgu,pt %XCC, 1b | |
163 | add %o1, 0x1, %o1 | |
164 | ||
165 | add %o1, %GLOBAL_SPARE, %o0 | |
166 | ||
167 | 2: cmp %g2, 0x0 | |
168 | and %o1, 0x7, %g1 | |
169 | be,pt %icc, 3f | |
170 | alignaddr %o1, %g0, %o1 | |
171 | ||
172 | EX_LD(LOAD(ldd, %o1, %f4)) | |
173 | 1: EX_LD(LOAD(ldd, %o1 + 0x8, %f6)) | |
174 | add %o1, 0x8, %o1 | |
175 | subcc %g2, 0x8, %g2 | |
176 | faligndata %f4, %f6, %f0 | |
177 | EX_ST(STORE(std, %f0, %o0)) | |
178 | be,pn %icc, 3f | |
179 | add %o0, 0x8, %o0 | |
180 | ||
181 | EX_LD(LOAD(ldd, %o1 + 0x8, %f4)) | |
182 | add %o1, 0x8, %o1 | |
183 | subcc %g2, 0x8, %g2 | |
184 | faligndata %f6, %f4, %f0 | |
185 | EX_ST(STORE(std, %f0, %o0)) | |
186 | bne,pt %icc, 1b | |
187 | add %o0, 0x8, %o0 | |
188 | ||
189 | /* Destination is 64-byte aligned. */ | |
190 | 3: | |
191 | membar #LoadStore | #StoreStore | #StoreLoad | |
192 | ||
193 | subcc %o2, 0x40, %GLOBAL_SPARE | |
194 | add %o1, %g1, %g1 | |
195 | andncc %GLOBAL_SPARE, (0x40 - 1), %GLOBAL_SPARE | |
196 | srl %g1, 3, %g2 | |
197 | sub %o2, %GLOBAL_SPARE, %g3 | |
198 | andn %o1, (0x40 - 1), %o1 | |
199 | and %g2, 7, %g2 | |
200 | andncc %g3, 0x7, %g3 | |
201 | fmovd %f0, %f2 | |
202 | sub %g3, 0x8, %g3 | |
203 | sub %o2, %GLOBAL_SPARE, %o2 | |
204 | ||
205 | add %g1, %GLOBAL_SPARE, %g1 | |
206 | subcc %o2, %g3, %o2 | |
207 | ||
208 | EX_LD(LOAD_BLK(%o1, %f0)) | |
209 | add %o1, 0x40, %o1 | |
210 | add %g1, %g3, %g1 | |
211 | EX_LD(LOAD_BLK(%o1, %f16)) | |
212 | add %o1, 0x40, %o1 | |
213 | sub %GLOBAL_SPARE, 0x80, %GLOBAL_SPARE | |
214 | EX_LD(LOAD_BLK(%o1, %f32)) | |
215 | add %o1, 0x40, %o1 | |
216 | ||
217 | /* There are 8 instances of the unrolled loop, | |
218 | * one for each possible alignment of the | |
219 | * source buffer. Each loop instance is 452 | |
220 | * bytes. | |
221 | */ | |
222 | sll %g2, 3, %o3 | |
223 | sub %o3, %g2, %o3 | |
224 | sllx %o3, 4, %o3 | |
225 | add %o3, %g2, %o3 | |
226 | sllx %o3, 2, %g2 | |
227 | 1: rd %pc, %o3 | |
228 | add %o3, %lo(1f - 1b), %o3 | |
229 | jmpl %o3 + %g2, %g0 | |
230 | nop | |
231 | ||
232 | .align 64 | |
233 | 1: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) | |
234 | LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) | |
235 | FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) | |
236 | LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) | |
237 | FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) | |
238 | LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) | |
239 | ba,pt %xcc, 1b+4 | |
240 | faligndata %f0, %f2, %f48 | |
241 | 1: FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) | |
242 | STORE_SYNC(o0, f48) membar #Sync | |
243 | FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) | |
244 | STORE_JUMP(o0, f48, 40f) membar #Sync | |
245 | 2: FREG_FROB(f32,f34,f36,f38,f40,f42,f44,f46,f0) | |
246 | STORE_SYNC(o0, f48) membar #Sync | |
247 | FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) | |
248 | STORE_JUMP(o0, f48, 48f) membar #Sync | |
249 | 3: FREG_FROB(f0, f2, f4, f6, f8, f10,f12,f14,f16) | |
250 | STORE_SYNC(o0, f48) membar #Sync | |
251 | FREG_FROB(f16,f18,f20,f22,f24,f26,f28,f30,f32) | |
252 | STORE_JUMP(o0, f48, 56f) membar #Sync | |
253 | ||
254 | 1: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) | |
255 | LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) | |
256 | FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) | |
257 | LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) | |
258 | FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) | |
259 | LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) | |
260 | ba,pt %xcc, 1b+4 | |
261 | faligndata %f2, %f4, %f48 | |
262 | 1: FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) | |
263 | STORE_SYNC(o0, f48) membar #Sync | |
264 | FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) | |
265 | STORE_JUMP(o0, f48, 41f) membar #Sync | |
266 | 2: FREG_FROB(f34,f36,f38,f40,f42,f44,f46,f0, f2) | |
267 | STORE_SYNC(o0, f48) membar #Sync | |
268 | FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) | |
269 | STORE_JUMP(o0, f48, 49f) membar #Sync | |
270 | 3: FREG_FROB(f2, f4, f6, f8, f10,f12,f14,f16,f18) | |
271 | STORE_SYNC(o0, f48) membar #Sync | |
272 | FREG_FROB(f18,f20,f22,f24,f26,f28,f30,f32,f34) | |
273 | STORE_JUMP(o0, f48, 57f) membar #Sync | |
274 | ||
275 | 1: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) | |
276 | LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) | |
277 | FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) | |
278 | LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) | |
279 | FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) | |
280 | LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) | |
281 | ba,pt %xcc, 1b+4 | |
282 | faligndata %f4, %f6, %f48 | |
283 | 1: FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) | |
284 | STORE_SYNC(o0, f48) membar #Sync | |
285 | FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) | |
286 | STORE_JUMP(o0, f48, 42f) membar #Sync | |
287 | 2: FREG_FROB(f36,f38,f40,f42,f44,f46,f0, f2, f4) | |
288 | STORE_SYNC(o0, f48) membar #Sync | |
289 | FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) | |
290 | STORE_JUMP(o0, f48, 50f) membar #Sync | |
291 | 3: FREG_FROB(f4, f6, f8, f10,f12,f14,f16,f18,f20) | |
292 | STORE_SYNC(o0, f48) membar #Sync | |
293 | FREG_FROB(f20,f22,f24,f26,f28,f30,f32,f34,f36) | |
294 | STORE_JUMP(o0, f48, 58f) membar #Sync | |
295 | ||
296 | 1: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) | |
297 | LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) | |
298 | FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) | |
299 | LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) | |
300 | FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) | |
301 | LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) | |
302 | ba,pt %xcc, 1b+4 | |
303 | faligndata %f6, %f8, %f48 | |
304 | 1: FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) | |
305 | STORE_SYNC(o0, f48) membar #Sync | |
306 | FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) | |
307 | STORE_JUMP(o0, f48, 43f) membar #Sync | |
308 | 2: FREG_FROB(f38,f40,f42,f44,f46,f0, f2, f4, f6) | |
309 | STORE_SYNC(o0, f48) membar #Sync | |
310 | FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) | |
311 | STORE_JUMP(o0, f48, 51f) membar #Sync | |
312 | 3: FREG_FROB(f6, f8, f10,f12,f14,f16,f18,f20,f22) | |
313 | STORE_SYNC(o0, f48) membar #Sync | |
314 | FREG_FROB(f22,f24,f26,f28,f30,f32,f34,f36,f38) | |
315 | STORE_JUMP(o0, f48, 59f) membar #Sync | |
316 | ||
317 | 1: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) | |
318 | LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) | |
319 | FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) | |
320 | LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) | |
321 | FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) | |
322 | LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) | |
323 | ba,pt %xcc, 1b+4 | |
324 | faligndata %f8, %f10, %f48 | |
325 | 1: FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) | |
326 | STORE_SYNC(o0, f48) membar #Sync | |
327 | FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) | |
328 | STORE_JUMP(o0, f48, 44f) membar #Sync | |
329 | 2: FREG_FROB(f40,f42,f44,f46,f0, f2, f4, f6, f8) | |
330 | STORE_SYNC(o0, f48) membar #Sync | |
331 | FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) | |
332 | STORE_JUMP(o0, f48, 52f) membar #Sync | |
333 | 3: FREG_FROB(f8, f10,f12,f14,f16,f18,f20,f22,f24) | |
334 | STORE_SYNC(o0, f48) membar #Sync | |
335 | FREG_FROB(f24,f26,f28,f30,f32,f34,f36,f38,f40) | |
336 | STORE_JUMP(o0, f48, 60f) membar #Sync | |
337 | ||
338 | 1: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) | |
339 | LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) | |
340 | FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) | |
341 | LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) | |
342 | FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) | |
343 | LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) | |
344 | ba,pt %xcc, 1b+4 | |
345 | faligndata %f10, %f12, %f48 | |
346 | 1: FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) | |
347 | STORE_SYNC(o0, f48) membar #Sync | |
348 | FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) | |
349 | STORE_JUMP(o0, f48, 45f) membar #Sync | |
350 | 2: FREG_FROB(f42,f44,f46,f0, f2, f4, f6, f8, f10) | |
351 | STORE_SYNC(o0, f48) membar #Sync | |
352 | FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) | |
353 | STORE_JUMP(o0, f48, 53f) membar #Sync | |
354 | 3: FREG_FROB(f10,f12,f14,f16,f18,f20,f22,f24,f26) | |
355 | STORE_SYNC(o0, f48) membar #Sync | |
356 | FREG_FROB(f26,f28,f30,f32,f34,f36,f38,f40,f42) | |
357 | STORE_JUMP(o0, f48, 61f) membar #Sync | |
358 | ||
359 | 1: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) | |
360 | LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) | |
361 | FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) | |
362 | LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) | |
363 | FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) | |
364 | LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) | |
365 | ba,pt %xcc, 1b+4 | |
366 | faligndata %f12, %f14, %f48 | |
367 | 1: FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) | |
368 | STORE_SYNC(o0, f48) membar #Sync | |
369 | FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) | |
370 | STORE_JUMP(o0, f48, 46f) membar #Sync | |
371 | 2: FREG_FROB(f44,f46,f0, f2, f4, f6, f8, f10,f12) | |
372 | STORE_SYNC(o0, f48) membar #Sync | |
373 | FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) | |
374 | STORE_JUMP(o0, f48, 54f) membar #Sync | |
375 | 3: FREG_FROB(f12,f14,f16,f18,f20,f22,f24,f26,f28) | |
376 | STORE_SYNC(o0, f48) membar #Sync | |
377 | FREG_FROB(f28,f30,f32,f34,f36,f38,f40,f42,f44) | |
378 | STORE_JUMP(o0, f48, 62f) membar #Sync | |
379 | ||
380 | 1: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) | |
381 | LOOP_CHUNK1(o1, o0, GLOBAL_SPARE, 1f) | |
382 | FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) | |
383 | LOOP_CHUNK2(o1, o0, GLOBAL_SPARE, 2f) | |
384 | FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) | |
385 | LOOP_CHUNK3(o1, o0, GLOBAL_SPARE, 3f) | |
386 | ba,pt %xcc, 1b+4 | |
387 | faligndata %f14, %f16, %f48 | |
388 | 1: FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) | |
389 | STORE_SYNC(o0, f48) membar #Sync | |
390 | FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) | |
391 | STORE_JUMP(o0, f48, 47f) membar #Sync | |
392 | 2: FREG_FROB(f46,f0, f2, f4, f6, f8, f10,f12,f14) | |
393 | STORE_SYNC(o0, f48) membar #Sync | |
394 | FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) | |
395 | STORE_JUMP(o0, f48, 55f) membar #Sync | |
396 | 3: FREG_FROB(f14,f16,f18,f20,f22,f24,f26,f28,f30) | |
397 | STORE_SYNC(o0, f48) membar #Sync | |
398 | FREG_FROB(f30,f32,f34,f36,f38,f40,f42,f44,f46) | |
399 | STORE_JUMP(o0, f48, 63f) membar #Sync | |
400 | ||
401 | 40: FINISH_VISCHUNK(o0, f0, f2, g3) | |
402 | 41: FINISH_VISCHUNK(o0, f2, f4, g3) | |
403 | 42: FINISH_VISCHUNK(o0, f4, f6, g3) | |
404 | 43: FINISH_VISCHUNK(o0, f6, f8, g3) | |
405 | 44: FINISH_VISCHUNK(o0, f8, f10, g3) | |
406 | 45: FINISH_VISCHUNK(o0, f10, f12, g3) | |
407 | 46: FINISH_VISCHUNK(o0, f12, f14, g3) | |
408 | 47: UNEVEN_VISCHUNK(o0, f14, f0, g3) | |
409 | 48: FINISH_VISCHUNK(o0, f16, f18, g3) | |
410 | 49: FINISH_VISCHUNK(o0, f18, f20, g3) | |
411 | 50: FINISH_VISCHUNK(o0, f20, f22, g3) | |
412 | 51: FINISH_VISCHUNK(o0, f22, f24, g3) | |
413 | 52: FINISH_VISCHUNK(o0, f24, f26, g3) | |
414 | 53: FINISH_VISCHUNK(o0, f26, f28, g3) | |
415 | 54: FINISH_VISCHUNK(o0, f28, f30, g3) | |
416 | 55: UNEVEN_VISCHUNK(o0, f30, f0, g3) | |
417 | 56: FINISH_VISCHUNK(o0, f32, f34, g3) | |
418 | 57: FINISH_VISCHUNK(o0, f34, f36, g3) | |
419 | 58: FINISH_VISCHUNK(o0, f36, f38, g3) | |
420 | 59: FINISH_VISCHUNK(o0, f38, f40, g3) | |
421 | 60: FINISH_VISCHUNK(o0, f40, f42, g3) | |
422 | 61: FINISH_VISCHUNK(o0, f42, f44, g3) | |
423 | 62: FINISH_VISCHUNK(o0, f44, f46, g3) | |
424 | 63: UNEVEN_VISCHUNK_LAST(o0, f46, f0, g3) | |
425 | ||
426 | 93: EX_LD(LOAD(ldd, %o1, %f2)) | |
427 | add %o1, 8, %o1 | |
428 | subcc %g3, 8, %g3 | |
429 | faligndata %f0, %f2, %f8 | |
430 | EX_ST(STORE(std, %f8, %o0)) | |
431 | bl,pn %xcc, 95f | |
432 | add %o0, 8, %o0 | |
433 | EX_LD(LOAD(ldd, %o1, %f0)) | |
434 | add %o1, 8, %o1 | |
435 | subcc %g3, 8, %g3 | |
436 | faligndata %f2, %f0, %f8 | |
437 | EX_ST(STORE(std, %f8, %o0)) | |
438 | bge,pt %xcc, 93b | |
439 | add %o0, 8, %o0 | |
440 | ||
441 | 95: brz,pt %o2, 2f | |
442 | mov %g1, %o1 | |
443 | ||
444 | 1: EX_LD(LOAD(ldub, %o1, %o3)) | |
445 | add %o1, 1, %o1 | |
446 | subcc %o2, 1, %o2 | |
447 | EX_ST(STORE(stb, %o3, %o0)) | |
448 | bne,pt %xcc, 1b | |
449 | add %o0, 1, %o0 | |
450 | ||
451 | 2: membar #StoreLoad | #StoreStore | |
452 | VISExit | |
453 | retl | |
454 | mov EX_RETVAL(%o4), %o0 | |
455 | ||
456 | .align 64 | |
457 | 70: /* 16 < len <= (5 * 64) */ | |
458 | bne,pn %XCC, 75f | |
459 | sub %o0, %o1, %o3 | |
460 | ||
461 | 72: andn %o2, 0xf, %GLOBAL_SPARE | |
462 | and %o2, 0xf, %o2 | |
463 | 1: EX_LD(LOAD(ldx, %o1 + 0x00, %o5)) | |
464 | EX_LD(LOAD(ldx, %o1 + 0x08, %g1)) | |
465 | subcc %GLOBAL_SPARE, 0x10, %GLOBAL_SPARE | |
466 | EX_ST(STORE(stx, %o5, %o1 + %o3)) | |
467 | add %o1, 0x8, %o1 | |
468 | EX_ST(STORE(stx, %g1, %o1 + %o3)) | |
469 | bgu,pt %XCC, 1b | |
470 | add %o1, 0x8, %o1 | |
471 | 73: andcc %o2, 0x8, %g0 | |
472 | be,pt %XCC, 1f | |
473 | nop | |
474 | EX_LD(LOAD(ldx, %o1, %o5)) | |
475 | sub %o2, 0x8, %o2 | |
476 | EX_ST(STORE(stx, %o5, %o1 + %o3)) | |
477 | add %o1, 0x8, %o1 | |
478 | 1: andcc %o2, 0x4, %g0 | |
479 | be,pt %XCC, 1f | |
480 | nop | |
481 | EX_LD(LOAD(lduw, %o1, %o5)) | |
482 | sub %o2, 0x4, %o2 | |
483 | EX_ST(STORE(stw, %o5, %o1 + %o3)) | |
484 | add %o1, 0x4, %o1 | |
485 | 1: cmp %o2, 0 | |
486 | be,pt %XCC, 85f | |
487 | nop | |
488 | ba,pt %xcc, 90f | |
489 | nop | |
490 | ||
491 | 75: andcc %o0, 0x7, %g1 | |
492 | sub %g1, 0x8, %g1 | |
493 | be,pn %icc, 2f | |
494 | sub %g0, %g1, %g1 | |
495 | sub %o2, %g1, %o2 | |
496 | ||
497 | 1: EX_LD(LOAD(ldub, %o1, %o5)) | |
498 | subcc %g1, 1, %g1 | |
499 | EX_ST(STORE(stb, %o5, %o1 + %o3)) | |
500 | bgu,pt %icc, 1b | |
501 | add %o1, 1, %o1 | |
502 | ||
503 | 2: add %o1, %o3, %o0 | |
504 | andcc %o1, 0x7, %g1 | |
505 | bne,pt %icc, 8f | |
506 | sll %g1, 3, %g1 | |
507 | ||
508 | cmp %o2, 16 | |
509 | bgeu,pt %icc, 72b | |
510 | nop | |
511 | ba,a,pt %xcc, 73b | |
512 | ||
513 | 8: mov 64, %o3 | |
514 | andn %o1, 0x7, %o1 | |
515 | EX_LD(LOAD(ldx, %o1, %g2)) | |
516 | sub %o3, %g1, %o3 | |
517 | andn %o2, 0x7, %GLOBAL_SPARE | |
518 | sllx %g2, %g1, %g2 | |
519 | 1: EX_LD(LOAD(ldx, %o1 + 0x8, %g3)) | |
520 | subcc %GLOBAL_SPARE, 0x8, %GLOBAL_SPARE | |
521 | add %o1, 0x8, %o1 | |
522 | srlx %g3, %o3, %o5 | |
523 | or %o5, %g2, %o5 | |
524 | EX_ST(STORE(stx, %o5, %o0)) | |
525 | add %o0, 0x8, %o0 | |
526 | bgu,pt %icc, 1b | |
527 | sllx %g3, %g1, %g2 | |
528 | ||
529 | srl %g1, 3, %g1 | |
530 | andcc %o2, 0x7, %o2 | |
531 | be,pn %icc, 85f | |
532 | add %o1, %g1, %o1 | |
533 | ba,pt %xcc, 90f | |
534 | sub %o0, %o1, %o3 | |
535 | ||
536 | .align 64 | |
537 | 80: /* 0 < len <= 16 */ | |
538 | andcc %o3, 0x3, %g0 | |
539 | bne,pn %XCC, 90f | |
540 | sub %o0, %o1, %o3 | |
541 | ||
542 | 1: EX_LD(LOAD(lduw, %o1, %g1)) | |
543 | subcc %o2, 4, %o2 | |
544 | EX_ST(STORE(stw, %g1, %o1 + %o3)) | |
545 | bgu,pt %XCC, 1b | |
546 | add %o1, 4, %o1 | |
547 | ||
548 | 85: retl | |
549 | mov EX_RETVAL(%o4), %o0 | |
550 | ||
551 | .align 32 | |
552 | 90: EX_LD(LOAD(ldub, %o1, %g1)) | |
553 | subcc %o2, 1, %o2 | |
554 | EX_ST(STORE(stb, %g1, %o1 + %o3)) | |
555 | bgu,pt %XCC, 90b | |
556 | add %o1, 1, %o1 | |
557 | retl | |
558 | mov EX_RETVAL(%o4), %o0 | |
559 | ||
560 | .size FUNC_NAME, .-FUNC_NAME |