Commit | Line | Data |
---|---|---|
1da177e4 LT |
1 | /* Normally compiler builtins are used, but sometimes the compiler calls out |
2 | of line code. Based on asm-i386/string.h. | |
3 | */ | |
4 | #define _STRING_C | |
5 | #include <linux/string.h> | |
2ee60e17 | 6 | #include <linux/module.h> |
1da177e4 LT |
7 | |
8 | #undef memmove | |
e9406597 | 9 | void *memmove(void *dest, const void *src, size_t count) |
1da177e4 | 10 | { |
3b4b682b ML |
11 | unsigned long d0,d1,d2,d3,d4,d5,d6,d7; |
12 | char *ret; | |
13 | ||
14 | __asm__ __volatile__( | |
15 | /* Handle more 32bytes in loop */ | |
16 | "mov %2, %3\n\t" | |
17 | "cmp $0x20, %0\n\t" | |
18 | "jb 1f\n\t" | |
19 | ||
20 | /* Decide forward/backward copy mode */ | |
21 | "cmp %2, %1\n\t" | |
22 | "jb 2f\n\t" | |
23 | ||
24 | /* | |
25 | * movsq instruction have many startup latency | |
26 | * so we handle small size by general register. | |
27 | */ | |
28 | "cmp $680, %0\n\t" | |
29 | "jb 3f\n\t" | |
30 | /* | |
31 | * movsq instruction is only good for aligned case. | |
32 | */ | |
33 | "cmpb %%dil, %%sil\n\t" | |
34 | "je 4f\n\t" | |
35 | "3:\n\t" | |
36 | "sub $0x20, %0\n\t" | |
37 | /* | |
38 | * We gobble 32byts forward in each loop. | |
39 | */ | |
40 | "5:\n\t" | |
41 | "sub $0x20, %0\n\t" | |
42 | "movq 0*8(%1), %4\n\t" | |
43 | "movq 1*8(%1), %5\n\t" | |
44 | "movq 2*8(%1), %6\n\t" | |
45 | "movq 3*8(%1), %7\n\t" | |
46 | "leaq 4*8(%1), %1\n\t" | |
47 | ||
48 | "movq %4, 0*8(%2)\n\t" | |
49 | "movq %5, 1*8(%2)\n\t" | |
50 | "movq %6, 2*8(%2)\n\t" | |
51 | "movq %7, 3*8(%2)\n\t" | |
52 | "leaq 4*8(%2), %2\n\t" | |
53 | "jae 5b\n\t" | |
54 | "addq $0x20, %0\n\t" | |
55 | "jmp 1f\n\t" | |
56 | /* | |
57 | * Handle data forward by movsq. | |
58 | */ | |
59 | ".p2align 4\n\t" | |
60 | "4:\n\t" | |
61 | "movq %0, %8\n\t" | |
62 | "movq -8(%1, %0), %4\n\t" | |
63 | "lea -8(%2, %0), %5\n\t" | |
64 | "shrq $3, %8\n\t" | |
65 | "rep movsq\n\t" | |
66 | "movq %4, (%5)\n\t" | |
67 | "jmp 13f\n\t" | |
68 | /* | |
69 | * Handle data backward by movsq. | |
70 | */ | |
71 | ".p2align 4\n\t" | |
72 | "7:\n\t" | |
73 | "movq %0, %8\n\t" | |
74 | "movq (%1), %4\n\t" | |
75 | "movq %2, %5\n\t" | |
76 | "leaq -8(%1, %0), %1\n\t" | |
77 | "leaq -8(%2, %0), %2\n\t" | |
78 | "shrq $3, %8\n\t" | |
79 | "std\n\t" | |
80 | "rep movsq\n\t" | |
81 | "cld\n\t" | |
82 | "movq %4, (%5)\n\t" | |
83 | "jmp 13f\n\t" | |
84 | ||
85 | /* | |
86 | * Start to prepare for backward copy. | |
87 | */ | |
88 | ".p2align 4\n\t" | |
89 | "2:\n\t" | |
90 | "cmp $680, %0\n\t" | |
91 | "jb 6f \n\t" | |
92 | "cmp %%dil, %%sil\n\t" | |
93 | "je 7b \n\t" | |
94 | "6:\n\t" | |
95 | /* | |
96 | * Calculate copy position to tail. | |
97 | */ | |
98 | "addq %0, %1\n\t" | |
99 | "addq %0, %2\n\t" | |
100 | "subq $0x20, %0\n\t" | |
101 | /* | |
102 | * We gobble 32byts backward in each loop. | |
103 | */ | |
104 | "8:\n\t" | |
105 | "subq $0x20, %0\n\t" | |
106 | "movq -1*8(%1), %4\n\t" | |
107 | "movq -2*8(%1), %5\n\t" | |
108 | "movq -3*8(%1), %6\n\t" | |
109 | "movq -4*8(%1), %7\n\t" | |
110 | "leaq -4*8(%1), %1\n\t" | |
111 | ||
112 | "movq %4, -1*8(%2)\n\t" | |
113 | "movq %5, -2*8(%2)\n\t" | |
114 | "movq %6, -3*8(%2)\n\t" | |
115 | "movq %7, -4*8(%2)\n\t" | |
116 | "leaq -4*8(%2), %2\n\t" | |
117 | "jae 8b\n\t" | |
118 | /* | |
119 | * Calculate copy position to head. | |
120 | */ | |
121 | "addq $0x20, %0\n\t" | |
122 | "subq %0, %1\n\t" | |
123 | "subq %0, %2\n\t" | |
124 | "1:\n\t" | |
125 | "cmpq $16, %0\n\t" | |
126 | "jb 9f\n\t" | |
127 | /* | |
128 | * Move data from 16 bytes to 31 bytes. | |
129 | */ | |
130 | "movq 0*8(%1), %4\n\t" | |
131 | "movq 1*8(%1), %5\n\t" | |
132 | "movq -2*8(%1, %0), %6\n\t" | |
133 | "movq -1*8(%1, %0), %7\n\t" | |
134 | "movq %4, 0*8(%2)\n\t" | |
135 | "movq %5, 1*8(%2)\n\t" | |
136 | "movq %6, -2*8(%2, %0)\n\t" | |
137 | "movq %7, -1*8(%2, %0)\n\t" | |
138 | "jmp 13f\n\t" | |
139 | ".p2align 4\n\t" | |
140 | "9:\n\t" | |
141 | "cmpq $8, %0\n\t" | |
142 | "jb 10f\n\t" | |
143 | /* | |
144 | * Move data from 8 bytes to 15 bytes. | |
145 | */ | |
146 | "movq 0*8(%1), %4\n\t" | |
147 | "movq -1*8(%1, %0), %5\n\t" | |
148 | "movq %4, 0*8(%2)\n\t" | |
149 | "movq %5, -1*8(%2, %0)\n\t" | |
150 | "jmp 13f\n\t" | |
151 | "10:\n\t" | |
152 | "cmpq $4, %0\n\t" | |
153 | "jb 11f\n\t" | |
154 | /* | |
155 | * Move data from 4 bytes to 7 bytes. | |
156 | */ | |
157 | "movl (%1), %4d\n\t" | |
158 | "movl -4(%1, %0), %5d\n\t" | |
159 | "movl %4d, (%2)\n\t" | |
160 | "movl %5d, -4(%2, %0)\n\t" | |
161 | "jmp 13f\n\t" | |
162 | "11:\n\t" | |
163 | "cmp $2, %0\n\t" | |
164 | "jb 12f\n\t" | |
165 | /* | |
166 | * Move data from 2 bytes to 3 bytes. | |
167 | */ | |
168 | "movw (%1), %4w\n\t" | |
169 | "movw -2(%1, %0), %5w\n\t" | |
170 | "movw %4w, (%2)\n\t" | |
171 | "movw %5w, -2(%2, %0)\n\t" | |
172 | "jmp 13f\n\t" | |
173 | "12:\n\t" | |
174 | "cmp $1, %0\n\t" | |
175 | "jb 13f\n\t" | |
176 | /* | |
177 | * Move data for 1 byte. | |
178 | */ | |
179 | "movb (%1), %4b\n\t" | |
180 | "movb %4b, (%2)\n\t" | |
181 | "13:\n\t" | |
182 | : "=&d" (d0), "=&S" (d1), "=&D" (d2), "=&a" (ret) , | |
183 | "=r"(d3), "=r"(d4), "=r"(d5), "=r"(d6), "=&c" (d7) | |
184 | :"0" (count), | |
185 | "1" (src), | |
186 | "2" (dest) | |
187 | :"memory"); | |
188 | ||
189 | return ret; | |
190 | ||
e9406597 | 191 | } |
2ee60e17 | 192 | EXPORT_SYMBOL(memmove); |