1 /* Copyright 2002 Andi Kleen */
3 #include <linux/linkage.h>
4 #include <asm/cpufeatures.h>
5 #include <asm/alternative-asm.h>
8 * We build a jump to memcpy_orig by default which gets NOPped out on
9 * the majority of x86 CPUs which set REP_GOOD. In addition, CPUs which
10 * have the enhanced REP MOVSB/STOSB feature (ERMS), change those NOPs
11 * to a jmp to memcpy_erms which does the REP; MOVSB mem copy.
17 * memcpy - Copy a memory block.
25 * rax original destination
29 ALTERNATIVE_2 "jmp memcpy_orig", "", X86_FEATURE_REP_GOOD, \
30 "jmp memcpy_erms", X86_FEATURE_ERMS
44 * memcpy_erms() - enhanced fast string memcpy. This is faster and
45 * simpler than memcpy. Use memcpy_erms when possible.
61 * We check whether memory false dependence could occur,
62 * then jump to corresponding copy mode.
71 * Move in blocks of 4x8 bytes:
84 jae .Lcopy_forward_loop
90 * Calculate copy position to tail.
96 * At most 3 ALU operations in one cycle,
97 * so append NOPS in the same 16 bytes trunk.
100 .Lcopy_backward_loop:
104 movq -3*8(%rsi), %r10
105 movq -4*8(%rsi), %r11
106 leaq -4*8(%rsi), %rsi
109 movq %r10, -3*8(%rdi)
110 movq %r11, -4*8(%rdi)
111 leaq -4*8(%rdi), %rdi
112 jae .Lcopy_backward_loop
115 * Calculate copy position to head.
125 * Move data from 16 bytes to 31 bytes.
129 movq -2*8(%rsi, %rdx), %r10
130 movq -1*8(%rsi, %rdx), %r11
133 movq %r10, -2*8(%rdi, %rdx)
134 movq %r11, -1*8(%rdi, %rdx)
141 * Move data from 8 bytes to 15 bytes.
144 movq -1*8(%rsi, %rdx), %r9
146 movq %r9, -1*8(%rdi, %rdx)
154 * Move data from 4 bytes to 7 bytes.
157 movl -4(%rsi, %rdx), %r8d
159 movl %r8d, -4(%rdi, %rdx)
166 * Move data from 1 bytes to 3 bytes.
171 movzbq (%rsi, %rdx), %r9
173 movb %r9b, (%rdi, %rdx)
183 * memcpy_mcsafe - memory copy with machine check exception handling
184 * Note that we only catch machine checks when reading the source addresses.
185 * Writes to target are posted and don't generate machine checks.
189 /* Less than 8 bytes? Go to byte copy loop */
192 /* Check for bad alignment of source */
194 /* Already aligned */
197 /* Copy one byte at a time until source is 8-byte aligned */
203 .L_copy_leading_bytes:
209 jnz .L_copy_leading_bytes
212 /* Figure out how many whole cache lines (64-bytes) to copy */
216 jz .L_no_whole_cache_lines
218 /* Loop copying whole cache lines */
219 .L_cache_w0: movq (%rsi), %r8
220 .L_cache_w1: movq 1*8(%rsi), %r9
221 .L_cache_w2: movq 2*8(%rsi), %r10
222 .L_cache_w3: movq 3*8(%rsi), %r11
227 .L_cache_w4: movq 4*8(%rsi), %r8
228 .L_cache_w5: movq 5*8(%rsi), %r9
229 .L_cache_w6: movq 6*8(%rsi), %r10
230 .L_cache_w7: movq 7*8(%rsi), %r11
240 /* Are there any trailing 8-byte words? */
241 .L_no_whole_cache_lines:
247 /* Copy trailing words */
248 .L_copy_trailing_words:
254 jnz .L_copy_trailing_words
256 /* Any trailing bytes? */
259 jz .L_done_memcpy_trap
261 /* Copy trailing bytes */
263 .L_copy_trailing_bytes:
269 jnz .L_copy_trailing_bytes
271 /* Copy successful. Return true */
275 ENDPROC(memcpy_mcsafe)
277 .section .fixup, "ax"
278 /* Return false for any failure */
279 .L_memcpy_mcsafe_fail:
285 _ASM_EXTABLE_FAULT(.L_copy_leading_bytes, .L_memcpy_mcsafe_fail)
286 _ASM_EXTABLE_FAULT(.L_cache_w0, .L_memcpy_mcsafe_fail)
287 _ASM_EXTABLE_FAULT(.L_cache_w1, .L_memcpy_mcsafe_fail)
288 _ASM_EXTABLE_FAULT(.L_cache_w3, .L_memcpy_mcsafe_fail)
289 _ASM_EXTABLE_FAULT(.L_cache_w3, .L_memcpy_mcsafe_fail)
290 _ASM_EXTABLE_FAULT(.L_cache_w4, .L_memcpy_mcsafe_fail)
291 _ASM_EXTABLE_FAULT(.L_cache_w5, .L_memcpy_mcsafe_fail)
292 _ASM_EXTABLE_FAULT(.L_cache_w6, .L_memcpy_mcsafe_fail)
293 _ASM_EXTABLE_FAULT(.L_cache_w7, .L_memcpy_mcsafe_fail)
294 _ASM_EXTABLE_FAULT(.L_copy_trailing_words, .L_memcpy_mcsafe_fail)
295 _ASM_EXTABLE_FAULT(.L_copy_trailing_bytes, .L_memcpy_mcsafe_fail)