[deliverable/linux.git] / arch / x86 / lib / memmove_64.S

/*
 * Normally compiler builtins are used, but sometimes the compiler calls out
 * of line code. Based on asm-i386/string.h.
 *
 * This assembly file is re-written from memmove_64.c file.
 *	- Copyright 2011 Fenghua Yu <fenghua.yu@intel.com>
 */
#define _STRING_C
#include <linux/linkage.h>
#include <asm/dwarf2.h>
#include <asm/cpufeature.h>
#include <asm/alternative-asm.h>

#undef memmove

/*
 * Implement memmove(). This can handle overlap between src and dst.
 *
 * Input:
 * rdi: dest
 * rsi: src
 * rdx: count
 *
 * Output:
 * rax: dest
 */
ENTRY(memmove)
	CFI_STARTPROC

	/* Handle more 32 bytes in loop */
	mov %rdi, %rax
	cmp $0x20, %rdx
	jb	1f

	/* Decide forward/backward copy mode */
	cmp %rdi, %rsi
	jge .Lmemmove_begin_forward
	mov %rsi, %r8
	add %rdx, %r8
	cmp %rdi, %r8
	jg 2f

.Lmemmove_begin_forward:
	/*
	 * movsq instruction have many startup latency
	 * so we handle small size by general register.
	 */
	cmp  $680, %rdx
	jb	3f
	/*
	 * movsq instruction is only good for aligned case.
	 */

	cmpb %dil, %sil
	je 4f
3:
	sub $0x20, %rdx
	/*
	 * We gobble 32 bytes forward in each loop.
	 */
5:
	sub $0x20, %rdx
	movq 0*8(%rsi), %r11
	movq 1*8(%rsi), %r10
	movq 2*8(%rsi), %r9
	movq 3*8(%rsi), %r8
	leaq 4*8(%rsi), %rsi

	movq %r11, 0*8(%rdi)
	movq %r10, 1*8(%rdi)
	movq %r9, 2*8(%rdi)
	movq %r8, 3*8(%rdi)
	leaq 4*8(%rdi), %rdi
	jae 5b
	addq $0x20, %rdx
	jmp 1f
	/*
	 * Handle data forward by movsq.
	 */
	.p2align 4
4:
	movq %rdx, %rcx
	movq -8(%rsi, %rdx), %r11
	lea -8(%rdi, %rdx), %r10
	shrq $3, %rcx
	rep movsq
	movq %r11, (%r10)
	jmp 13f
.Lmemmove_end_forward:

	/*
	 * Handle data backward by movsq.
	 */
	.p2align 4
7:
	movq %rdx, %rcx
	movq (%rsi), %r11
	movq %rdi, %r10
	leaq -8(%rsi, %rdx), %rsi
	leaq -8(%rdi, %rdx), %rdi
	shrq $3, %rcx
	std
	rep movsq
	cld
	movq %r11, (%r10)
	jmp 13f

	/*
	 * Start to prepare for backward copy.
	 */
	.p2align 4
2:
	cmp $680, %rdx
	jb 6f
	cmp %dil, %sil
	je 7b
6:
	/*
	 * Calculate copy position to tail.
	 */
	addq %rdx, %rsi
	addq %rdx, %rdi
	subq $0x20, %rdx
	/*
	 * We gobble 32 bytes backward in each loop.
	 */
8:
	subq $0x20, %rdx
	movq -1*8(%rsi), %r11
	movq -2*8(%rsi), %r10
	movq -3*8(%rsi), %r9
	movq -4*8(%rsi), %r8
	leaq -4*8(%rsi), %rsi

	movq %r11, -1*8(%rdi)
	movq %r10, -2*8(%rdi)
	movq %r9, -3*8(%rdi)
	movq %r8, -4*8(%rdi)
	leaq -4*8(%rdi), %rdi
	jae 8b
	/*
	 * Calculate copy position to head.
	 */
	addq $0x20, %rdx
	subq %rdx, %rsi
	subq %rdx, %rdi
1:
	cmpq $16, %rdx
	jb 9f
	/*
	 * Move data from 16 bytes to 31 bytes.
	 */
	movq 0*8(%rsi), %r11
	movq 1*8(%rsi), %r10
	movq -2*8(%rsi, %rdx), %r9
	movq -1*8(%rsi, %rdx), %r8
	movq %r11, 0*8(%rdi)
	movq %r10, 1*8(%rdi)
	movq %r9, -2*8(%rdi, %rdx)
	movq %r8, -1*8(%rdi, %rdx)
	jmp 13f
	.p2align 4
9:
	cmpq $8, %rdx
	jb 10f
	/*
	 * Move data from 8 bytes to 15 bytes.
	 */
	movq 0*8(%rsi), %r11
	movq -1*8(%rsi, %rdx), %r10
	movq %r11, 0*8(%rdi)
	movq %r10, -1*8(%rdi, %rdx)
	jmp 13f
10:
	cmpq $4, %rdx
	jb 11f
	/*
	 * Move data from 4 bytes to 7 bytes.
	 */
	movl (%rsi), %r11d
	movl -4(%rsi, %rdx), %r10d
	movl %r11d, (%rdi)
	movl %r10d, -4(%rdi, %rdx)
	jmp 13f
11:
	cmp $2, %rdx
	jb 12f
	/*
	 * Move data from 2 bytes to 3 bytes.
	 */
	movw (%rsi), %r11w
	movw -2(%rsi, %rdx), %r10w
	movw %r11w, (%rdi)
	movw %r10w, -2(%rdi, %rdx)
	jmp 13f
12:
	cmp $1, %rdx
	jb 13f
	/*
	 * Move data for 1 byte.
	 */
	movb (%rsi), %r11b
	movb %r11b, (%rdi)
13:
	retq
	CFI_ENDPROC

	.section .altinstr_replacement,"ax"
.Lmemmove_begin_forward_efs:
	/* Forward moving data. */
	movq %rdx, %rcx
	rep movsb
	retq
.Lmemmove_end_forward_efs:
	.previous

	.section .altinstructions,"a"
	altinstruction_entry .Lmemmove_begin_forward,		\
		.Lmemmove_begin_forward_efs,X86_FEATURE_ERMS,	\
		.Lmemmove_end_forward-.Lmemmove_begin_forward,	\
		.Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs
	.previous
ENDPROC(memmove)
Commit	Line	Data
9599ec04 FY	1	/*
	2	* Normally compiler builtins are used, but sometimes the compiler calls out
	3	* of line code. Based on asm-i386/string.h.
	4	*
	5	* This assembly file is re-written from memmove_64.c file.
	6	* - Copyright 2011 Fenghua Yu <fenghua.yu@intel.com>
	7	*/
	8	#define _STRING_C
	9	#include <linux/linkage.h>
	10	#include <asm/dwarf2.h>
057e05c1	11	#include <asm/cpufeature.h>
59e97e4d	12	#include <asm/alternative-asm.h>
9599ec04 FY	13
	14	#undef memmove
	15
	16	/*
	17	* Implement memmove(). This can handle overlap between src and dst.
	18	*
	19	* Input:
	20	* rdi: dest
	21	* rsi: src
	22	* rdx: count
	23	*
	24	* Output:
	25	* rax: dest
	26	*/
	27	ENTRY(memmove)
	28	CFI_STARTPROC
057e05c1	29
d50ba368	30	/* Handle more 32 bytes in loop */
9599ec04 FY	31	mov %rdi, %rax
	32	cmp $0x20, %rdx
	33	jb 1f
	34
	35	/* Decide forward/backward copy mode */
	36	cmp %rdi, %rsi
057e05c1 FY	37	jge .Lmemmove_begin_forward
	38	mov %rsi, %r8
	39	add %rdx, %r8
	40	cmp %rdi, %r8
	41	jg 2f
9599ec04	42
057e05c1	43	.Lmemmove_begin_forward:
9599ec04 FY	44	/*
	45	* movsq instruction have many startup latency
	46	* so we handle small size by general register.
	47	*/
	48	cmp $680, %rdx
	49	jb 3f
	50	/*
	51	* movsq instruction is only good for aligned case.
	52	*/
	53
	54	cmpb %dil, %sil
	55	je 4f
	56	3:
	57	sub $0x20, %rdx
	58	/*
bb916ff7	59	* We gobble 32 bytes forward in each loop.
9599ec04 FY	60	*/
	61	5:
	62	sub $0x20, %rdx
	63	movq 0*8(%rsi), %r11
	64	movq 1*8(%rsi), %r10
	65	movq 2*8(%rsi), %r9
	66	movq 3*8(%rsi), %r8
	67	leaq 4*8(%rsi), %rsi
	68
	69	movq %r11, 0*8(%rdi)
	70	movq %r10, 1*8(%rdi)
	71	movq %r9, 2*8(%rdi)
	72	movq %r8, 3*8(%rdi)
	73	leaq 4*8(%rdi), %rdi
	74	jae 5b
	75	addq $0x20, %rdx
	76	jmp 1f
	77	/*
	78	* Handle data forward by movsq.
	79	*/
	80	.p2align 4
	81	4:
	82	movq %rdx, %rcx
	83	movq -8(%rsi, %rdx), %r11
	84	lea -8(%rdi, %rdx), %r10
	85	shrq $3, %rcx
	86	rep movsq
	87	movq %r11, (%r10)
	88	jmp 13f
057e05c1 FY	89	.Lmemmove_end_forward:
057e05c1 FY	90
9599ec04 FY	91	/*
	92	* Handle data backward by movsq.
	93	*/
	94	.p2align 4
	95	7:
	96	movq %rdx, %rcx
	97	movq (%rsi), %r11
	98	movq %rdi, %r10
	99	leaq -8(%rsi, %rdx), %rsi
	100	leaq -8(%rdi, %rdx), %rdi
	101	shrq $3, %rcx
	102	std
	103	rep movsq
	104	cld
	105	movq %r11, (%r10)
	106	jmp 13f
	107
	108	/*
	109	* Start to prepare for backward copy.
	110	*/
	111	.p2align 4
	112	2:
	113	cmp $680, %rdx
	114	jb 6f
	115	cmp %dil, %sil
	116	je 7b
	117	6:
	118	/*
	119	* Calculate copy position to tail.
	120	*/
	121	addq %rdx, %rsi
	122	addq %rdx, %rdi
	123	subq $0x20, %rdx
	124	/*
bb916ff7	125	* We gobble 32 bytes backward in each loop.
9599ec04 FY	126	*/
	127	8:
	128	subq $0x20, %rdx
	129	movq -1*8(%rsi), %r11
	130	movq -2*8(%rsi), %r10
	131	movq -3*8(%rsi), %r9
	132	movq -4*8(%rsi), %r8
	133	leaq -4*8(%rsi), %rsi
	134
	135	movq %r11, -1*8(%rdi)
	136	movq %r10, -2*8(%rdi)
	137	movq %r9, -3*8(%rdi)
	138	movq %r8, -4*8(%rdi)
	139	leaq -4*8(%rdi), %rdi
	140	jae 8b
	141	/*
	142	* Calculate copy position to head.
	143	*/
	144	addq $0x20, %rdx
	145	subq %rdx, %rsi
	146	subq %rdx, %rdi
	147	1:
	148	cmpq $16, %rdx
	149	jb 9f
	150	/*
	151	* Move data from 16 bytes to 31 bytes.
	152	*/
	153	movq 0*8(%rsi), %r11
	154	movq 1*8(%rsi), %r10
	155	movq -2*8(%rsi, %rdx), %r9
	156	movq -1*8(%rsi, %rdx), %r8
	157	movq %r11, 0*8(%rdi)
	158	movq %r10, 1*8(%rdi)
	159	movq %r9, -2*8(%rdi, %rdx)
	160	movq %r8, -1*8(%rdi, %rdx)
	161	jmp 13f
	162	.p2align 4
	163	9:
	164	cmpq $8, %rdx
	165	jb 10f
	166	/*
	167	* Move data from 8 bytes to 15 bytes.
	168	*/
	169	movq 0*8(%rsi), %r11
	170	movq -1*8(%rsi, %rdx), %r10
	171	movq %r11, 0*8(%rdi)
	172	movq %r10, -1*8(%rdi, %rdx)
	173	jmp 13f
	174	10:
	175	cmpq $4, %rdx
	176	jb 11f
	177	/*
	178	* Move data from 4 bytes to 7 bytes.
	179	*/
	180	movl (%rsi), %r11d
	181	movl -4(%rsi, %rdx), %r10d
	182	movl %r11d, (%rdi)
	183	movl %r10d, -4(%rdi, %rdx)
	184	jmp 13f
	185	11:
	186	cmp $2, %rdx
	187	jb 12f
	188	/*
	189	* Move data from 2 bytes to 3 bytes.
190	*/
191	movw (%rsi), %r11w
192	movw -2(%rsi, %rdx), %r10w
193	movw %r11w, (%rdi)
194	movw %r10w, -2(%rdi, %rdx)
195	jmp 13f
196	12:
197	cmp $1, %rdx
198	jb 13f
199	/*
200	* Move data for 1 byte.
201	*/
202	movb (%rsi), %r11b
203	movb %r11b, (%rdi)
204	13:
205	retq
206	CFI_ENDPROC
057e05c1 FY	207
	208	.section .altinstr_replacement,"ax"
	209	.Lmemmove_begin_forward_efs:
	210	/* Forward moving data. */
	211	movq %rdx, %rcx
	212	rep movsb
	213	retq
	214	.Lmemmove_end_forward_efs:
	215	.previous
	216
	217	.section .altinstructions,"a"
59e97e4d AL	218	altinstruction_entry .Lmemmove_begin_forward, \
	219	.Lmemmove_begin_forward_efs,X86_FEATURE_ERMS, \
	220	.Lmemmove_end_forward-.Lmemmove_begin_forward, \
	221	.Lmemmove_end_forward_efs-.Lmemmove_begin_forward_efs
057e05c1	222	.previous
9599ec04	223	ENDPROC(memmove)