[deliverable/linux.git] / arch / arm / lib / memmove.S

/*
 *  linux/arch/arm/lib/memmove.S
 *
 *  Author:	Nicolas Pitre
 *  Created:	Sep 28, 2005
 *  Copyright:	(C) MontaVista Software Inc.
 *
 *  This program is free software; you can redistribute it and/or modify
 *  it under the terms of the GNU General Public License version 2 as
 *  published by the Free Software Foundation.
 */

#include <linux/linkage.h>
#include <asm/assembler.h>

/*
 * This can be used to enable code to cacheline align the source pointer.
 * Experiments on tested architectures (StrongARM and XScale) didn't show
 * this a worthwhile thing to do.  That might be different in the future.
 */
//#define CALGN(code...)        code
#define CALGN(code...)

		.text

/*
 * Prototype: void *memmove(void *dest, const void *src, size_t n);
 *
 * Note:
 *
 * If the memory regions don't overlap, we simply branch to memcpy which is
 * normally a bit faster. Otherwise the copy is done going downwards.  This
 * is a transposition of the code from copy_template.S but with the copy
 * occurring in the opposite direction.
 */

ENTRY(memmove)

		subs	ip, r0, r1
		cmphi	r2, ip
		bls	memcpy

		stmfd	sp!, {r0, r4, lr}
		add	r1, r1, r2
		add	r0, r0, r2
		subs	r2, r2, #4
		blt	8f
		ands	ip, r0, #3
	PLD(	pld	[r1, #-4]		)
		bne	9f
		ands	ip, r1, #3
		bne	10f

1:		subs	r2, r2, #(28)
		stmfd	sp!, {r5 - r8}
		blt	5f

	CALGN(	ands	ip, r1, #31		)
	CALGN(	sbcnes	r4, ip, r2		)  @ C is always set here
	CALGN(	bcs	2f			)
	CALGN(	adr	r4, 6f			)
	CALGN(	subs	r2, r2, ip		)  @ C is set here
	CALGN(	add	pc, r4, ip		)

	PLD(	pld	[r1, #-4]		)
2:	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #-32]		)
	PLD(	blt	4f			)
	PLD(	pld	[r1, #-64]		)
	PLD(	pld	[r1, #-96]		)

3:	PLD(	pld	[r1, #-128]		)
4:		ldmdb	r1!, {r3, r4, r5, r6, r7, r8, ip, lr}
		subs	r2, r2, #32
		stmdb	r0!, {r3, r4, r5, r6, r7, r8, ip, lr}
		bge	3b
	PLD(	cmn	r2, #96			)
	PLD(	bge	4b			)

5:		ands	ip, r2, #28
		rsb	ip, ip, #32
		addne	pc, pc, ip		@ C is always clear here
		b	7f
6:		nop
		ldr	r3, [r1, #-4]!
		ldr	r4, [r1, #-4]!
		ldr	r5, [r1, #-4]!
		ldr	r6, [r1, #-4]!
		ldr	r7, [r1, #-4]!
		ldr	r8, [r1, #-4]!
		ldr	lr, [r1, #-4]!

		add	pc, pc, ip
		nop
		nop
		str	r3, [r0, #-4]!
		str	r4, [r0, #-4]!
		str	r5, [r0, #-4]!
		str	r6, [r0, #-4]!
		str	r7, [r0, #-4]!
		str	r8, [r0, #-4]!
		str	lr, [r0, #-4]!

	CALGN(	bcs	2b			)

7:		ldmfd	sp!, {r5 - r8}

8:		movs	r2, r2, lsl #31
		ldrneb	r3, [r1, #-1]!
		ldrcsb	r4, [r1, #-1]!
		ldrcsb	ip, [r1, #-1]
		strneb	r3, [r0, #-1]!
		strcsb	r4, [r0, #-1]!
		strcsb	ip, [r0, #-1]
		ldmfd	sp!, {r0, r4, pc}

9:		cmp	ip, #2
		ldrgtb	r3, [r1, #-1]!
		ldrgeb	r4, [r1, #-1]!
		ldrb	lr, [r1, #-1]!
		strgtb	r3, [r0, #-1]!
		strgeb	r4, [r0, #-1]!
		subs	r2, r2, ip
		strb	lr, [r0, #-1]!
		blt	8b
		ands	ip, r1, #3
		beq	1b

10:		bic	r1, r1, #3
		cmp	ip, #2
		ldr	r3, [r1, #0]
		beq	17f
		blt	18f


		.macro	backward_copy_shift push pull

		subs	r2, r2, #28
		blt	14f

	CALGN(	ands	ip, r1, #31		)
	CALGN(	rsb	ip, ip, #32		)
	CALGN(	sbcnes	r4, ip, r2		)  @ C is always set here
	CALGN(	subcc	r2, r2, ip		)
	CALGN(	bcc	15f			)

11:		stmfd	sp!, {r5 - r9}

	PLD(	pld	[r1, #-4]		)
	PLD(	subs	r2, r2, #96		)
	PLD(	pld	[r1, #-32]		)
	PLD(	blt	13f			)
	PLD(	pld	[r1, #-64]		)
	PLD(	pld	[r1, #-96]		)

12:	PLD(	pld	[r1, #-128]		)
13:		ldmdb   r1!, {r7, r8, r9, ip}
		mov     lr, r3, push #\push
		subs    r2, r2, #32
		ldmdb   r1!, {r3, r4, r5, r6}
		orr     lr, lr, ip, pull #\pull
		mov     ip, ip, push #\push
		orr     ip, ip, r9, pull #\pull
		mov     r9, r9, push #\push
		orr     r9, r9, r8, pull #\pull
		mov     r8, r8, push #\push
		orr     r8, r8, r7, pull #\pull
		mov     r7, r7, push #\push
		orr     r7, r7, r6, pull #\pull
		mov     r6, r6, push #\push
		orr     r6, r6, r5, pull #\pull
		mov     r5, r5, push #\push
		orr     r5, r5, r4, pull #\pull
		mov     r4, r4, push #\push
		orr     r4, r4, r3, pull #\pull
		stmdb   r0!, {r4 - r9, ip, lr}
		bge	12b
	PLD(	cmn	r2, #96			)
	PLD(	bge	13b			)

		ldmfd	sp!, {r5 - r9}

14:		ands	ip, r2, #28
		beq	16f

15:		mov     lr, r3, push #\push
		ldr	r3, [r1, #-4]!
		subs	ip, ip, #4
		orr	lr, lr, r3, pull #\pull
		str	lr, [r0, #-4]!
		bgt	15b
	CALGN(	cmp	r2, #0			)
	CALGN(	bge	11b			)

16:		add	r1, r1, #(\pull / 8)
		b	8b

		.endm


		backward_copy_shift	push=8	pull=24

17:		backward_copy_shift	push=16	pull=16

18:		backward_copy_shift	push=24	pull=8
Commit	Line	Data
75494230 NP	1	/*
	2	* linux/arch/arm/lib/memmove.S
	3	*
	4	* Author: Nicolas Pitre
	5	* Created: Sep 28, 2005
	6	* Copyright: (C) MontaVista Software Inc.
	7	*
	8	* This program is free software; you can redistribute it and/or modify
	9	* it under the terms of the GNU General Public License version 2 as
	10	* published by the Free Software Foundation.
	11	*/
	12
	13	#include <linux/linkage.h>
	14	#include <asm/assembler.h>
	15
	16	/*
	17	* This can be used to enable code to cacheline align the source pointer.
	18	* Experiments on tested architectures (StrongARM and XScale) didn't show
	19	* this a worthwhile thing to do. That might be different in the future.
	20	*/
	21	//#define CALGN(code...) code
	22	#define CALGN(code...)
	23
	24	.text
	25
	26	/*
	27	* Prototype: void memmove(void dest, const void *src, size_t n);
	28	*
	29	* Note:
	30	*
	31	* If the memory regions don't overlap, we simply branch to memcpy which is
	32	* normally a bit faster. Otherwise the copy is done going downwards. This
	33	* is a transposition of the code from copy_template.S but with the copy
	34	* occurring in the opposite direction.
	35	*/
	36
	37	ENTRY(memmove)
	38
	39	subs ip, r0, r1
	40	cmphi r2, ip
	41	bls memcpy
	42
	43	stmfd sp!, {r0, r4, lr}
	44	add r1, r1, r2
	45	add r0, r0, r2
	46	subs r2, r2, #4
	47	blt 8f
	48	ands ip, r0, #3
	49	PLD( pld [r1, #-4] )
	50	bne 9f
	51	ands ip, r1, #3
	52	bne 10f
	53
	54	1: subs r2, r2, #(28)
	55	stmfd sp!, {r5 - r8}
	56	blt 5f
	57
	58	CALGN( ands ip, r1, #31 )
	59	CALGN( sbcnes r4, ip, r2 ) @ C is always set here
	60	CALGN( bcs 2f )
	61	CALGN( adr r4, 6f )
	62	CALGN( subs r2, r2, ip ) @ C is set here
	63	CALGN( add pc, r4, ip )
	64
65	PLD( pld [r1, #-4] )
66	2: PLD( subs r2, r2, #96 )
67	PLD( pld [r1, #-32] )
68	PLD( blt 4f )
69	PLD( pld [r1, #-64] )
70	PLD( pld [r1, #-96] )
71
72	3: PLD( pld [r1, #-128] )
73	4: ldmdb r1!, {r3, r4, r5, r6, r7, r8, ip, lr}
74	subs r2, r2, #32
75	stmdb r0!, {r3, r4, r5, r6, r7, r8, ip, lr}
76	bge 3b
77	PLD( cmn r2, #96 )
78	PLD( bge 4b )
79
80	5: ands ip, r2, #28
81	rsb ip, ip, #32
82	addne pc, pc, ip @ C is always clear here
83	b 7f
84	6: nop
85	ldr r3, [r1, #-4]!
86	ldr r4, [r1, #-4]!
87	ldr r5, [r1, #-4]!
88	ldr r6, [r1, #-4]!
89	ldr r7, [r1, #-4]!
90	ldr r8, [r1, #-4]!
91	ldr lr, [r1, #-4]!
92
93	add pc, pc, ip
94	nop
95	nop
96	str r3, [r0, #-4]!
97	str r4, [r0, #-4]!
98	str r5, [r0, #-4]!
99	str r6, [r0, #-4]!
100	str r7, [r0, #-4]!
101	str r8, [r0, #-4]!
102	str lr, [r0, #-4]!
103
104	CALGN( bcs 2b )
105
106	7: ldmfd sp!, {r5 - r8}
107
108	8: movs r2, r2, lsl #31
109	ldrneb r3, [r1, #-1]!
110	ldrcsb r4, [r1, #-1]!
111	ldrcsb ip, [r1, #-1]
112	strneb r3, [r0, #-1]!
113	strcsb r4, [r0, #-1]!
114	strcsb ip, [r0, #-1]
115	ldmfd sp!, {r0, r4, pc}
116
117	9: cmp ip, #2
118	ldrgtb r3, [r1, #-1]!
119	ldrgeb r4, [r1, #-1]!
120	ldrb lr, [r1, #-1]!
121	strgtb r3, [r0, #-1]!
122	strgeb r4, [r0, #-1]!
123	subs r2, r2, ip
124	strb lr, [r0, #-1]!
125	blt 8b
126	ands ip, r1, #3
127	beq 1b
128
129	10: bic r1, r1, #3
130	cmp ip, #2
131	ldr r3, [r1, #0]
132	beq 17f
133	blt 18f
134
135
136	.macro backward_copy_shift push pull
137
138	subs r2, r2, #28
139	blt 14f
140
141	CALGN( ands ip, r1, #31 )
142	CALGN( rsb ip, ip, #32 )
143	CALGN( sbcnes r4, ip, r2 ) @ C is always set here
144	CALGN( subcc r2, r2, ip )
145	CALGN( bcc 15f )
146
147	11: stmfd sp!, {r5 - r9}
148
149	PLD( pld [r1, #-4] )
150	PLD( subs r2, r2, #96 )
151	PLD( pld [r1, #-32] )
152	PLD( blt 13f )
153	PLD( pld [r1, #-64] )
154	PLD( pld [r1, #-96] )
155
156	12: PLD( pld [r1, #-128] )
157	13: ldmdb r1!, {r7, r8, r9, ip}
158	mov lr, r3, push #\push
159	subs r2, r2, #32
160	ldmdb r1!, {r3, r4, r5, r6}
161	orr lr, lr, ip, pull #\pull
162	mov ip, ip, push #\push
163	orr ip, ip, r9, pull #\pull
164	mov r9, r9, push #\push
165	orr r9, r9, r8, pull #\pull
166	mov r8, r8, push #\push
167	orr r8, r8, r7, pull #\pull
168	mov r7, r7, push #\push
169	orr r7, r7, r6, pull #\pull
170	mov r6, r6, push #\push
171	orr r6, r6, r5, pull #\pull
172	mov r5, r5, push #\push
173	orr r5, r5, r4, pull #\pull
174	mov r4, r4, push #\push
175	orr r4, r4, r3, pull #\pull
176	stmdb r0!, {r4 - r9, ip, lr}
177	bge 12b
178	PLD( cmn r2, #96 )
179	PLD( bge 13b )
180
181	ldmfd sp!, {r5 - r9}
182
183	14: ands ip, r2, #28
184	beq 16f
185
186	15: mov lr, r3, push #\push
187	ldr r3, [r1, #-4]!
188	subs ip, ip, #4
189	orr lr, lr, r3, pull #\pull
190	str lr, [r0, #-4]!
191	bgt 15b
192	CALGN( cmp r2, #0 )
193	CALGN( bge 11b )
194
195	16: add r1, r1, #(\pull / 8)
196	b 8b
197
198	.endm
199
200
201	backward_copy_shift push=8 pull=24
202
203	17: backward_copy_shift push=16 pull=16
204
205	18: backward_copy_shift push=24 pull=8
206