[deliverable/linux.git] / arch / arm / lib / uaccess.S

/*
 *  linux/arch/arm/lib/uaccess.S
 *
 *  Copyright (C) 1995, 1996,1997,1998 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 *  Routines to block copy data to/from user memory
 *   These are highly optimised both for the 4k page size
 *   and for various alignments.
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/errno.h>
#include <asm/domain.h>

		.text

#define PAGE_SHIFT 12

/* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
 * Purpose  : copy a block to user memory from kernel memory
 * Params   : to   - user memory
 *          : from - kernel memory
 *          : n    - number of bytes to copy
 * Returns  : Number of bytes NOT copied.
 */

.Lc2u_dest_not_aligned:
		rsb	ip, ip, #4
		cmp	ip, #2
		ldrb	r3, [r1], #1
USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
		ldrgeb	r3, [r1], #1
USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #1
USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
		sub	r2, r2, ip
		b	.Lc2u_dest_aligned

ENTRY(__copy_to_user)
		stmfd	sp!, {r2, r4 - r7, lr}
		cmp	r2, #4
		blt	.Lc2u_not_enough
		ands	ip, r0, #3
		bne	.Lc2u_dest_not_aligned
.Lc2u_dest_aligned:

		ands	ip, r1, #3
		bne	.Lc2u_src_not_aligned
/*
 * Seeing as there has to be at least 8 bytes to copy, we can
 * copy one word, and force a user-mode page fault...
 */

.Lc2u_0fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lc2u_0nowords
		ldr	r3, [r1], #4
USER(	TUSER(	str)	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT	@ On each page, use a ld/st??t instruction
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lc2u_0fupi
/*
 * ip = max no. of bytes to copy before needing another "strt" insn
 */
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #32
		blt	.Lc2u_0rem8lp

.Lc2u_0cpy8lp:	ldmia	r1!, {r3 - r6}
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		ldmia	r1!, {r3 - r6}
		subs	ip, ip, #32
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		bpl	.Lc2u_0cpy8lp

.Lc2u_0rem8lp:	cmn	ip, #16
		ldmgeia	r1!, {r3 - r6}
		stmgeia	r0!, {r3 - r6}			@ Shouldnt fault
		tst	ip, #8
		ldmneia	r1!, {r3 - r4}
		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
		tst	ip, #4
		ldrne	r3, [r1], #4
	TUSER(	strne) r3, [r0], #4			@ Shouldnt fault
		ands	ip, ip, #3
		beq	.Lc2u_0fupi
.Lc2u_0nowords:	teq	ip, #0
		beq	.Lc2u_finished
.Lc2u_nowords:	cmp	ip, #2
		ldrb	r3, [r1], #1
USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
		ldrgeb	r3, [r1], #1
USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #1
USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
		b	.Lc2u_finished

.Lc2u_not_enough:
		movs	ip, r2
		bne	.Lc2u_nowords
.Lc2u_finished:	mov	r0, #0
		ldmfd	sp!, {r2, r4 - r7, pc}

.Lc2u_src_not_aligned:
		bic	r1, r1, #3
		ldr	r7, [r1], #4
		cmp	ip, #2
		bgt	.Lc2u_3fupi
		beq	.Lc2u_2fupi
.Lc2u_1fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lc2u_1nowords
		mov	r3, r7, lspull #8
		ldr	r7, [r1], #4
		orr	r3, r3, r7, lspush #24
USER(	TUSER(	str)	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lc2u_1fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lc2u_1rem8lp

.Lc2u_1cpy8lp:	mov	r3, r7, lspull #8
		ldmia	r1!, {r4 - r7}
		subs	ip, ip, #16
		orr	r3, r3, r4, lspush #24
		mov	r4, r4, lspull #8
		orr	r4, r4, r5, lspush #24
		mov	r5, r5, lspull #8
		orr	r5, r5, r6, lspush #24
		mov	r6, r6, lspull #8
		orr	r6, r6, r7, lspush #24
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		bpl	.Lc2u_1cpy8lp

.Lc2u_1rem8lp:	tst	ip, #8
		movne	r3, r7, lspull #8
		ldmneia	r1!, {r4, r7}
		orrne	r3, r3, r4, lspush #24
		movne	r4, r4, lspull #8
		orrne	r4, r4, r7, lspush #24
		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
		tst	ip, #4
		movne	r3, r7, lspull #8
		ldrne	r7, [r1], #4
		orrne	r3, r3, r7, lspush #24
	TUSER(	strne) r3, [r0], #4			@ Shouldnt fault
		ands	ip, ip, #3
		beq	.Lc2u_1fupi
.Lc2u_1nowords:	mov	r3, r7, get_byte_1
		teq	ip, #0
		beq	.Lc2u_finished
		cmp	ip, #2
USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
		movge	r3, r7, get_byte_2
USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
		movgt	r3, r7, get_byte_3
USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
		b	.Lc2u_finished

.Lc2u_2fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lc2u_2nowords
		mov	r3, r7, lspull #16
		ldr	r7, [r1], #4
		orr	r3, r3, r7, lspush #16
USER(	TUSER(	str)	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lc2u_2fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lc2u_2rem8lp

.Lc2u_2cpy8lp:	mov	r3, r7, lspull #16
		ldmia	r1!, {r4 - r7}
		subs	ip, ip, #16
		orr	r3, r3, r4, lspush #16
		mov	r4, r4, lspull #16
		orr	r4, r4, r5, lspush #16
		mov	r5, r5, lspull #16
		orr	r5, r5, r6, lspush #16
		mov	r6, r6, lspull #16
		orr	r6, r6, r7, lspush #16
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		bpl	.Lc2u_2cpy8lp

.Lc2u_2rem8lp:	tst	ip, #8
		movne	r3, r7, lspull #16
		ldmneia	r1!, {r4, r7}
		orrne	r3, r3, r4, lspush #16
		movne	r4, r4, lspull #16
		orrne	r4, r4, r7, lspush #16
		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
		tst	ip, #4
		movne	r3, r7, lspull #16
		ldrne	r7, [r1], #4
		orrne	r3, r3, r7, lspush #16
	TUSER(	strne) r3, [r0], #4			@ Shouldnt fault
		ands	ip, ip, #3
		beq	.Lc2u_2fupi
.Lc2u_2nowords:	mov	r3, r7, get_byte_2
		teq	ip, #0
		beq	.Lc2u_finished
		cmp	ip, #2
USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
		movge	r3, r7, get_byte_3
USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #0
USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
		b	.Lc2u_finished

.Lc2u_3fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lc2u_3nowords
		mov	r3, r7, lspull #24
		ldr	r7, [r1], #4
		orr	r3, r3, r7, lspush #8
USER(	TUSER(	str)	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lc2u_3fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lc2u_3rem8lp

.Lc2u_3cpy8lp:	mov	r3, r7, lspull #24
		ldmia	r1!, {r4 - r7}
		subs	ip, ip, #16
		orr	r3, r3, r4, lspush #8
		mov	r4, r4, lspull #24
		orr	r4, r4, r5, lspush #8
		mov	r5, r5, lspull #24
		orr	r5, r5, r6, lspush #8
		mov	r6, r6, lspull #24
		orr	r6, r6, r7, lspush #8
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		bpl	.Lc2u_3cpy8lp

.Lc2u_3rem8lp:	tst	ip, #8
		movne	r3, r7, lspull #24
		ldmneia	r1!, {r4, r7}
		orrne	r3, r3, r4, lspush #8
		movne	r4, r4, lspull #24
		orrne	r4, r4, r7, lspush #8
		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
		tst	ip, #4
		movne	r3, r7, lspull #24
		ldrne	r7, [r1], #4
		orrne	r3, r3, r7, lspush #8
	TUSER(	strne) r3, [r0], #4			@ Shouldnt fault
		ands	ip, ip, #3
		beq	.Lc2u_3fupi
.Lc2u_3nowords:	mov	r3, r7, get_byte_3
		teq	ip, #0
		beq	.Lc2u_finished
		cmp	ip, #2
USER(	TUSER(	strb)	r3, [r0], #1)			@ May fault
		ldrgeb	r3, [r1], #1
USER(	TUSER(	strgeb) r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #0
USER(	TUSER(	strgtb) r3, [r0], #1)			@ May fault
		b	.Lc2u_finished
ENDPROC(__copy_to_user)

		.pushsection .fixup,"ax"
		.align	0
9001:		ldmfd	sp!, {r0, r4 - r7, pc}
		.popsection

/* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
 * Purpose  : copy a block from user memory to kernel memory
 * Params   : to   - kernel memory
 *          : from - user memory
 *          : n    - number of bytes to copy
 * Returns  : Number of bytes NOT copied.
 */
.Lcfu_dest_not_aligned:
		rsb	ip, ip, #4
		cmp	ip, #2
USER(	TUSER(	ldrb)	r3, [r1], #1)			@ May fault
		strb	r3, [r0], #1
USER(	TUSER(	ldrgeb) r3, [r1], #1)			@ May fault
		strgeb	r3, [r0], #1
USER(	TUSER(	ldrgtb) r3, [r1], #1)			@ May fault
		strgtb	r3, [r0], #1
		sub	r2, r2, ip
		b	.Lcfu_dest_aligned

ENTRY(__copy_from_user)
		stmfd	sp!, {r0, r2, r4 - r7, lr}
		cmp	r2, #4
		blt	.Lcfu_not_enough
		ands	ip, r0, #3
		bne	.Lcfu_dest_not_aligned
.Lcfu_dest_aligned:
		ands	ip, r1, #3
		bne	.Lcfu_src_not_aligned

/*
 * Seeing as there has to be at least 8 bytes to copy, we can
 * copy one word, and force a user-mode page fault...
 */

.Lcfu_0fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lcfu_0nowords
USER(	TUSER(	ldr)	r3, [r1], #4)
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT	@ On each page, use a ld/st??t instruction
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lcfu_0fupi
/*
 * ip = max no. of bytes to copy before needing another "strt" insn
 */
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #32
		blt	.Lcfu_0rem8lp

.Lcfu_0cpy8lp:	ldmia	r1!, {r3 - r6}			@ Shouldnt fault
		stmia	r0!, {r3 - r6}
		ldmia	r1!, {r3 - r6}			@ Shouldnt fault
		subs	ip, ip, #32
		stmia	r0!, {r3 - r6}
		bpl	.Lcfu_0cpy8lp

.Lcfu_0rem8lp:	cmn	ip, #16
		ldmgeia	r1!, {r3 - r6}			@ Shouldnt fault
		stmgeia	r0!, {r3 - r6}
		tst	ip, #8
		ldmneia	r1!, {r3 - r4}			@ Shouldnt fault
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
	TUSER(	ldrne) r3, [r1], #4			@ Shouldnt fault
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.Lcfu_0fupi
.Lcfu_0nowords:	teq	ip, #0
		beq	.Lcfu_finished
.Lcfu_nowords:	cmp	ip, #2
USER(	TUSER(	ldrb)	r3, [r1], #1)			@ May fault
		strb	r3, [r0], #1
USER(	TUSER(	ldrgeb) r3, [r1], #1)			@ May fault
		strgeb	r3, [r0], #1
USER(	TUSER(	ldrgtb) r3, [r1], #1)			@ May fault
		strgtb	r3, [r0], #1
		b	.Lcfu_finished

.Lcfu_not_enough:
		movs	ip, r2
		bne	.Lcfu_nowords
.Lcfu_finished:	mov	r0, #0
		add	sp, sp, #8
		ldmfd	sp!, {r4 - r7, pc}

.Lcfu_src_not_aligned:
		bic	r1, r1, #3
USER(	TUSER(	ldr)	r7, [r1], #4)			@ May fault
		cmp	ip, #2
		bgt	.Lcfu_3fupi
		beq	.Lcfu_2fupi
.Lcfu_1fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lcfu_1nowords
		mov	r3, r7, lspull #8
USER(	TUSER(	ldr)	r7, [r1], #4)			@ May fault
		orr	r3, r3, r7, lspush #24
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lcfu_1fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lcfu_1rem8lp

.Lcfu_1cpy8lp:	mov	r3, r7, lspull #8
		ldmia	r1!, {r4 - r7}			@ Shouldnt fault
		subs	ip, ip, #16
		orr	r3, r3, r4, lspush #24
		mov	r4, r4, lspull #8
		orr	r4, r4, r5, lspush #24
		mov	r5, r5, lspull #8
		orr	r5, r5, r6, lspush #24
		mov	r6, r6, lspull #8
		orr	r6, r6, r7, lspush #24
		stmia	r0!, {r3 - r6}
		bpl	.Lcfu_1cpy8lp

.Lcfu_1rem8lp:	tst	ip, #8
		movne	r3, r7, lspull #8
		ldmneia	r1!, {r4, r7}			@ Shouldnt fault
		orrne	r3, r3, r4, lspush #24
		movne	r4, r4, lspull #8
		orrne	r4, r4, r7, lspush #24
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		movne	r3, r7, lspull #8
USER(	TUSER(	ldrne) r7, [r1], #4)			@ May fault
		orrne	r3, r3, r7, lspush #24
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.Lcfu_1fupi
.Lcfu_1nowords:	mov	r3, r7, get_byte_1
		teq	ip, #0
		beq	.Lcfu_finished
		cmp	ip, #2
		strb	r3, [r0], #1
		movge	r3, r7, get_byte_2
		strgeb	r3, [r0], #1
		movgt	r3, r7, get_byte_3
		strgtb	r3, [r0], #1
		b	.Lcfu_finished

.Lcfu_2fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lcfu_2nowords
		mov	r3, r7, lspull #16
USER(	TUSER(	ldr)	r7, [r1], #4)			@ May fault
		orr	r3, r3, r7, lspush #16
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lcfu_2fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lcfu_2rem8lp


.Lcfu_2cpy8lp:	mov	r3, r7, lspull #16
		ldmia	r1!, {r4 - r7}			@ Shouldnt fault
		subs	ip, ip, #16
		orr	r3, r3, r4, lspush #16
		mov	r4, r4, lspull #16
		orr	r4, r4, r5, lspush #16
		mov	r5, r5, lspull #16
		orr	r5, r5, r6, lspush #16
		mov	r6, r6, lspull #16
		orr	r6, r6, r7, lspush #16
		stmia	r0!, {r3 - r6}
		bpl	.Lcfu_2cpy8lp

.Lcfu_2rem8lp:	tst	ip, #8
		movne	r3, r7, lspull #16
		ldmneia	r1!, {r4, r7}			@ Shouldnt fault
		orrne	r3, r3, r4, lspush #16
		movne	r4, r4, lspull #16
		orrne	r4, r4, r7, lspush #16
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		movne	r3, r7, lspull #16
USER(	TUSER(	ldrne) r7, [r1], #4)			@ May fault
		orrne	r3, r3, r7, lspush #16
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.Lcfu_2fupi
.Lcfu_2nowords:	mov	r3, r7, get_byte_2
		teq	ip, #0
		beq	.Lcfu_finished
		cmp	ip, #2
		strb	r3, [r0], #1
		movge	r3, r7, get_byte_3
		strgeb	r3, [r0], #1
USER(	TUSER(	ldrgtb) r3, [r1], #0)			@ May fault
		strgtb	r3, [r0], #1
		b	.Lcfu_finished

.Lcfu_3fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lcfu_3nowords
		mov	r3, r7, lspull #24
USER(	TUSER(	ldr)	r7, [r1], #4)			@ May fault
		orr	r3, r3, r7, lspush #8
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lcfu_3fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lcfu_3rem8lp

.Lcfu_3cpy8lp:	mov	r3, r7, lspull #24
		ldmia	r1!, {r4 - r7}			@ Shouldnt fault
		orr	r3, r3, r4, lspush #8
		mov	r4, r4, lspull #24
		orr	r4, r4, r5, lspush #8
		mov	r5, r5, lspull #24
		orr	r5, r5, r6, lspush #8
		mov	r6, r6, lspull #24
		orr	r6, r6, r7, lspush #8
		stmia	r0!, {r3 - r6}
		subs	ip, ip, #16
		bpl	.Lcfu_3cpy8lp

.Lcfu_3rem8lp:	tst	ip, #8
		movne	r3, r7, lspull #24
		ldmneia	r1!, {r4, r7}			@ Shouldnt fault
		orrne	r3, r3, r4, lspush #8
		movne	r4, r4, lspull #24
		orrne	r4, r4, r7, lspush #8
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		movne	r3, r7, lspull #24
USER(	TUSER(	ldrne) r7, [r1], #4)			@ May fault
		orrne	r3, r3, r7, lspush #8
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.Lcfu_3fupi
.Lcfu_3nowords:	mov	r3, r7, get_byte_3
		teq	ip, #0
		beq	.Lcfu_finished
		cmp	ip, #2
		strb	r3, [r0], #1
USER(	TUSER(	ldrgeb) r3, [r1], #1)			@ May fault
		strgeb	r3, [r0], #1
USER(	TUSER(	ldrgtb) r3, [r1], #1)			@ May fault
		strgtb	r3, [r0], #1
		b	.Lcfu_finished
ENDPROC(__copy_from_user)

		.pushsection .fixup,"ax"
		.align	0
		/*
		 * We took an exception.  r0 contains a pointer to
		 * the byte not copied.
		 */
9001:		ldr	r2, [sp], #4			@ void *to
		sub	r2, r0, r2			@ bytes copied
		ldr	r1, [sp], #4			@ unsigned long count
		subs	r4, r1, r2			@ bytes left to copy
		movne	r1, r4
		blne	__memzero
		mov	r0, r4
		ldmfd	sp!, {r4 - r7, pc}
		.popsection
Commit	Line	Data
080fc66f RK	1	/*
	2	* linux/arch/arm/lib/uaccess.S
	3	*
	4	* Copyright (C) 1995, 1996,1997,1998 Russell King
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*
	10	* Routines to block copy data to/from user memory
	11	* These are highly optimised both for the 4k page size
	12	* and for various alignments.
	13	*/
	14	#include <linux/linkage.h>
	15	#include <asm/assembler.h>
	16	#include <asm/errno.h>
	17	#include <asm/domain.h>
	18
	19	.text
	20
	21	#define PAGE_SHIFT 12
	22
	23	/* Prototype: int __copy_to_user(void to, const char from, size_t n)
	24	* Purpose : copy a block to user memory from kernel memory
	25	* Params : to - user memory
	26	* : from - kernel memory
	27	* : n - number of bytes to copy
	28	* Returns : Number of bytes NOT copied.
	29	*/
	30
	31	.Lc2u_dest_not_aligned:
	32	rsb ip, ip, #4
	33	cmp ip, #2
	34	ldrb r3, [r1], #1
	35	USER( TUSER( strb) r3, [r0], #1) @ May fault
	36	ldrgeb r3, [r1], #1
	37	USER( TUSER( strgeb) r3, [r0], #1) @ May fault
	38	ldrgtb r3, [r1], #1
	39	USER( TUSER( strgtb) r3, [r0], #1) @ May fault
	40	sub r2, r2, ip
	41	b .Lc2u_dest_aligned
	42
	43	ENTRY(__copy_to_user)
	44	stmfd sp!, {r2, r4 - r7, lr}
	45	cmp r2, #4
	46	blt .Lc2u_not_enough
	47	ands ip, r0, #3
	48	bne .Lc2u_dest_not_aligned
	49	.Lc2u_dest_aligned:
	50
	51	ands ip, r1, #3
	52	bne .Lc2u_src_not_aligned
	53	/*
	54	* Seeing as there has to be at least 8 bytes to copy, we can
	55	* copy one word, and force a user-mode page fault...
	56	*/
	57
	58	.Lc2u_0fupi: subs r2, r2, #4
	59	addmi ip, r2, #4
	60	bmi .Lc2u_0nowords
	61	ldr r3, [r1], #4
	62	USER( TUSER( str) r3, [r0], #4) @ May fault
	63	mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
	64	rsb ip, ip, #0
65	movs ip, ip, lsr #32 - PAGE_SHIFT
66	beq .Lc2u_0fupi
67	/*
68	* ip = max no. of bytes to copy before needing another "strt" insn
69	*/
70	cmp r2, ip
71	movlt ip, r2
72	sub r2, r2, ip
73	subs ip, ip, #32
74	blt .Lc2u_0rem8lp
75
76	.Lc2u_0cpy8lp: ldmia r1!, {r3 - r6}
77	stmia r0!, {r3 - r6} @ Shouldnt fault
78	ldmia r1!, {r3 - r6}
79	subs ip, ip, #32
80	stmia r0!, {r3 - r6} @ Shouldnt fault
81	bpl .Lc2u_0cpy8lp
82
83	.Lc2u_0rem8lp: cmn ip, #16
84	ldmgeia r1!, {r3 - r6}
85	stmgeia r0!, {r3 - r6} @ Shouldnt fault
86	tst ip, #8
87	ldmneia r1!, {r3 - r4}
88	stmneia r0!, {r3 - r4} @ Shouldnt fault
89	tst ip, #4
90	ldrne r3, [r1], #4
91	TUSER( strne) r3, [r0], #4 @ Shouldnt fault
92	ands ip, ip, #3
93	beq .Lc2u_0fupi
94	.Lc2u_0nowords: teq ip, #0
95	beq .Lc2u_finished
96	.Lc2u_nowords: cmp ip, #2
97	ldrb r3, [r1], #1
98	USER( TUSER( strb) r3, [r0], #1) @ May fault
99	ldrgeb r3, [r1], #1
100	USER( TUSER( strgeb) r3, [r0], #1) @ May fault
101	ldrgtb r3, [r1], #1
102	USER( TUSER( strgtb) r3, [r0], #1) @ May fault
103	b .Lc2u_finished
104
105	.Lc2u_not_enough:
106	movs ip, r2
107	bne .Lc2u_nowords
108	.Lc2u_finished: mov r0, #0
109	ldmfd sp!, {r2, r4 - r7, pc}
110
111	.Lc2u_src_not_aligned:
112	bic r1, r1, #3
113	ldr r7, [r1], #4
114	cmp ip, #2
115	bgt .Lc2u_3fupi
116	beq .Lc2u_2fupi
117	.Lc2u_1fupi: subs r2, r2, #4
118	addmi ip, r2, #4
119	bmi .Lc2u_1nowords
d98b90ea	120	mov r3, r7, lspull #8
080fc66f	121	ldr r7, [r1], #4
d98b90ea	122	orr r3, r3, r7, lspush #24
080fc66f RK	123	USER( TUSER( str) r3, [r0], #4) @ May fault
	124	mov ip, r0, lsl #32 - PAGE_SHIFT
	125	rsb ip, ip, #0
	126	movs ip, ip, lsr #32 - PAGE_SHIFT
	127	beq .Lc2u_1fupi
	128	cmp r2, ip
	129	movlt ip, r2
	130	sub r2, r2, ip
	131	subs ip, ip, #16
	132	blt .Lc2u_1rem8lp
	133
d98b90ea	134	.Lc2u_1cpy8lp: mov r3, r7, lspull #8
080fc66f RK	135	ldmia r1!, {r4 - r7}
080fc66f RK	136	subs ip, ip, #16
d98b90ea VK	137	orr r3, r3, r4, lspush #24
	138	mov r4, r4, lspull #8
	139	orr r4, r4, r5, lspush #24
	140	mov r5, r5, lspull #8
	141	orr r5, r5, r6, lspush #24
	142	mov r6, r6, lspull #8
	143	orr r6, r6, r7, lspush #24
080fc66f RK	144	stmia r0!, {r3 - r6} @ Shouldnt fault
	145	bpl .Lc2u_1cpy8lp
	146
	147	.Lc2u_1rem8lp: tst ip, #8
d98b90ea	148	movne r3, r7, lspull #8
080fc66f	149	ldmneia r1!, {r4, r7}
d98b90ea VK	150	orrne r3, r3, r4, lspush #24
	151	movne r4, r4, lspull #8
	152	orrne r4, r4, r7, lspush #24
080fc66f RK	153	stmneia r0!, {r3 - r4} @ Shouldnt fault
080fc66f RK	154	tst ip, #4
d98b90ea	155	movne r3, r7, lspull #8
080fc66f	156	ldrne r7, [r1], #4
d98b90ea	157	orrne r3, r3, r7, lspush #24
080fc66f RK	158	TUSER( strne) r3, [r0], #4 @ Shouldnt fault
	159	ands ip, ip, #3
	160	beq .Lc2u_1fupi
	161	.Lc2u_1nowords: mov r3, r7, get_byte_1
	162	teq ip, #0
	163	beq .Lc2u_finished
	164	cmp ip, #2
	165	USER( TUSER( strb) r3, [r0], #1) @ May fault
	166	movge r3, r7, get_byte_2
	167	USER( TUSER( strgeb) r3, [r0], #1) @ May fault
	168	movgt r3, r7, get_byte_3
	169	USER( TUSER( strgtb) r3, [r0], #1) @ May fault
	170	b .Lc2u_finished
	171
	172	.Lc2u_2fupi: subs r2, r2, #4
	173	addmi ip, r2, #4
	174	bmi .Lc2u_2nowords
d98b90ea	175	mov r3, r7, lspull #16
080fc66f	176	ldr r7, [r1], #4
d98b90ea	177	orr r3, r3, r7, lspush #16
080fc66f RK	178	USER( TUSER( str) r3, [r0], #4) @ May fault
	179	mov ip, r0, lsl #32 - PAGE_SHIFT
	180	rsb ip, ip, #0
	181	movs ip, ip, lsr #32 - PAGE_SHIFT
	182	beq .Lc2u_2fupi
	183	cmp r2, ip
	184	movlt ip, r2
	185	sub r2, r2, ip
	186	subs ip, ip, #16
	187	blt .Lc2u_2rem8lp
	188
d98b90ea	189	.Lc2u_2cpy8lp: mov r3, r7, lspull #16
080fc66f RK	190	ldmia r1!, {r4 - r7}
080fc66f RK	191	subs ip, ip, #16
d98b90ea VK	192	orr r3, r3, r4, lspush #16
	193	mov r4, r4, lspull #16
	194	orr r4, r4, r5, lspush #16
	195	mov r5, r5, lspull #16
	196	orr r5, r5, r6, lspush #16
	197	mov r6, r6, lspull #16
	198	orr r6, r6, r7, lspush #16
080fc66f RK	199	stmia r0!, {r3 - r6} @ Shouldnt fault
	200	bpl .Lc2u_2cpy8lp
	201
	202	.Lc2u_2rem8lp: tst ip, #8
d98b90ea	203	movne r3, r7, lspull #16
080fc66f	204	ldmneia r1!, {r4, r7}
d98b90ea VK	205	orrne r3, r3, r4, lspush #16
	206	movne r4, r4, lspull #16
	207	orrne r4, r4, r7, lspush #16
080fc66f RK	208	stmneia r0!, {r3 - r4} @ Shouldnt fault
080fc66f RK	209	tst ip, #4
d98b90ea	210	movne r3, r7, lspull #16
080fc66f	211	ldrne r7, [r1], #4
d98b90ea	212	orrne r3, r3, r7, lspush #16
080fc66f RK	213	TUSER( strne) r3, [r0], #4 @ Shouldnt fault
	214	ands ip, ip, #3
	215	beq .Lc2u_2fupi
	216	.Lc2u_2nowords: mov r3, r7, get_byte_2
	217	teq ip, #0
	218	beq .Lc2u_finished
	219	cmp ip, #2
	220	USER( TUSER( strb) r3, [r0], #1) @ May fault
	221	movge r3, r7, get_byte_3
	222	USER( TUSER( strgeb) r3, [r0], #1) @ May fault
	223	ldrgtb r3, [r1], #0
	224	USER( TUSER( strgtb) r3, [r0], #1) @ May fault
	225	b .Lc2u_finished
	226
	227	.Lc2u_3fupi: subs r2, r2, #4
	228	addmi ip, r2, #4
	229	bmi .Lc2u_3nowords
d98b90ea	230	mov r3, r7, lspull #24
080fc66f	231	ldr r7, [r1], #4
d98b90ea	232	orr r3, r3, r7, lspush #8
080fc66f RK	233	USER( TUSER( str) r3, [r0], #4) @ May fault
	234	mov ip, r0, lsl #32 - PAGE_SHIFT
	235	rsb ip, ip, #0
	236	movs ip, ip, lsr #32 - PAGE_SHIFT
	237	beq .Lc2u_3fupi
	238	cmp r2, ip
	239	movlt ip, r2
	240	sub r2, r2, ip
	241	subs ip, ip, #16
	242	blt .Lc2u_3rem8lp
	243
d98b90ea	244	.Lc2u_3cpy8lp: mov r3, r7, lspull #24
080fc66f RK	245	ldmia r1!, {r4 - r7}
080fc66f RK	246	subs ip, ip, #16
d98b90ea VK	247	orr r3, r3, r4, lspush #8
	248	mov r4, r4, lspull #24
	249	orr r4, r4, r5, lspush #8
	250	mov r5, r5, lspull #24
	251	orr r5, r5, r6, lspush #8
	252	mov r6, r6, lspull #24
	253	orr r6, r6, r7, lspush #8
080fc66f RK	254	stmia r0!, {r3 - r6} @ Shouldnt fault
	255	bpl .Lc2u_3cpy8lp
	256
	257	.Lc2u_3rem8lp: tst ip, #8
d98b90ea	258	movne r3, r7, lspull #24
080fc66f	259	ldmneia r1!, {r4, r7}
d98b90ea VK	260	orrne r3, r3, r4, lspush #8
	261	movne r4, r4, lspull #24
	262	orrne r4, r4, r7, lspush #8
080fc66f RK	263	stmneia r0!, {r3 - r4} @ Shouldnt fault
080fc66f RK	264	tst ip, #4
d98b90ea	265	movne r3, r7, lspull #24
080fc66f	266	ldrne r7, [r1], #4
d98b90ea	267	orrne r3, r3, r7, lspush #8
080fc66f RK	268	TUSER( strne) r3, [r0], #4 @ Shouldnt fault
	269	ands ip, ip, #3
	270	beq .Lc2u_3fupi
	271	.Lc2u_3nowords: mov r3, r7, get_byte_3
	272	teq ip, #0
	273	beq .Lc2u_finished
	274	cmp ip, #2
	275	USER( TUSER( strb) r3, [r0], #1) @ May fault
	276	ldrgeb r3, [r1], #1
	277	USER( TUSER( strgeb) r3, [r0], #1) @ May fault
	278	ldrgtb r3, [r1], #0
	279	USER( TUSER( strgtb) r3, [r0], #1) @ May fault
	280	b .Lc2u_finished
	281	ENDPROC(__copy_to_user)
	282
	283	.pushsection .fixup,"ax"
	284	.align 0
	285	9001: ldmfd sp!, {r0, r4 - r7, pc}
	286	.popsection
	287
	288	/* Prototype: unsigned long __copy_from_user(void to,const void from,unsigned long n);
	289	* Purpose : copy a block from user memory to kernel memory
	290	* Params : to - kernel memory
	291	* : from - user memory
	292	* : n - number of bytes to copy
	293	* Returns : Number of bytes NOT copied.
	294	*/
	295	.Lcfu_dest_not_aligned:
	296	rsb ip, ip, #4
	297	cmp ip, #2
	298	USER( TUSER( ldrb) r3, [r1], #1) @ May fault
	299	strb r3, [r0], #1
	300	USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
	301	strgeb r3, [r0], #1
	302	USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
	303	strgtb r3, [r0], #1
	304	sub r2, r2, ip
	305	b .Lcfu_dest_aligned
	306
	307	ENTRY(__copy_from_user)
	308	stmfd sp!, {r0, r2, r4 - r7, lr}
	309	cmp r2, #4
	310	blt .Lcfu_not_enough
	311	ands ip, r0, #3
	312	bne .Lcfu_dest_not_aligned
	313	.Lcfu_dest_aligned:
	314	ands ip, r1, #3
	315	bne .Lcfu_src_not_aligned
	316
	317	/*
	318	* Seeing as there has to be at least 8 bytes to copy, we can
	319	* copy one word, and force a user-mode page fault...
	320	*/
	321
	322	.Lcfu_0fupi: subs r2, r2, #4
	323	addmi ip, r2, #4
	324	bmi .Lcfu_0nowords
	325	USER( TUSER( ldr) r3, [r1], #4)
	326	str r3, [r0], #4
	327	mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
	328	rsb ip, ip, #0
	329	movs ip, ip, lsr #32 - PAGE_SHIFT
	330	beq .Lcfu_0fupi
	331	/*
332	* ip = max no. of bytes to copy before needing another "strt" insn
333	*/
334	cmp r2, ip
335	movlt ip, r2
336	sub r2, r2, ip
337	subs ip, ip, #32
338	blt .Lcfu_0rem8lp
339
340	.Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault
341	stmia r0!, {r3 - r6}
342	ldmia r1!, {r3 - r6} @ Shouldnt fault
343	subs ip, ip, #32
344	stmia r0!, {r3 - r6}
345	bpl .Lcfu_0cpy8lp
346
347	.Lcfu_0rem8lp: cmn ip, #16
348	ldmgeia r1!, {r3 - r6} @ Shouldnt fault
349	stmgeia r0!, {r3 - r6}
350	tst ip, #8
351	ldmneia r1!, {r3 - r4} @ Shouldnt fault
352	stmneia r0!, {r3 - r4}
353	tst ip, #4
354	TUSER( ldrne) r3, [r1], #4 @ Shouldnt fault
355	strne r3, [r0], #4
356	ands ip, ip, #3
357	beq .Lcfu_0fupi
358	.Lcfu_0nowords: teq ip, #0
359	beq .Lcfu_finished
360	.Lcfu_nowords: cmp ip, #2
361	USER( TUSER( ldrb) r3, [r1], #1) @ May fault
362	strb r3, [r0], #1
363	USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
364	strgeb r3, [r0], #1
365	USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
366	strgtb r3, [r0], #1
367	b .Lcfu_finished
368
369	.Lcfu_not_enough:
370	movs ip, r2
371	bne .Lcfu_nowords
372	.Lcfu_finished: mov r0, #0
373	add sp, sp, #8
374	ldmfd sp!, {r4 - r7, pc}
375
376	.Lcfu_src_not_aligned:
377	bic r1, r1, #3
378	USER( TUSER( ldr) r7, [r1], #4) @ May fault
379	cmp ip, #2
380	bgt .Lcfu_3fupi
381	beq .Lcfu_2fupi
382	.Lcfu_1fupi: subs r2, r2, #4
383	addmi ip, r2, #4
384	bmi .Lcfu_1nowords
d98b90ea	385	mov r3, r7, lspull #8
080fc66f	386	USER( TUSER( ldr) r7, [r1], #4) @ May fault
d98b90ea	387	orr r3, r3, r7, lspush #24
080fc66f RK	388	str r3, [r0], #4
	389	mov ip, r1, lsl #32 - PAGE_SHIFT
	390	rsb ip, ip, #0
	391	movs ip, ip, lsr #32 - PAGE_SHIFT
	392	beq .Lcfu_1fupi
	393	cmp r2, ip
	394	movlt ip, r2
	395	sub r2, r2, ip
	396	subs ip, ip, #16
	397	blt .Lcfu_1rem8lp
	398
d98b90ea	399	.Lcfu_1cpy8lp: mov r3, r7, lspull #8
080fc66f RK	400	ldmia r1!, {r4 - r7} @ Shouldnt fault
080fc66f RK	401	subs ip, ip, #16
d98b90ea VK	402	orr r3, r3, r4, lspush #24
	403	mov r4, r4, lspull #8
	404	orr r4, r4, r5, lspush #24
	405	mov r5, r5, lspull #8
	406	orr r5, r5, r6, lspush #24
	407	mov r6, r6, lspull #8
	408	orr r6, r6, r7, lspush #24
080fc66f RK	409	stmia r0!, {r3 - r6}
	410	bpl .Lcfu_1cpy8lp
	411
	412	.Lcfu_1rem8lp: tst ip, #8
d98b90ea	413	movne r3, r7, lspull #8
080fc66f	414	ldmneia r1!, {r4, r7} @ Shouldnt fault
d98b90ea VK	415	orrne r3, r3, r4, lspush #24
	416	movne r4, r4, lspull #8
	417	orrne r4, r4, r7, lspush #24
080fc66f RK	418	stmneia r0!, {r3 - r4}
080fc66f RK	419	tst ip, #4
d98b90ea	420	movne r3, r7, lspull #8
080fc66f	421	USER( TUSER( ldrne) r7, [r1], #4) @ May fault
d98b90ea	422	orrne r3, r3, r7, lspush #24
080fc66f RK	423	strne r3, [r0], #4
	424	ands ip, ip, #3
	425	beq .Lcfu_1fupi
	426	.Lcfu_1nowords: mov r3, r7, get_byte_1
	427	teq ip, #0
	428	beq .Lcfu_finished
	429	cmp ip, #2
	430	strb r3, [r0], #1
	431	movge r3, r7, get_byte_2
	432	strgeb r3, [r0], #1
	433	movgt r3, r7, get_byte_3
	434	strgtb r3, [r0], #1
	435	b .Lcfu_finished
	436
	437	.Lcfu_2fupi: subs r2, r2, #4
	438	addmi ip, r2, #4
	439	bmi .Lcfu_2nowords
d98b90ea	440	mov r3, r7, lspull #16
080fc66f	441	USER( TUSER( ldr) r7, [r1], #4) @ May fault
d98b90ea	442	orr r3, r3, r7, lspush #16
080fc66f RK	443	str r3, [r0], #4
	444	mov ip, r1, lsl #32 - PAGE_SHIFT
	445	rsb ip, ip, #0
	446	movs ip, ip, lsr #32 - PAGE_SHIFT
	447	beq .Lcfu_2fupi
	448	cmp r2, ip
	449	movlt ip, r2
	450	sub r2, r2, ip
	451	subs ip, ip, #16
	452	blt .Lcfu_2rem8lp
	453
	454
d98b90ea	455	.Lcfu_2cpy8lp: mov r3, r7, lspull #16
080fc66f RK	456	ldmia r1!, {r4 - r7} @ Shouldnt fault
080fc66f RK	457	subs ip, ip, #16
d98b90ea VK	458	orr r3, r3, r4, lspush #16
	459	mov r4, r4, lspull #16
	460	orr r4, r4, r5, lspush #16
	461	mov r5, r5, lspull #16
	462	orr r5, r5, r6, lspush #16
	463	mov r6, r6, lspull #16
	464	orr r6, r6, r7, lspush #16
080fc66f RK	465	stmia r0!, {r3 - r6}
	466	bpl .Lcfu_2cpy8lp
	467
	468	.Lcfu_2rem8lp: tst ip, #8
d98b90ea	469	movne r3, r7, lspull #16
080fc66f	470	ldmneia r1!, {r4, r7} @ Shouldnt fault
d98b90ea VK	471	orrne r3, r3, r4, lspush #16
	472	movne r4, r4, lspull #16
	473	orrne r4, r4, r7, lspush #16
080fc66f RK	474	stmneia r0!, {r3 - r4}
080fc66f RK	475	tst ip, #4
d98b90ea	476	movne r3, r7, lspull #16
080fc66f	477	USER( TUSER( ldrne) r7, [r1], #4) @ May fault
d98b90ea	478	orrne r3, r3, r7, lspush #16
080fc66f RK	479	strne r3, [r0], #4
	480	ands ip, ip, #3
	481	beq .Lcfu_2fupi
	482	.Lcfu_2nowords: mov r3, r7, get_byte_2
	483	teq ip, #0
	484	beq .Lcfu_finished
	485	cmp ip, #2
	486	strb r3, [r0], #1
	487	movge r3, r7, get_byte_3
	488	strgeb r3, [r0], #1
	489	USER( TUSER( ldrgtb) r3, [r1], #0) @ May fault
	490	strgtb r3, [r0], #1
	491	b .Lcfu_finished
	492
	493	.Lcfu_3fupi: subs r2, r2, #4
	494	addmi ip, r2, #4
	495	bmi .Lcfu_3nowords
d98b90ea	496	mov r3, r7, lspull #24
080fc66f	497	USER( TUSER( ldr) r7, [r1], #4) @ May fault
d98b90ea	498	orr r3, r3, r7, lspush #8
080fc66f RK	499	str r3, [r0], #4
	500	mov ip, r1, lsl #32 - PAGE_SHIFT
	501	rsb ip, ip, #0
	502	movs ip, ip, lsr #32 - PAGE_SHIFT
	503	beq .Lcfu_3fupi
	504	cmp r2, ip
	505	movlt ip, r2
	506	sub r2, r2, ip
	507	subs ip, ip, #16
	508	blt .Lcfu_3rem8lp
	509
d98b90ea	510	.Lcfu_3cpy8lp: mov r3, r7, lspull #24
080fc66f	511	ldmia r1!, {r4 - r7} @ Shouldnt fault
d98b90ea VK	512	orr r3, r3, r4, lspush #8
	513	mov r4, r4, lspull #24
	514	orr r4, r4, r5, lspush #8
	515	mov r5, r5, lspull #24
	516	orr r5, r5, r6, lspush #8
	517	mov r6, r6, lspull #24
	518	orr r6, r6, r7, lspush #8
080fc66f RK	519	stmia r0!, {r3 - r6}
	520	subs ip, ip, #16
	521	bpl .Lcfu_3cpy8lp
	522
	523	.Lcfu_3rem8lp: tst ip, #8
d98b90ea	524	movne r3, r7, lspull #24
080fc66f	525	ldmneia r1!, {r4, r7} @ Shouldnt fault
d98b90ea VK	526	orrne r3, r3, r4, lspush #8
	527	movne r4, r4, lspull #24
	528	orrne r4, r4, r7, lspush #8
080fc66f RK	529	stmneia r0!, {r3 - r4}
080fc66f RK	530	tst ip, #4
d98b90ea	531	movne r3, r7, lspull #24
080fc66f	532	USER( TUSER( ldrne) r7, [r1], #4) @ May fault
d98b90ea	533	orrne r3, r3, r7, lspush #8
080fc66f RK	534	strne r3, [r0], #4
	535	ands ip, ip, #3
	536	beq .Lcfu_3fupi
	537	.Lcfu_3nowords: mov r3, r7, get_byte_3
	538	teq ip, #0
	539	beq .Lcfu_finished
	540	cmp ip, #2
	541	strb r3, [r0], #1
	542	USER( TUSER( ldrgeb) r3, [r1], #1) @ May fault
	543	strgeb r3, [r0], #1
	544	USER( TUSER( ldrgtb) r3, [r1], #1) @ May fault
	545	strgtb r3, [r0], #1
	546	b .Lcfu_finished
	547	ENDPROC(__copy_from_user)
	548
	549	.pushsection .fixup,"ax"
	550	.align 0
	551	/*
	552	* We took an exception. r0 contains a pointer to
	553	* the byte not copied.
	554	*/
	555	9001: ldr r2, [sp], #4 @ void *to
	556	sub r2, r0, r2 @ bytes copied
	557	ldr r1, [sp], #4 @ unsigned long count
	558	subs r4, r1, r2 @ bytes left to copy
	559	movne r1, r4
	560	blne __memzero
	561	mov r0, r4
	562	ldmfd sp!, {r4 - r7, pc}
	563	.popsection
	564