[deliverable/linux.git] / arch / arm / lib / uaccess.S

/*
 *  linux/arch/arm/lib/uaccess.S
 *
 *  Copyright (C) 1995, 1996,1997,1998 Russell King
 *
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License version 2 as
 * published by the Free Software Foundation.
 *
 *  Routines to block copy data to/from user memory
 *   These are highly optimised both for the 4k page size
 *   and for various alignments.
 */
#include <linux/linkage.h>
#include <asm/assembler.h>
#include <asm/errno.h>

		.text

#define PAGE_SHIFT 12

/* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
 * Purpose  : copy a block to user memory from kernel memory
 * Params   : to   - user memory
 *          : from - kernel memory
 *          : n    - number of bytes to copy
 * Returns  : Number of bytes NOT copied.
 */

.Lc2u_dest_not_aligned:
		rsb	ip, ip, #4
		cmp	ip, #2
		ldrb	r3, [r1], #1
USER(		strbt	r3, [r0], #1)			@ May fault
		ldrgeb	r3, [r1], #1
USER(		strgebt	r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #1
USER(		strgtbt	r3, [r0], #1)			@ May fault
		sub	r2, r2, ip
		b	.Lc2u_dest_aligned

ENTRY(__copy_to_user)
		stmfd	sp!, {r2, r4 - r7, lr}
		cmp	r2, #4
		blt	.Lc2u_not_enough
		ands	ip, r0, #3
		bne	.Lc2u_dest_not_aligned
.Lc2u_dest_aligned:

		ands	ip, r1, #3
		bne	.Lc2u_src_not_aligned
/*
 * Seeing as there has to be at least 8 bytes to copy, we can
 * copy one word, and force a user-mode page fault...
 */

.Lc2u_0fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lc2u_0nowords
		ldr	r3, [r1], #4
USER(		strt	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT	@ On each page, use a ld/st??t instruction
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lc2u_0fupi
/*
 * ip = max no. of bytes to copy before needing another "strt" insn
 */
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #32
		blt	.Lc2u_0rem8lp

.Lc2u_0cpy8lp:	ldmia	r1!, {r3 - r6}
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		ldmia	r1!, {r3 - r6}
		subs	ip, ip, #32
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		bpl	.Lc2u_0cpy8lp

.Lc2u_0rem8lp:	cmn	ip, #16
		ldmgeia	r1!, {r3 - r6}
		stmgeia	r0!, {r3 - r6}			@ Shouldnt fault
		tst	ip, #8
		ldmneia	r1!, {r3 - r4}
		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
		tst	ip, #4
		ldrne	r3, [r1], #4
		strnet	r3, [r0], #4			@ Shouldnt fault
		ands	ip, ip, #3
		beq	.Lc2u_0fupi
.Lc2u_0nowords:	teq	ip, #0
		beq	.Lc2u_finished
.Lc2u_nowords:	cmp	ip, #2
		ldrb	r3, [r1], #1
USER(		strbt	r3, [r0], #1)			@ May fault
		ldrgeb	r3, [r1], #1
USER(		strgebt	r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #1
USER(		strgtbt	r3, [r0], #1)			@ May fault
		b	.Lc2u_finished

.Lc2u_not_enough:
		movs	ip, r2
		bne	.Lc2u_nowords
.Lc2u_finished:	mov	r0, #0
		ldmfd	sp!, {r2, r4 - r7, pc}

.Lc2u_src_not_aligned:
		bic	r1, r1, #3
		ldr	r7, [r1], #4
		cmp	ip, #2
		bgt	.Lc2u_3fupi
		beq	.Lc2u_2fupi
.Lc2u_1fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lc2u_1nowords
		mov	r3, r7, pull #8
		ldr	r7, [r1], #4
		orr	r3, r3, r7, push #24
USER(		strt	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lc2u_1fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lc2u_1rem8lp

.Lc2u_1cpy8lp:	mov	r3, r7, pull #8
		ldmia	r1!, {r4 - r7}
		subs	ip, ip, #16
		orr	r3, r3, r4, push #24
		mov	r4, r4, pull #8
		orr	r4, r4, r5, push #24
		mov	r5, r5, pull #8
		orr	r5, r5, r6, push #24
		mov	r6, r6, pull #8
		orr	r6, r6, r7, push #24
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		bpl	.Lc2u_1cpy8lp

.Lc2u_1rem8lp:	tst	ip, #8
		movne	r3, r7, pull #8
		ldmneia	r1!, {r4, r7}
		orrne	r3, r3, r4, push #24
		movne	r4, r4, pull #8
		orrne	r4, r4, r7, push #24
		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
		tst	ip, #4
		movne	r3, r7, pull #8
		ldrne	r7, [r1], #4
		orrne	r3, r3, r7, push #24
		strnet	r3, [r0], #4			@ Shouldnt fault
		ands	ip, ip, #3
		beq	.Lc2u_1fupi
.Lc2u_1nowords:	mov	r3, r7, get_byte_1
		teq	ip, #0
		beq	.Lc2u_finished
		cmp	ip, #2
USER(		strbt	r3, [r0], #1)			@ May fault
		movge	r3, r7, get_byte_2
USER(		strgebt	r3, [r0], #1)			@ May fault
		movgt	r3, r7, get_byte_3
USER(		strgtbt	r3, [r0], #1)			@ May fault
		b	.Lc2u_finished

.Lc2u_2fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lc2u_2nowords
		mov	r3, r7, pull #16
		ldr	r7, [r1], #4
		orr	r3, r3, r7, push #16
USER(		strt	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lc2u_2fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lc2u_2rem8lp

.Lc2u_2cpy8lp:	mov	r3, r7, pull #16
		ldmia	r1!, {r4 - r7}
		subs	ip, ip, #16
		orr	r3, r3, r4, push #16
		mov	r4, r4, pull #16
		orr	r4, r4, r5, push #16
		mov	r5, r5, pull #16
		orr	r5, r5, r6, push #16
		mov	r6, r6, pull #16
		orr	r6, r6, r7, push #16
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		bpl	.Lc2u_2cpy8lp

.Lc2u_2rem8lp:	tst	ip, #8
		movne	r3, r7, pull #16
		ldmneia	r1!, {r4, r7}
		orrne	r3, r3, r4, push #16
		movne	r4, r4, pull #16
		orrne	r4, r4, r7, push #16
		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
		tst	ip, #4
		movne	r3, r7, pull #16
		ldrne	r7, [r1], #4
		orrne	r3, r3, r7, push #16
		strnet	r3, [r0], #4			@ Shouldnt fault
		ands	ip, ip, #3
		beq	.Lc2u_2fupi
.Lc2u_2nowords:	mov	r3, r7, get_byte_2
		teq	ip, #0
		beq	.Lc2u_finished
		cmp	ip, #2
USER(		strbt	r3, [r0], #1)			@ May fault
		movge	r3, r7, get_byte_3
USER(		strgebt	r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #0
USER(		strgtbt	r3, [r0], #1)			@ May fault
		b	.Lc2u_finished

.Lc2u_3fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lc2u_3nowords
		mov	r3, r7, pull #24
		ldr	r7, [r1], #4
		orr	r3, r3, r7, push #8
USER(		strt	r3, [r0], #4)			@ May fault
		mov	ip, r0, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lc2u_3fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lc2u_3rem8lp

.Lc2u_3cpy8lp:	mov	r3, r7, pull #24
		ldmia	r1!, {r4 - r7}
		subs	ip, ip, #16
		orr	r3, r3, r4, push #8
		mov	r4, r4, pull #24
		orr	r4, r4, r5, push #8
		mov	r5, r5, pull #24
		orr	r5, r5, r6, push #8
		mov	r6, r6, pull #24
		orr	r6, r6, r7, push #8
		stmia	r0!, {r3 - r6}			@ Shouldnt fault
		bpl	.Lc2u_3cpy8lp

.Lc2u_3rem8lp:	tst	ip, #8
		movne	r3, r7, pull #24
		ldmneia	r1!, {r4, r7}
		orrne	r3, r3, r4, push #8
		movne	r4, r4, pull #24
		orrne	r4, r4, r7, push #8
		stmneia	r0!, {r3 - r4}			@ Shouldnt fault
		tst	ip, #4
		movne	r3, r7, pull #24
		ldrne	r7, [r1], #4
		orrne	r3, r3, r7, push #8
		strnet	r3, [r0], #4			@ Shouldnt fault
		ands	ip, ip, #3
		beq	.Lc2u_3fupi
.Lc2u_3nowords:	mov	r3, r7, get_byte_3
		teq	ip, #0
		beq	.Lc2u_finished
		cmp	ip, #2
USER(		strbt	r3, [r0], #1)			@ May fault
		ldrgeb	r3, [r1], #1
USER(		strgebt	r3, [r0], #1)			@ May fault
		ldrgtb	r3, [r1], #0
USER(		strgtbt	r3, [r0], #1)			@ May fault
		b	.Lc2u_finished
ENDPROC(__copy_to_user)

		.pushsection .fixup,"ax"
		.align	0
9001:		ldmfd	sp!, {r0, r4 - r7, pc}
		.popsection

/* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
 * Purpose  : copy a block from user memory to kernel memory
 * Params   : to   - kernel memory
 *          : from - user memory
 *          : n    - number of bytes to copy
 * Returns  : Number of bytes NOT copied.
 */
.Lcfu_dest_not_aligned:
		rsb	ip, ip, #4
		cmp	ip, #2
USER(		ldrbt	r3, [r1], #1)			@ May fault
		strb	r3, [r0], #1
USER(		ldrgebt	r3, [r1], #1)			@ May fault
		strgeb	r3, [r0], #1
USER(		ldrgtbt	r3, [r1], #1)			@ May fault
		strgtb	r3, [r0], #1
		sub	r2, r2, ip
		b	.Lcfu_dest_aligned

ENTRY(__copy_from_user)
		stmfd	sp!, {r0, r2, r4 - r7, lr}
		cmp	r2, #4
		blt	.Lcfu_not_enough
		ands	ip, r0, #3
		bne	.Lcfu_dest_not_aligned
.Lcfu_dest_aligned:
		ands	ip, r1, #3
		bne	.Lcfu_src_not_aligned

/*
 * Seeing as there has to be at least 8 bytes to copy, we can
 * copy one word, and force a user-mode page fault...
 */

.Lcfu_0fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lcfu_0nowords
USER(		ldrt	r3, [r1], #4)
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT	@ On each page, use a ld/st??t instruction
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lcfu_0fupi
/*
 * ip = max no. of bytes to copy before needing another "strt" insn
 */
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #32
		blt	.Lcfu_0rem8lp

.Lcfu_0cpy8lp:	ldmia	r1!, {r3 - r6}			@ Shouldnt fault
		stmia	r0!, {r3 - r6}
		ldmia	r1!, {r3 - r6}			@ Shouldnt fault
		subs	ip, ip, #32
		stmia	r0!, {r3 - r6}
		bpl	.Lcfu_0cpy8lp

.Lcfu_0rem8lp:	cmn	ip, #16
		ldmgeia	r1!, {r3 - r6}			@ Shouldnt fault
		stmgeia	r0!, {r3 - r6}
		tst	ip, #8
		ldmneia	r1!, {r3 - r4}			@ Shouldnt fault
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		ldrnet	r3, [r1], #4			@ Shouldnt fault
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.Lcfu_0fupi
.Lcfu_0nowords:	teq	ip, #0
		beq	.Lcfu_finished
.Lcfu_nowords:	cmp	ip, #2
USER(		ldrbt	r3, [r1], #1)			@ May fault
		strb	r3, [r0], #1
USER(		ldrgebt	r3, [r1], #1)			@ May fault
		strgeb	r3, [r0], #1
USER(		ldrgtbt	r3, [r1], #1)			@ May fault
		strgtb	r3, [r0], #1
		b	.Lcfu_finished

.Lcfu_not_enough:
		movs	ip, r2
		bne	.Lcfu_nowords
.Lcfu_finished:	mov	r0, #0
		add	sp, sp, #8
		ldmfd	sp!, {r4 - r7, pc}

.Lcfu_src_not_aligned:
		bic	r1, r1, #3
USER(		ldrt	r7, [r1], #4)			@ May fault
		cmp	ip, #2
		bgt	.Lcfu_3fupi
		beq	.Lcfu_2fupi
.Lcfu_1fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lcfu_1nowords
		mov	r3, r7, pull #8
USER(		ldrt	r7, [r1], #4)			@ May fault
		orr	r3, r3, r7, push #24
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lcfu_1fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lcfu_1rem8lp

.Lcfu_1cpy8lp:	mov	r3, r7, pull #8
		ldmia	r1!, {r4 - r7}			@ Shouldnt fault
		subs	ip, ip, #16
		orr	r3, r3, r4, push #24
		mov	r4, r4, pull #8
		orr	r4, r4, r5, push #24
		mov	r5, r5, pull #8
		orr	r5, r5, r6, push #24
		mov	r6, r6, pull #8
		orr	r6, r6, r7, push #24
		stmia	r0!, {r3 - r6}
		bpl	.Lcfu_1cpy8lp

.Lcfu_1rem8lp:	tst	ip, #8
		movne	r3, r7, pull #8
		ldmneia	r1!, {r4, r7}			@ Shouldnt fault
		orrne	r3, r3, r4, push #24
		movne	r4, r4, pull #8
		orrne	r4, r4, r7, push #24
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		movne	r3, r7, pull #8
USER(		ldrnet	r7, [r1], #4)			@ May fault
		orrne	r3, r3, r7, push #24
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.Lcfu_1fupi
.Lcfu_1nowords:	mov	r3, r7, get_byte_1
		teq	ip, #0
		beq	.Lcfu_finished
		cmp	ip, #2
		strb	r3, [r0], #1
		movge	r3, r7, get_byte_2
		strgeb	r3, [r0], #1
		movgt	r3, r7, get_byte_3
		strgtb	r3, [r0], #1
		b	.Lcfu_finished

.Lcfu_2fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lcfu_2nowords
		mov	r3, r7, pull #16
USER(		ldrt	r7, [r1], #4)			@ May fault
		orr	r3, r3, r7, push #16
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lcfu_2fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lcfu_2rem8lp


.Lcfu_2cpy8lp:	mov	r3, r7, pull #16
		ldmia	r1!, {r4 - r7}			@ Shouldnt fault
		subs	ip, ip, #16
		orr	r3, r3, r4, push #16
		mov	r4, r4, pull #16
		orr	r4, r4, r5, push #16
		mov	r5, r5, pull #16
		orr	r5, r5, r6, push #16
		mov	r6, r6, pull #16
		orr	r6, r6, r7, push #16
		stmia	r0!, {r3 - r6}
		bpl	.Lcfu_2cpy8lp

.Lcfu_2rem8lp:	tst	ip, #8
		movne	r3, r7, pull #16
		ldmneia	r1!, {r4, r7}			@ Shouldnt fault
		orrne	r3, r3, r4, push #16
		movne	r4, r4, pull #16
		orrne	r4, r4, r7, push #16
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		movne	r3, r7, pull #16
USER(		ldrnet	r7, [r1], #4)			@ May fault
		orrne	r3, r3, r7, push #16
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.Lcfu_2fupi
.Lcfu_2nowords:	mov	r3, r7, get_byte_2
		teq	ip, #0
		beq	.Lcfu_finished
		cmp	ip, #2
		strb	r3, [r0], #1
		movge	r3, r7, get_byte_3
		strgeb	r3, [r0], #1
USER(		ldrgtbt	r3, [r1], #0)			@ May fault
		strgtb	r3, [r0], #1
		b	.Lcfu_finished

.Lcfu_3fupi:	subs	r2, r2, #4
		addmi	ip, r2, #4
		bmi	.Lcfu_3nowords
		mov	r3, r7, pull #24
USER(		ldrt	r7, [r1], #4)			@ May fault
		orr	r3, r3, r7, push #8
		str	r3, [r0], #4
		mov	ip, r1, lsl #32 - PAGE_SHIFT
		rsb	ip, ip, #0
		movs	ip, ip, lsr #32 - PAGE_SHIFT
		beq	.Lcfu_3fupi
		cmp	r2, ip
		movlt	ip, r2
		sub	r2, r2, ip
		subs	ip, ip, #16
		blt	.Lcfu_3rem8lp

.Lcfu_3cpy8lp:	mov	r3, r7, pull #24
		ldmia	r1!, {r4 - r7}			@ Shouldnt fault
		orr	r3, r3, r4, push #8
		mov	r4, r4, pull #24
		orr	r4, r4, r5, push #8
		mov	r5, r5, pull #24
		orr	r5, r5, r6, push #8
		mov	r6, r6, pull #24
		orr	r6, r6, r7, push #8
		stmia	r0!, {r3 - r6}
		subs	ip, ip, #16
		bpl	.Lcfu_3cpy8lp

.Lcfu_3rem8lp:	tst	ip, #8
		movne	r3, r7, pull #24
		ldmneia	r1!, {r4, r7}			@ Shouldnt fault
		orrne	r3, r3, r4, push #8
		movne	r4, r4, pull #24
		orrne	r4, r4, r7, push #8
		stmneia	r0!, {r3 - r4}
		tst	ip, #4
		movne	r3, r7, pull #24
USER(		ldrnet	r7, [r1], #4)			@ May fault
		orrne	r3, r3, r7, push #8
		strne	r3, [r0], #4
		ands	ip, ip, #3
		beq	.Lcfu_3fupi
.Lcfu_3nowords:	mov	r3, r7, get_byte_3
		teq	ip, #0
		beq	.Lcfu_finished
		cmp	ip, #2
		strb	r3, [r0], #1
USER(		ldrgebt	r3, [r1], #1)			@ May fault
		strgeb	r3, [r0], #1
USER(		ldrgtbt	r3, [r1], #1)			@ May fault
		strgtb	r3, [r0], #1
		b	.Lcfu_finished
ENDPROC(__copy_from_user)

		.pushsection .fixup,"ax"
		.align	0
		/*
		 * We took an exception.  r0 contains a pointer to
		 * the byte not copied.
		 */
9001:		ldr	r2, [sp], #4			@ void *to
		sub	r2, r0, r2			@ bytes copied
		ldr	r1, [sp], #4			@ unsigned long count
		subs	r4, r1, r2			@ bytes left to copy
		movne	r1, r4
		blne	__memzero
		mov	r0, r4
		ldmfd	sp!, {r4 - r7, pc}
		.popsection
Commit	Line	Data
1da177e4 LT	1	/*
	2	* linux/arch/arm/lib/uaccess.S
	3	*
	4	* Copyright (C) 1995, 1996,1997,1998 Russell King
	5	*
	6	* This program is free software; you can redistribute it and/or modify
	7	* it under the terms of the GNU General Public License version 2 as
	8	* published by the Free Software Foundation.
	9	*
	10	* Routines to block copy data to/from user memory
	11	* These are highly optimised both for the 4k page size
	12	* and for various alignments.
	13	*/
	14	#include <linux/linkage.h>
	15	#include <asm/assembler.h>
	16	#include <asm/errno.h>
	17
	18	.text
	19
	20	#define PAGE_SHIFT 12
	21
02fcb974	22	/* Prototype: int __copy_to_user(void to, const char from, size_t n)
1da177e4 LT	23	* Purpose : copy a block to user memory from kernel memory
	24	* Params : to - user memory
	25	* : from - kernel memory
	26	* : n - number of bytes to copy
	27	* Returns : Number of bytes NOT copied.
	28	*/
	29
7ba11a9c	30	.Lc2u_dest_not_aligned:
1da177e4 LT	31	rsb ip, ip, #4
	32	cmp ip, #2
	33	ldrb r3, [r1], #1
	34	USER( strbt r3, [r0], #1) @ May fault
	35	ldrgeb r3, [r1], #1
	36	USER( strgebt r3, [r0], #1) @ May fault
	37	ldrgtb r3, [r1], #1
	38	USER( strgtbt r3, [r0], #1) @ May fault
	39	sub r2, r2, ip
7ba11a9c	40	b .Lc2u_dest_aligned
1da177e4	41
02fcb974	42	ENTRY(__copy_to_user)
1da177e4 LT	43	stmfd sp!, {r2, r4 - r7, lr}
1da177e4 LT	44	cmp r2, #4
7ba11a9c	45	blt .Lc2u_not_enough
1da177e4	46	ands ip, r0, #3
7ba11a9c NP	47	bne .Lc2u_dest_not_aligned
7ba11a9c NP	48	.Lc2u_dest_aligned:
1da177e4 LT	49
1da177e4 LT	50	ands ip, r1, #3
7ba11a9c	51	bne .Lc2u_src_not_aligned
1da177e4 LT	52	/*
	53	* Seeing as there has to be at least 8 bytes to copy, we can
	54	* copy one word, and force a user-mode page fault...
	55	*/
	56
7ba11a9c	57	.Lc2u_0fupi: subs r2, r2, #4
1da177e4	58	addmi ip, r2, #4
7ba11a9c	59	bmi .Lc2u_0nowords
1da177e4 LT	60	ldr r3, [r1], #4
	61	USER( strt r3, [r0], #4) @ May fault
	62	mov ip, r0, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
	63	rsb ip, ip, #0
	64	movs ip, ip, lsr #32 - PAGE_SHIFT
7ba11a9c	65	beq .Lc2u_0fupi
1da177e4 LT	66	/*
	67	* ip = max no. of bytes to copy before needing another "strt" insn
	68	*/
	69	cmp r2, ip
	70	movlt ip, r2
	71	sub r2, r2, ip
	72	subs ip, ip, #32
7ba11a9c	73	blt .Lc2u_0rem8lp
73f0f7c7	74
7ba11a9c	75	.Lc2u_0cpy8lp: ldmia r1!, {r3 - r6}
1da177e4 LT	76	stmia r0!, {r3 - r6} @ Shouldnt fault
	77	ldmia r1!, {r3 - r6}
	78	subs ip, ip, #32
	79	stmia r0!, {r3 - r6} @ Shouldnt fault
7ba11a9c	80	bpl .Lc2u_0cpy8lp
1da177e4	81
7ba11a9c	82	.Lc2u_0rem8lp: cmn ip, #16
1da177e4 LT	83	ldmgeia r1!, {r3 - r6}
	84	stmgeia r0!, {r3 - r6} @ Shouldnt fault
	85	tst ip, #8
	86	ldmneia r1!, {r3 - r4}
	87	stmneia r0!, {r3 - r4} @ Shouldnt fault
	88	tst ip, #4
	89	ldrne r3, [r1], #4
	90	strnet r3, [r0], #4 @ Shouldnt fault
	91	ands ip, ip, #3
7ba11a9c NP	92	beq .Lc2u_0fupi
	93	.Lc2u_0nowords: teq ip, #0
	94	beq .Lc2u_finished
	95	.Lc2u_nowords: cmp ip, #2
1da177e4 LT	96	ldrb r3, [r1], #1
	97	USER( strbt r3, [r0], #1) @ May fault
	98	ldrgeb r3, [r1], #1
	99	USER( strgebt r3, [r0], #1) @ May fault
	100	ldrgtb r3, [r1], #1
	101	USER( strgtbt r3, [r0], #1) @ May fault
7ba11a9c	102	b .Lc2u_finished
1da177e4	103
7ba11a9c	104	.Lc2u_not_enough:
1da177e4	105	movs ip, r2
7ba11a9c NP	106	bne .Lc2u_nowords
7ba11a9c NP	107	.Lc2u_finished: mov r0, #0
1b93a717	108	ldmfd sp!, {r2, r4 - r7, pc}
1da177e4	109
7ba11a9c	110	.Lc2u_src_not_aligned:
1da177e4 LT	111	bic r1, r1, #3
	112	ldr r7, [r1], #4
	113	cmp ip, #2
7ba11a9c NP	114	bgt .Lc2u_3fupi
	115	beq .Lc2u_2fupi
	116	.Lc2u_1fupi: subs r2, r2, #4
1da177e4	117	addmi ip, r2, #4
7ba11a9c	118	bmi .Lc2u_1nowords
1da177e4 LT	119	mov r3, r7, pull #8
	120	ldr r7, [r1], #4
	121	orr r3, r3, r7, push #24
	122	USER( strt r3, [r0], #4) @ May fault
	123	mov ip, r0, lsl #32 - PAGE_SHIFT
	124	rsb ip, ip, #0
	125	movs ip, ip, lsr #32 - PAGE_SHIFT
7ba11a9c	126	beq .Lc2u_1fupi
1da177e4 LT	127	cmp r2, ip
	128	movlt ip, r2
	129	sub r2, r2, ip
	130	subs ip, ip, #16
7ba11a9c	131	blt .Lc2u_1rem8lp
73f0f7c7	132
7ba11a9c	133	.Lc2u_1cpy8lp: mov r3, r7, pull #8
1da177e4 LT	134	ldmia r1!, {r4 - r7}
	135	subs ip, ip, #16
	136	orr r3, r3, r4, push #24
	137	mov r4, r4, pull #8
	138	orr r4, r4, r5, push #24
	139	mov r5, r5, pull #8
	140	orr r5, r5, r6, push #24
	141	mov r6, r6, pull #8
	142	orr r6, r6, r7, push #24
	143	stmia r0!, {r3 - r6} @ Shouldnt fault
7ba11a9c	144	bpl .Lc2u_1cpy8lp
1da177e4	145
7ba11a9c	146	.Lc2u_1rem8lp: tst ip, #8
1da177e4 LT	147	movne r3, r7, pull #8
	148	ldmneia r1!, {r4, r7}
	149	orrne r3, r3, r4, push #24
	150	movne r4, r4, pull #8
	151	orrne r4, r4, r7, push #24
	152	stmneia r0!, {r3 - r4} @ Shouldnt fault
	153	tst ip, #4
	154	movne r3, r7, pull #8
	155	ldrne r7, [r1], #4
	156	orrne r3, r3, r7, push #24
	157	strnet r3, [r0], #4 @ Shouldnt fault
	158	ands ip, ip, #3
7ba11a9c NP	159	beq .Lc2u_1fupi
7ba11a9c NP	160	.Lc2u_1nowords: mov r3, r7, get_byte_1
1da177e4	161	teq ip, #0
7ba11a9c	162	beq .Lc2u_finished
1da177e4 LT	163	cmp ip, #2
	164	USER( strbt r3, [r0], #1) @ May fault
	165	movge r3, r7, get_byte_2
	166	USER( strgebt r3, [r0], #1) @ May fault
	167	movgt r3, r7, get_byte_3
	168	USER( strgtbt r3, [r0], #1) @ May fault
7ba11a9c	169	b .Lc2u_finished
1da177e4	170
7ba11a9c	171	.Lc2u_2fupi: subs r2, r2, #4
1da177e4	172	addmi ip, r2, #4
7ba11a9c	173	bmi .Lc2u_2nowords
1da177e4 LT	174	mov r3, r7, pull #16
	175	ldr r7, [r1], #4
	176	orr r3, r3, r7, push #16
	177	USER( strt r3, [r0], #4) @ May fault
	178	mov ip, r0, lsl #32 - PAGE_SHIFT
	179	rsb ip, ip, #0
	180	movs ip, ip, lsr #32 - PAGE_SHIFT
7ba11a9c	181	beq .Lc2u_2fupi
1da177e4 LT	182	cmp r2, ip
	183	movlt ip, r2
	184	sub r2, r2, ip
	185	subs ip, ip, #16
7ba11a9c	186	blt .Lc2u_2rem8lp
73f0f7c7	187
7ba11a9c	188	.Lc2u_2cpy8lp: mov r3, r7, pull #16
1da177e4 LT	189	ldmia r1!, {r4 - r7}
	190	subs ip, ip, #16
	191	orr r3, r3, r4, push #16
	192	mov r4, r4, pull #16
	193	orr r4, r4, r5, push #16
	194	mov r5, r5, pull #16
	195	orr r5, r5, r6, push #16
	196	mov r6, r6, pull #16
	197	orr r6, r6, r7, push #16
	198	stmia r0!, {r3 - r6} @ Shouldnt fault
7ba11a9c	199	bpl .Lc2u_2cpy8lp
1da177e4	200
7ba11a9c	201	.Lc2u_2rem8lp: tst ip, #8
1da177e4 LT	202	movne r3, r7, pull #16
	203	ldmneia r1!, {r4, r7}
	204	orrne r3, r3, r4, push #16
	205	movne r4, r4, pull #16
	206	orrne r4, r4, r7, push #16
	207	stmneia r0!, {r3 - r4} @ Shouldnt fault
	208	tst ip, #4
	209	movne r3, r7, pull #16
	210	ldrne r7, [r1], #4
	211	orrne r3, r3, r7, push #16
	212	strnet r3, [r0], #4 @ Shouldnt fault
	213	ands ip, ip, #3
7ba11a9c NP	214	beq .Lc2u_2fupi
7ba11a9c NP	215	.Lc2u_2nowords: mov r3, r7, get_byte_2
1da177e4	216	teq ip, #0
7ba11a9c	217	beq .Lc2u_finished
1da177e4 LT	218	cmp ip, #2
	219	USER( strbt r3, [r0], #1) @ May fault
	220	movge r3, r7, get_byte_3
	221	USER( strgebt r3, [r0], #1) @ May fault
	222	ldrgtb r3, [r1], #0
	223	USER( strgtbt r3, [r0], #1) @ May fault
7ba11a9c	224	b .Lc2u_finished
1da177e4	225
7ba11a9c	226	.Lc2u_3fupi: subs r2, r2, #4
1da177e4	227	addmi ip, r2, #4
7ba11a9c	228	bmi .Lc2u_3nowords
1da177e4 LT	229	mov r3, r7, pull #24
	230	ldr r7, [r1], #4
	231	orr r3, r3, r7, push #8
	232	USER( strt r3, [r0], #4) @ May fault
	233	mov ip, r0, lsl #32 - PAGE_SHIFT
	234	rsb ip, ip, #0
	235	movs ip, ip, lsr #32 - PAGE_SHIFT
7ba11a9c	236	beq .Lc2u_3fupi
1da177e4 LT	237	cmp r2, ip
	238	movlt ip, r2
	239	sub r2, r2, ip
	240	subs ip, ip, #16
7ba11a9c	241	blt .Lc2u_3rem8lp
73f0f7c7	242
7ba11a9c	243	.Lc2u_3cpy8lp: mov r3, r7, pull #24
1da177e4 LT	244	ldmia r1!, {r4 - r7}
	245	subs ip, ip, #16
	246	orr r3, r3, r4, push #8
	247	mov r4, r4, pull #24
	248	orr r4, r4, r5, push #8
	249	mov r5, r5, pull #24
	250	orr r5, r5, r6, push #8
	251	mov r6, r6, pull #24
	252	orr r6, r6, r7, push #8
	253	stmia r0!, {r3 - r6} @ Shouldnt fault
7ba11a9c	254	bpl .Lc2u_3cpy8lp
1da177e4	255
7ba11a9c	256	.Lc2u_3rem8lp: tst ip, #8
1da177e4 LT	257	movne r3, r7, pull #24
	258	ldmneia r1!, {r4, r7}
	259	orrne r3, r3, r4, push #8
	260	movne r4, r4, pull #24
	261	orrne r4, r4, r7, push #8
	262	stmneia r0!, {r3 - r4} @ Shouldnt fault
	263	tst ip, #4
	264	movne r3, r7, pull #24
	265	ldrne r7, [r1], #4
	266	orrne r3, r3, r7, push #8
	267	strnet r3, [r0], #4 @ Shouldnt fault
	268	ands ip, ip, #3
7ba11a9c NP	269	beq .Lc2u_3fupi
7ba11a9c NP	270	.Lc2u_3nowords: mov r3, r7, get_byte_3
1da177e4	271	teq ip, #0
7ba11a9c	272	beq .Lc2u_finished
1da177e4 LT	273	cmp ip, #2
	274	USER( strbt r3, [r0], #1) @ May fault
	275	ldrgeb r3, [r1], #1
	276	USER( strgebt r3, [r0], #1) @ May fault
	277	ldrgtb r3, [r1], #0
	278	USER( strgtbt r3, [r0], #1) @ May fault
7ba11a9c	279	b .Lc2u_finished
93ed3970	280	ENDPROC(__copy_to_user)
1da177e4	281
4260415f	282	.pushsection .fixup,"ax"
1da177e4	283	.align 0
1b93a717	284	9001: ldmfd sp!, {r0, r4 - r7, pc}
4260415f	285	.popsection
1da177e4	286
02fcb974	287	/* Prototype: unsigned long __copy_from_user(void to,const void from,unsigned long n);
1da177e4 LT	288	* Purpose : copy a block from user memory to kernel memory
	289	* Params : to - kernel memory
	290	* : from - user memory
	291	* : n - number of bytes to copy
	292	* Returns : Number of bytes NOT copied.
	293	*/
7ba11a9c	294	.Lcfu_dest_not_aligned:
1da177e4 LT	295	rsb ip, ip, #4
	296	cmp ip, #2
	297	USER( ldrbt r3, [r1], #1) @ May fault
	298	strb r3, [r0], #1
	299	USER( ldrgebt r3, [r1], #1) @ May fault
	300	strgeb r3, [r0], #1
	301	USER( ldrgtbt r3, [r1], #1) @ May fault
	302	strgtb r3, [r0], #1
	303	sub r2, r2, ip
7ba11a9c	304	b .Lcfu_dest_aligned
1da177e4	305
02fcb974	306	ENTRY(__copy_from_user)
1da177e4 LT	307	stmfd sp!, {r0, r2, r4 - r7, lr}
1da177e4 LT	308	cmp r2, #4
7ba11a9c	309	blt .Lcfu_not_enough
1da177e4	310	ands ip, r0, #3
7ba11a9c NP	311	bne .Lcfu_dest_not_aligned
7ba11a9c NP	312	.Lcfu_dest_aligned:
1da177e4	313	ands ip, r1, #3
7ba11a9c NP	314	bne .Lcfu_src_not_aligned
7ba11a9c NP	315
1da177e4 LT	316	/*
	317	* Seeing as there has to be at least 8 bytes to copy, we can
	318	* copy one word, and force a user-mode page fault...
	319	*/
	320
7ba11a9c	321	.Lcfu_0fupi: subs r2, r2, #4
1da177e4	322	addmi ip, r2, #4
7ba11a9c	323	bmi .Lcfu_0nowords
1da177e4 LT	324	USER( ldrt r3, [r1], #4)
	325	str r3, [r0], #4
	326	mov ip, r1, lsl #32 - PAGE_SHIFT @ On each page, use a ld/st??t instruction
	327	rsb ip, ip, #0
	328	movs ip, ip, lsr #32 - PAGE_SHIFT
7ba11a9c	329	beq .Lcfu_0fupi
1da177e4 LT	330	/*
	331	* ip = max no. of bytes to copy before needing another "strt" insn
	332	*/
	333	cmp r2, ip
	334	movlt ip, r2
	335	sub r2, r2, ip
	336	subs ip, ip, #32
7ba11a9c	337	blt .Lcfu_0rem8lp
73f0f7c7	338
7ba11a9c	339	.Lcfu_0cpy8lp: ldmia r1!, {r3 - r6} @ Shouldnt fault
1da177e4 LT	340	stmia r0!, {r3 - r6}
	341	ldmia r1!, {r3 - r6} @ Shouldnt fault
	342	subs ip, ip, #32
	343	stmia r0!, {r3 - r6}
7ba11a9c	344	bpl .Lcfu_0cpy8lp
1da177e4	345
7ba11a9c	346	.Lcfu_0rem8lp: cmn ip, #16
1da177e4 LT	347	ldmgeia r1!, {r3 - r6} @ Shouldnt fault
	348	stmgeia r0!, {r3 - r6}
	349	tst ip, #8
	350	ldmneia r1!, {r3 - r4} @ Shouldnt fault
	351	stmneia r0!, {r3 - r4}
	352	tst ip, #4
	353	ldrnet r3, [r1], #4 @ Shouldnt fault
	354	strne r3, [r0], #4
	355	ands ip, ip, #3
7ba11a9c NP	356	beq .Lcfu_0fupi
	357	.Lcfu_0nowords: teq ip, #0
	358	beq .Lcfu_finished
	359	.Lcfu_nowords: cmp ip, #2
1da177e4 LT	360	USER( ldrbt r3, [r1], #1) @ May fault
	361	strb r3, [r0], #1
	362	USER( ldrgebt r3, [r1], #1) @ May fault
	363	strgeb r3, [r0], #1
	364	USER( ldrgtbt r3, [r1], #1) @ May fault
	365	strgtb r3, [r0], #1
7ba11a9c	366	b .Lcfu_finished
1da177e4	367
7ba11a9c	368	.Lcfu_not_enough:
1da177e4	369	movs ip, r2
7ba11a9c NP	370	bne .Lcfu_nowords
7ba11a9c NP	371	.Lcfu_finished: mov r0, #0
1da177e4	372	add sp, sp, #8
1b93a717	373	ldmfd sp!, {r4 - r7, pc}
1da177e4	374
7ba11a9c	375	.Lcfu_src_not_aligned:
1da177e4 LT	376	bic r1, r1, #3
	377	USER( ldrt r7, [r1], #4) @ May fault
	378	cmp ip, #2
7ba11a9c NP	379	bgt .Lcfu_3fupi
	380	beq .Lcfu_2fupi
	381	.Lcfu_1fupi: subs r2, r2, #4
1da177e4	382	addmi ip, r2, #4
7ba11a9c	383	bmi .Lcfu_1nowords
1da177e4 LT	384	mov r3, r7, pull #8
	385	USER( ldrt r7, [r1], #4) @ May fault
	386	orr r3, r3, r7, push #24
	387	str r3, [r0], #4
	388	mov ip, r1, lsl #32 - PAGE_SHIFT
	389	rsb ip, ip, #0
	390	movs ip, ip, lsr #32 - PAGE_SHIFT
7ba11a9c	391	beq .Lcfu_1fupi
1da177e4 LT	392	cmp r2, ip
	393	movlt ip, r2
	394	sub r2, r2, ip
	395	subs ip, ip, #16
7ba11a9c	396	blt .Lcfu_1rem8lp
73f0f7c7	397
7ba11a9c	398	.Lcfu_1cpy8lp: mov r3, r7, pull #8
1da177e4 LT	399	ldmia r1!, {r4 - r7} @ Shouldnt fault
	400	subs ip, ip, #16
	401	orr r3, r3, r4, push #24
	402	mov r4, r4, pull #8
	403	orr r4, r4, r5, push #24
	404	mov r5, r5, pull #8
	405	orr r5, r5, r6, push #24
	406	mov r6, r6, pull #8
	407	orr r6, r6, r7, push #24
	408	stmia r0!, {r3 - r6}
7ba11a9c	409	bpl .Lcfu_1cpy8lp
1da177e4	410
7ba11a9c	411	.Lcfu_1rem8lp: tst ip, #8
1da177e4 LT	412	movne r3, r7, pull #8
	413	ldmneia r1!, {r4, r7} @ Shouldnt fault
	414	orrne r3, r3, r4, push #24
	415	movne r4, r4, pull #8
	416	orrne r4, r4, r7, push #24
	417	stmneia r0!, {r3 - r4}
	418	tst ip, #4
	419	movne r3, r7, pull #8
	420	USER( ldrnet r7, [r1], #4) @ May fault
	421	orrne r3, r3, r7, push #24
	422	strne r3, [r0], #4
	423	ands ip, ip, #3
7ba11a9c NP	424	beq .Lcfu_1fupi
7ba11a9c NP	425	.Lcfu_1nowords: mov r3, r7, get_byte_1
1da177e4	426	teq ip, #0
7ba11a9c	427	beq .Lcfu_finished
1da177e4 LT	428	cmp ip, #2
	429	strb r3, [r0], #1
	430	movge r3, r7, get_byte_2
	431	strgeb r3, [r0], #1
	432	movgt r3, r7, get_byte_3
	433	strgtb r3, [r0], #1
7ba11a9c	434	b .Lcfu_finished
1da177e4	435
7ba11a9c	436	.Lcfu_2fupi: subs r2, r2, #4
1da177e4	437	addmi ip, r2, #4
7ba11a9c	438	bmi .Lcfu_2nowords
1da177e4 LT	439	mov r3, r7, pull #16
	440	USER( ldrt r7, [r1], #4) @ May fault
	441	orr r3, r3, r7, push #16
	442	str r3, [r0], #4
	443	mov ip, r1, lsl #32 - PAGE_SHIFT
	444	rsb ip, ip, #0
	445	movs ip, ip, lsr #32 - PAGE_SHIFT
7ba11a9c	446	beq .Lcfu_2fupi
1da177e4 LT	447	cmp r2, ip
	448	movlt ip, r2
	449	sub r2, r2, ip
	450	subs ip, ip, #16
7ba11a9c NP	451	blt .Lcfu_2rem8lp
7ba11a9c NP	452
73f0f7c7	453
7ba11a9c	454	.Lcfu_2cpy8lp: mov r3, r7, pull #16
1da177e4 LT	455	ldmia r1!, {r4 - r7} @ Shouldnt fault
	456	subs ip, ip, #16
	457	orr r3, r3, r4, push #16
	458	mov r4, r4, pull #16
	459	orr r4, r4, r5, push #16
	460	mov r5, r5, pull #16
	461	orr r5, r5, r6, push #16
	462	mov r6, r6, pull #16
	463	orr r6, r6, r7, push #16
	464	stmia r0!, {r3 - r6}
7ba11a9c	465	bpl .Lcfu_2cpy8lp
1da177e4	466
7ba11a9c	467	.Lcfu_2rem8lp: tst ip, #8
1da177e4 LT	468	movne r3, r7, pull #16
	469	ldmneia r1!, {r4, r7} @ Shouldnt fault
	470	orrne r3, r3, r4, push #16
	471	movne r4, r4, pull #16
	472	orrne r4, r4, r7, push #16
	473	stmneia r0!, {r3 - r4}
	474	tst ip, #4
	475	movne r3, r7, pull #16
	476	USER( ldrnet r7, [r1], #4) @ May fault
	477	orrne r3, r3, r7, push #16
	478	strne r3, [r0], #4
	479	ands ip, ip, #3
7ba11a9c NP	480	beq .Lcfu_2fupi
7ba11a9c NP	481	.Lcfu_2nowords: mov r3, r7, get_byte_2
1da177e4	482	teq ip, #0
7ba11a9c	483	beq .Lcfu_finished
1da177e4 LT	484	cmp ip, #2
	485	strb r3, [r0], #1
	486	movge r3, r7, get_byte_3
	487	strgeb r3, [r0], #1
	488	USER( ldrgtbt r3, [r1], #0) @ May fault
	489	strgtb r3, [r0], #1
7ba11a9c	490	b .Lcfu_finished
1da177e4	491
7ba11a9c	492	.Lcfu_3fupi: subs r2, r2, #4
1da177e4	493	addmi ip, r2, #4
7ba11a9c	494	bmi .Lcfu_3nowords
1da177e4 LT	495	mov r3, r7, pull #24
	496	USER( ldrt r7, [r1], #4) @ May fault
	497	orr r3, r3, r7, push #8
	498	str r3, [r0], #4
	499	mov ip, r1, lsl #32 - PAGE_SHIFT
	500	rsb ip, ip, #0
	501	movs ip, ip, lsr #32 - PAGE_SHIFT
7ba11a9c	502	beq .Lcfu_3fupi
1da177e4 LT	503	cmp r2, ip
	504	movlt ip, r2
	505	sub r2, r2, ip
	506	subs ip, ip, #16
7ba11a9c	507	blt .Lcfu_3rem8lp
73f0f7c7	508
7ba11a9c	509	.Lcfu_3cpy8lp: mov r3, r7, pull #24
1da177e4 LT	510	ldmia r1!, {r4 - r7} @ Shouldnt fault
	511	orr r3, r3, r4, push #8
	512	mov r4, r4, pull #24
	513	orr r4, r4, r5, push #8
	514	mov r5, r5, pull #24
	515	orr r5, r5, r6, push #8
	516	mov r6, r6, pull #24
	517	orr r6, r6, r7, push #8
	518	stmia r0!, {r3 - r6}
	519	subs ip, ip, #16
7ba11a9c	520	bpl .Lcfu_3cpy8lp
1da177e4	521
7ba11a9c	522	.Lcfu_3rem8lp: tst ip, #8
1da177e4 LT	523	movne r3, r7, pull #24
	524	ldmneia r1!, {r4, r7} @ Shouldnt fault
	525	orrne r3, r3, r4, push #8
	526	movne r4, r4, pull #24
	527	orrne r4, r4, r7, push #8
	528	stmneia r0!, {r3 - r4}
	529	tst ip, #4
	530	movne r3, r7, pull #24
	531	USER( ldrnet r7, [r1], #4) @ May fault
	532	orrne r3, r3, r7, push #8
	533	strne r3, [r0], #4
	534	ands ip, ip, #3
7ba11a9c NP	535	beq .Lcfu_3fupi
7ba11a9c NP	536	.Lcfu_3nowords: mov r3, r7, get_byte_3
1da177e4	537	teq ip, #0
7ba11a9c	538	beq .Lcfu_finished
1da177e4 LT	539	cmp ip, #2
	540	strb r3, [r0], #1
	541	USER( ldrgebt r3, [r1], #1) @ May fault
	542	strgeb r3, [r0], #1
	543	USER( ldrgtbt r3, [r1], #1) @ May fault
	544	strgtb r3, [r0], #1
7ba11a9c	545	b .Lcfu_finished
93ed3970	546	ENDPROC(__copy_from_user)
1da177e4	547
4260415f	548	.pushsection .fixup,"ax"
1da177e4 LT	549	.align 0
	550	/*
	551	* We took an exception. r0 contains a pointer to
	552	* the byte not copied.
	553	*/
	554	9001: ldr r2, [sp], #4 @ void *to
	555	sub r2, r0, r2 @ bytes copied
	556	ldr r1, [sp], #4 @ unsigned long count
	557	subs r4, r1, r2 @ bytes left to copy
	558	movne r1, r4
	559	blne __memzero
	560	mov r0, r4
1b93a717	561	ldmfd sp!, {r4 - r7, pc}
4260415f	562	.popsection
1da177e4	563