[deliverable/linux.git] / arch / powerpc / lib / copyuser_power7.S

/*
 * This program is free software; you can redistribute it and/or modify
 * it under the terms of the GNU General Public License as published by
 * the Free Software Foundation; either version 2 of the License, or
 * (at your option) any later version.
 *
 * This program is distributed in the hope that it will be useful,
 * but WITHOUT ANY WARRANTY; without even the implied warranty of
 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
 * GNU General Public License for more details.
 *
 * You should have received a copy of the GNU General Public License
 * along with this program; if not, write to the Free Software
 * Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
 *
 * Copyright (C) IBM Corporation, 2011
 *
 * Author: Anton Blanchard <anton@au.ibm.com>
 */
#include <asm/ppc_asm.h>

#define STACKFRAMESIZE	256
#define STK_REG(i)	(112 + ((i)-14)*8)

	.macro err1
100:
	.section __ex_table,"a"
	.align 3
	.llong 100b,.Ldo_err1
	.previous
	.endm

	.macro err2
200:
	.section __ex_table,"a"
	.align 3
	.llong 200b,.Ldo_err2
	.previous
	.endm

#ifdef CONFIG_ALTIVEC
	.macro err3
300:
	.section __ex_table,"a"
	.align 3
	.llong 300b,.Ldo_err3
	.previous
	.endm

	.macro err4
400:
	.section __ex_table,"a"
	.align 3
	.llong 400b,.Ldo_err4
	.previous
	.endm


.Ldo_err4:
	ld	r16,STK_REG(R16)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r14,STK_REG(R14)(r1)
.Ldo_err3:
	bl	.exit_vmx_usercopy
	ld	r0,STACKFRAMESIZE+16(r1)
	mtlr	r0
	b	.Lexit
#endif /* CONFIG_ALTIVEC */

.Ldo_err2:
	ld	r22,STK_REG(R22)(r1)
	ld	r21,STK_REG(R21)(r1)
	ld	r20,STK_REG(R20)(r1)
	ld	r19,STK_REG(R19)(r1)
	ld	r18,STK_REG(R18)(r1)
	ld	r17,STK_REG(R17)(r1)
	ld	r16,STK_REG(R16)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r14,STK_REG(R14)(r1)
.Lexit:
	addi	r1,r1,STACKFRAMESIZE
.Ldo_err1:
	ld	r3,48(r1)
	ld	r4,56(r1)
	ld	r5,64(r1)
	b	__copy_tofrom_user_base


_GLOBAL(__copy_tofrom_user_power7)
#ifdef CONFIG_ALTIVEC
	cmpldi	r5,16
	cmpldi	cr1,r5,4096

	std	r3,48(r1)
	std	r4,56(r1)
	std	r5,64(r1)

	blt	.Lshort_copy
	bgt	cr1,.Lvmx_copy
#else
	cmpldi	r5,16

	std	r3,48(r1)
	std	r4,56(r1)
	std	r5,64(r1)

	blt	.Lshort_copy
#endif

.Lnonvmx_copy:
	/* Get the source 8B aligned */
	neg	r6,r4
	mtocrf	0x01,r6
	clrldi	r6,r6,(64-3)

	bf	cr7*4+3,1f
err1;	lbz	r0,0(r4)
	addi	r4,r4,1
err1;	stb	r0,0(r3)
	addi	r3,r3,1

1:	bf	cr7*4+2,2f
err1;	lhz	r0,0(r4)
	addi	r4,r4,2
err1;	sth	r0,0(r3)
	addi	r3,r3,2

2:	bf	cr7*4+1,3f
err1;	lwz	r0,0(r4)
	addi	r4,r4,4
err1;	stw	r0,0(r3)
	addi	r3,r3,4

3:	sub	r5,r5,r6
	cmpldi	r5,128
	blt	5f

	mflr	r0
	stdu	r1,-STACKFRAMESIZE(r1)
	std	r14,STK_REG(R14)(r1)
	std	r15,STK_REG(R15)(r1)
	std	r16,STK_REG(R16)(r1)
	std	r17,STK_REG(R17)(r1)
	std	r18,STK_REG(R18)(r1)
	std	r19,STK_REG(R19)(r1)
	std	r20,STK_REG(R20)(r1)
	std	r21,STK_REG(R21)(r1)
	std	r22,STK_REG(R22)(r1)
	std	r0,STACKFRAMESIZE+16(r1)

	srdi	r6,r5,7
	mtctr	r6

	/* Now do cacheline (128B) sized loads and stores. */
	.align	5
4:
err2;	ld	r0,0(r4)
err2;	ld	r6,8(r4)
err2;	ld	r7,16(r4)
err2;	ld	r8,24(r4)
err2;	ld	r9,32(r4)
err2;	ld	r10,40(r4)
err2;	ld	r11,48(r4)
err2;	ld	r12,56(r4)
err2;	ld	r14,64(r4)
err2;	ld	r15,72(r4)
err2;	ld	r16,80(r4)
err2;	ld	r17,88(r4)
err2;	ld	r18,96(r4)
err2;	ld	r19,104(r4)
err2;	ld	r20,112(r4)
err2;	ld	r21,120(r4)
	addi	r4,r4,128
err2;	std	r0,0(r3)
err2;	std	r6,8(r3)
err2;	std	r7,16(r3)
err2;	std	r8,24(r3)
err2;	std	r9,32(r3)
err2;	std	r10,40(r3)
err2;	std	r11,48(r3)
err2;	std	r12,56(r3)
err2;	std	r14,64(r3)
err2;	std	r15,72(r3)
err2;	std	r16,80(r3)
err2;	std	r17,88(r3)
err2;	std	r18,96(r3)
err2;	std	r19,104(r3)
err2;	std	r20,112(r3)
err2;	std	r21,120(r3)
	addi	r3,r3,128
	bdnz	4b

	clrldi	r5,r5,(64-7)

	ld	r14,STK_REG(R14)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r16,STK_REG(R16)(r1)
	ld	r17,STK_REG(R17)(r1)
	ld	r18,STK_REG(R18)(r1)
	ld	r19,STK_REG(R19)(r1)
	ld	r20,STK_REG(R20)(r1)
	ld	r21,STK_REG(R21)(r1)
	ld	r22,STK_REG(R22)(r1)
	addi	r1,r1,STACKFRAMESIZE

	/* Up to 127B to go */
5:	srdi	r6,r5,4
	mtocrf	0x01,r6

6:	bf	cr7*4+1,7f
err1;	ld	r0,0(r4)
err1;	ld	r6,8(r4)
err1;	ld	r7,16(r4)
err1;	ld	r8,24(r4)
err1;	ld	r9,32(r4)
err1;	ld	r10,40(r4)
err1;	ld	r11,48(r4)
err1;	ld	r12,56(r4)
	addi	r4,r4,64
err1;	std	r0,0(r3)
err1;	std	r6,8(r3)
err1;	std	r7,16(r3)
err1;	std	r8,24(r3)
err1;	std	r9,32(r3)
err1;	std	r10,40(r3)
err1;	std	r11,48(r3)
err1;	std	r12,56(r3)
	addi	r3,r3,64

	/* Up to 63B to go */
7:	bf	cr7*4+2,8f
err1;	ld	r0,0(r4)
err1;	ld	r6,8(r4)
err1;	ld	r7,16(r4)
err1;	ld	r8,24(r4)
	addi	r4,r4,32
err1;	std	r0,0(r3)
err1;	std	r6,8(r3)
err1;	std	r7,16(r3)
err1;	std	r8,24(r3)
	addi	r3,r3,32

	/* Up to 31B to go */
8:	bf	cr7*4+3,9f
err1;	ld	r0,0(r4)
err1;	ld	r6,8(r4)
	addi	r4,r4,16
err1;	std	r0,0(r3)
err1;	std	r6,8(r3)
	addi	r3,r3,16

9:	clrldi	r5,r5,(64-4)

	/* Up to 15B to go */
.Lshort_copy:
	mtocrf	0x01,r5
	bf	cr7*4+0,12f
err1;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
err1;	lwz	r6,4(r4)
	addi	r4,r4,8
err1;	stw	r0,0(r3)
err1;	stw	r6,4(r3)
	addi	r3,r3,8

12:	bf	cr7*4+1,13f
err1;	lwz	r0,0(r4)
	addi	r4,r4,4
err1;	stw	r0,0(r3)
	addi	r3,r3,4

13:	bf	cr7*4+2,14f
err1;	lhz	r0,0(r4)
	addi	r4,r4,2
err1;	sth	r0,0(r3)
	addi	r3,r3,2

14:	bf	cr7*4+3,15f
err1;	lbz	r0,0(r4)
err1;	stb	r0,0(r3)

15:	li	r3,0
	blr

.Lunwind_stack_nonvmx_copy:
	addi	r1,r1,STACKFRAMESIZE
	b	.Lnonvmx_copy

#ifdef CONFIG_ALTIVEC
.Lvmx_copy:
	mflr	r0
	std	r0,16(r1)
	stdu	r1,-STACKFRAMESIZE(r1)
	bl	.enter_vmx_usercopy
	cmpwi	r3,0
	ld	r0,STACKFRAMESIZE+16(r1)
	ld	r3,STACKFRAMESIZE+48(r1)
	ld	r4,STACKFRAMESIZE+56(r1)
	ld	r5,STACKFRAMESIZE+64(r1)
	mtlr	r0

	/*
	 * We prefetch both the source and destination using enhanced touch
	 * instructions. We use a stream ID of 0 for the load side and
	 * 1 for the store side.
	 */
	clrrdi	r6,r4,7
	clrrdi	r9,r3,7
	ori	r9,r9,1		/* stream=1 */

	srdi	r7,r5,7		/* length in cachelines, capped at 0x3FF */
	cmpldi	r7,0x3FF
	ble	1f
	li	r7,0x3FF
1:	lis	r0,0x0E00	/* depth=7 */
	sldi	r7,r7,7
	or	r7,r7,r0
	ori	r10,r7,1	/* stream=1 */

	lis	r8,0x8000	/* GO=1 */
	clrldi	r8,r8,32

.machine push
.machine "power4"
	dcbt	r0,r6,0b01000
	dcbt	r0,r7,0b01010
	dcbtst	r0,r9,0b01000
	dcbtst	r0,r10,0b01010
	eieio
	dcbt	r0,r8,0b01010	/* GO */
.machine pop

	/*
	 * We prefetch both the source and destination using enhanced touch
	 * instructions. We use a stream ID of 0 for the load side and
	 * 1 for the store side.
	 */
	clrrdi	r6,r4,7
	clrrdi	r9,r3,7
	ori	r9,r9,1		/* stream=1 */

	srdi	r7,r5,7		/* length in cachelines, capped at 0x3FF */
	cmpldi	cr1,r7,0x3FF
	ble	cr1,1f
	li	r7,0x3FF
1:	lis	r0,0x0E00	/* depth=7 */
	sldi	r7,r7,7
	or	r7,r7,r0
	ori	r10,r7,1	/* stream=1 */

	lis	r8,0x8000	/* GO=1 */
	clrldi	r8,r8,32

.machine push
.machine "power4"
	dcbt	r0,r6,0b01000
	dcbt	r0,r7,0b01010
	dcbtst	r0,r9,0b01000
	dcbtst	r0,r10,0b01010
	eieio
	dcbt	r0,r8,0b01010	/* GO */
.machine pop

	beq	.Lunwind_stack_nonvmx_copy

	/*
	 * If source and destination are not relatively aligned we use a
	 * slower permute loop.
	 */
	xor	r6,r4,r3
	rldicl.	r6,r6,0,(64-4)
	bne	.Lvmx_unaligned_copy

	/* Get the destination 16B aligned */
	neg	r6,r3
	mtocrf	0x01,r6
	clrldi	r6,r6,(64-4)

	bf	cr7*4+3,1f
err3;	lbz	r0,0(r4)
	addi	r4,r4,1
err3;	stb	r0,0(r3)
	addi	r3,r3,1

1:	bf	cr7*4+2,2f
err3;	lhz	r0,0(r4)
	addi	r4,r4,2
err3;	sth	r0,0(r3)
	addi	r3,r3,2

2:	bf	cr7*4+1,3f
err3;	lwz	r0,0(r4)
	addi	r4,r4,4
err3;	stw	r0,0(r3)
	addi	r3,r3,4

3:	bf	cr7*4+0,4f
err3;	ld	r0,0(r4)
	addi	r4,r4,8
err3;	std	r0,0(r3)
	addi	r3,r3,8

4:	sub	r5,r5,r6

	/* Get the desination 128B aligned */
	neg	r6,r3
	srdi	r7,r6,4
	mtocrf	0x01,r7
	clrldi	r6,r6,(64-7)

	li	r9,16
	li	r10,32
	li	r11,48

	bf	cr7*4+3,5f
err3;	lvx	vr1,r0,r4
	addi	r4,r4,16
err3;	stvx	vr1,r0,r3
	addi	r3,r3,16

5:	bf	cr7*4+2,6f
err3;	lvx	vr1,r0,r4
err3;	lvx	vr0,r4,r9
	addi	r4,r4,32
err3;	stvx	vr1,r0,r3
err3;	stvx	vr0,r3,r9
	addi	r3,r3,32

6:	bf	cr7*4+1,7f
err3;	lvx	vr3,r0,r4
err3;	lvx	vr2,r4,r9
err3;	lvx	vr1,r4,r10
err3;	lvx	vr0,r4,r11
	addi	r4,r4,64
err3;	stvx	vr3,r0,r3
err3;	stvx	vr2,r3,r9
err3;	stvx	vr1,r3,r10
err3;	stvx	vr0,r3,r11
	addi	r3,r3,64

7:	sub	r5,r5,r6
	srdi	r6,r5,7

	std	r14,STK_REG(R14)(r1)
	std	r15,STK_REG(R15)(r1)
	std	r16,STK_REG(R16)(r1)

	li	r12,64
	li	r14,80
	li	r15,96
	li	r16,112

	mtctr	r6

	/*
	 * Now do cacheline sized loads and stores. By this stage the
	 * cacheline stores are also cacheline aligned.
	 */
	.align	5
8:
err4;	lvx	vr7,r0,r4
err4;	lvx	vr6,r4,r9
err4;	lvx	vr5,r4,r10
err4;	lvx	vr4,r4,r11
err4;	lvx	vr3,r4,r12
err4;	lvx	vr2,r4,r14
err4;	lvx	vr1,r4,r15
err4;	lvx	vr0,r4,r16
	addi	r4,r4,128
err4;	stvx	vr7,r0,r3
err4;	stvx	vr6,r3,r9
err4;	stvx	vr5,r3,r10
err4;	stvx	vr4,r3,r11
err4;	stvx	vr3,r3,r12
err4;	stvx	vr2,r3,r14
err4;	stvx	vr1,r3,r15
err4;	stvx	vr0,r3,r16
	addi	r3,r3,128
	bdnz	8b

	ld	r14,STK_REG(R14)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r16,STK_REG(R16)(r1)

	/* Up to 127B to go */
	clrldi	r5,r5,(64-7)
	srdi	r6,r5,4
	mtocrf	0x01,r6

	bf	cr7*4+1,9f
err3;	lvx	vr3,r0,r4
err3;	lvx	vr2,r4,r9
err3;	lvx	vr1,r4,r10
err3;	lvx	vr0,r4,r11
	addi	r4,r4,64
err3;	stvx	vr3,r0,r3
err3;	stvx	vr2,r3,r9
err3;	stvx	vr1,r3,r10
err3;	stvx	vr0,r3,r11
	addi	r3,r3,64

9:	bf	cr7*4+2,10f
err3;	lvx	vr1,r0,r4
err3;	lvx	vr0,r4,r9
	addi	r4,r4,32
err3;	stvx	vr1,r0,r3
err3;	stvx	vr0,r3,r9
	addi	r3,r3,32

10:	bf	cr7*4+3,11f
err3;	lvx	vr1,r0,r4
	addi	r4,r4,16
err3;	stvx	vr1,r0,r3
	addi	r3,r3,16

	/* Up to 15B to go */
11:	clrldi	r5,r5,(64-4)
	mtocrf	0x01,r5
	bf	cr7*4+0,12f
err3;	ld	r0,0(r4)
	addi	r4,r4,8
err3;	std	r0,0(r3)
	addi	r3,r3,8

12:	bf	cr7*4+1,13f
err3;	lwz	r0,0(r4)
	addi	r4,r4,4
err3;	stw	r0,0(r3)
	addi	r3,r3,4

13:	bf	cr7*4+2,14f
err3;	lhz	r0,0(r4)
	addi	r4,r4,2
err3;	sth	r0,0(r3)
	addi	r3,r3,2

14:	bf	cr7*4+3,15f
err3;	lbz	r0,0(r4)
err3;	stb	r0,0(r3)

15:	addi	r1,r1,STACKFRAMESIZE
	b	.exit_vmx_usercopy	/* tail call optimise */

.Lvmx_unaligned_copy:
	/* Get the destination 16B aligned */
	neg	r6,r3
	mtocrf	0x01,r6
	clrldi	r6,r6,(64-4)

	bf	cr7*4+3,1f
err3;	lbz	r0,0(r4)
	addi	r4,r4,1
err3;	stb	r0,0(r3)
	addi	r3,r3,1

1:	bf	cr7*4+2,2f
err3;	lhz	r0,0(r4)
	addi	r4,r4,2
err3;	sth	r0,0(r3)
	addi	r3,r3,2

2:	bf	cr7*4+1,3f
err3;	lwz	r0,0(r4)
	addi	r4,r4,4
err3;	stw	r0,0(r3)
	addi	r3,r3,4

3:	bf	cr7*4+0,4f
err3;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
err3;	lwz	r7,4(r4)
	addi	r4,r4,8
err3;	stw	r0,0(r3)
err3;	stw	r7,4(r3)
	addi	r3,r3,8

4:	sub	r5,r5,r6

	/* Get the desination 128B aligned */
	neg	r6,r3
	srdi	r7,r6,4
	mtocrf	0x01,r7
	clrldi	r6,r6,(64-7)

	li	r9,16
	li	r10,32
	li	r11,48

	lvsl	vr16,0,r4	/* Setup permute control vector */
err3;	lvx	vr0,0,r4
	addi	r4,r4,16

	bf	cr7*4+3,5f
err3;	lvx	vr1,r0,r4
	vperm	vr8,vr0,vr1,vr16
	addi	r4,r4,16
err3;	stvx	vr8,r0,r3
	addi	r3,r3,16
	vor	vr0,vr1,vr1

5:	bf	cr7*4+2,6f
err3;	lvx	vr1,r0,r4
	vperm	vr8,vr0,vr1,vr16
err3;	lvx	vr0,r4,r9
	vperm	vr9,vr1,vr0,vr16
	addi	r4,r4,32
err3;	stvx	vr8,r0,r3
err3;	stvx	vr9,r3,r9
	addi	r3,r3,32

6:	bf	cr7*4+1,7f
err3;	lvx	vr3,r0,r4
	vperm	vr8,vr0,vr3,vr16
err3;	lvx	vr2,r4,r9
	vperm	vr9,vr3,vr2,vr16
err3;	lvx	vr1,r4,r10
	vperm	vr10,vr2,vr1,vr16
err3;	lvx	vr0,r4,r11
	vperm	vr11,vr1,vr0,vr16
	addi	r4,r4,64
err3;	stvx	vr8,r0,r3
err3;	stvx	vr9,r3,r9
err3;	stvx	vr10,r3,r10
err3;	stvx	vr11,r3,r11
	addi	r3,r3,64

7:	sub	r5,r5,r6
	srdi	r6,r5,7

	std	r14,STK_REG(R14)(r1)
	std	r15,STK_REG(R15)(r1)
	std	r16,STK_REG(R16)(r1)

	li	r12,64
	li	r14,80
	li	r15,96
	li	r16,112

	mtctr	r6

	/*
	 * Now do cacheline sized loads and stores. By this stage the
	 * cacheline stores are also cacheline aligned.
	 */
	.align	5
8:
err4;	lvx	vr7,r0,r4
	vperm	vr8,vr0,vr7,vr16
err4;	lvx	vr6,r4,r9
	vperm	vr9,vr7,vr6,vr16
err4;	lvx	vr5,r4,r10
	vperm	vr10,vr6,vr5,vr16
err4;	lvx	vr4,r4,r11
	vperm	vr11,vr5,vr4,vr16
err4;	lvx	vr3,r4,r12
	vperm	vr12,vr4,vr3,vr16
err4;	lvx	vr2,r4,r14
	vperm	vr13,vr3,vr2,vr16
err4;	lvx	vr1,r4,r15
	vperm	vr14,vr2,vr1,vr16
err4;	lvx	vr0,r4,r16
	vperm	vr15,vr1,vr0,vr16
	addi	r4,r4,128
err4;	stvx	vr8,r0,r3
err4;	stvx	vr9,r3,r9
err4;	stvx	vr10,r3,r10
err4;	stvx	vr11,r3,r11
err4;	stvx	vr12,r3,r12
err4;	stvx	vr13,r3,r14
err4;	stvx	vr14,r3,r15
err4;	stvx	vr15,r3,r16
	addi	r3,r3,128
	bdnz	8b

	ld	r14,STK_REG(R14)(r1)
	ld	r15,STK_REG(R15)(r1)
	ld	r16,STK_REG(R16)(r1)

	/* Up to 127B to go */
	clrldi	r5,r5,(64-7)
	srdi	r6,r5,4
	mtocrf	0x01,r6

	bf	cr7*4+1,9f
err3;	lvx	vr3,r0,r4
	vperm	vr8,vr0,vr3,vr16
err3;	lvx	vr2,r4,r9
	vperm	vr9,vr3,vr2,vr16
err3;	lvx	vr1,r4,r10
	vperm	vr10,vr2,vr1,vr16
err3;	lvx	vr0,r4,r11
	vperm	vr11,vr1,vr0,vr16
	addi	r4,r4,64
err3;	stvx	vr8,r0,r3
err3;	stvx	vr9,r3,r9
err3;	stvx	vr10,r3,r10
err3;	stvx	vr11,r3,r11
	addi	r3,r3,64

9:	bf	cr7*4+2,10f
err3;	lvx	vr1,r0,r4
	vperm	vr8,vr0,vr1,vr16
err3;	lvx	vr0,r4,r9
	vperm	vr9,vr1,vr0,vr16
	addi	r4,r4,32
err3;	stvx	vr8,r0,r3
err3;	stvx	vr9,r3,r9
	addi	r3,r3,32

10:	bf	cr7*4+3,11f
err3;	lvx	vr1,r0,r4
	vperm	vr8,vr0,vr1,vr16
	addi	r4,r4,16
err3;	stvx	vr8,r0,r3
	addi	r3,r3,16

	/* Up to 15B to go */
11:	clrldi	r5,r5,(64-4)
	addi	r4,r4,-16	/* Unwind the +16 load offset */
	mtocrf	0x01,r5
	bf	cr7*4+0,12f
err3;	lwz	r0,0(r4)	/* Less chance of a reject with word ops */
err3;	lwz	r6,4(r4)
	addi	r4,r4,8
err3;	stw	r0,0(r3)
err3;	stw	r6,4(r3)
	addi	r3,r3,8

12:	bf	cr7*4+1,13f
err3;	lwz	r0,0(r4)
	addi	r4,r4,4
err3;	stw	r0,0(r3)
	addi	r3,r3,4

13:	bf	cr7*4+2,14f
err3;	lhz	r0,0(r4)
	addi	r4,r4,2
err3;	sth	r0,0(r3)
	addi	r3,r3,2

14:	bf	cr7*4+3,15f
err3;	lbz	r0,0(r4)
err3;	stb	r0,0(r3)

15:	addi	r1,r1,STACKFRAMESIZE
	b	.exit_vmx_usercopy	/* tail call optimise */
#endif /* CONFiG_ALTIVEC */
Commit	Line	Data
a66086b8 AB	1	/*
	2	* This program is free software; you can redistribute it and/or modify
	3	* it under the terms of the GNU General Public License as published by
	4	* the Free Software Foundation; either version 2 of the License, or
	5	* (at your option) any later version.
	6	*
	7	* This program is distributed in the hope that it will be useful,
	8	* but WITHOUT ANY WARRANTY; without even the implied warranty of
	9	* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
	10	* GNU General Public License for more details.
	11	*
	12	* You should have received a copy of the GNU General Public License
	13	* along with this program; if not, write to the Free Software
	14	* Foundation, Inc., 59 Temple Place - Suite 330, Boston, MA 02111-1307, USA.
	15	*
	16	* Copyright (C) IBM Corporation, 2011
	17	*
	18	* Author: Anton Blanchard <anton@au.ibm.com>
	19	*/
	20	#include <asm/ppc_asm.h>
	21
	22	#define STACKFRAMESIZE 256
	23	#define STK_REG(i) (112 + ((i)-14)*8)
	24
	25	.macro err1
	26	100:
	27	.section __ex_table,"a"
	28	.align 3
	29	.llong 100b,.Ldo_err1
	30	.previous
	31	.endm
	32
	33	.macro err2
	34	200:
	35	.section __ex_table,"a"
	36	.align 3
	37	.llong 200b,.Ldo_err2
	38	.previous
	39	.endm
	40
	41	#ifdef CONFIG_ALTIVEC
	42	.macro err3
	43	300:
	44	.section __ex_table,"a"
	45	.align 3
	46	.llong 300b,.Ldo_err3
	47	.previous
	48	.endm
	49
	50	.macro err4
	51	400:
	52	.section __ex_table,"a"
	53	.align 3
	54	.llong 400b,.Ldo_err4
	55	.previous
	56	.endm
	57
	58
	59	.Ldo_err4:
c75df6f9 MN	60	ld r16,STK_REG(R16)(r1)
	61	ld r15,STK_REG(R15)(r1)
	62	ld r14,STK_REG(R14)(r1)
a66086b8	63	.Ldo_err3:
6f7839e5	64	bl .exit_vmx_usercopy
a66086b8 AB	65	ld r0,STACKFRAMESIZE+16(r1)
	66	mtlr r0
	67	b .Lexit
	68	#endif /* CONFIG_ALTIVEC */
	69
	70	.Ldo_err2:
c75df6f9 MN	71	ld r22,STK_REG(R22)(r1)
	72	ld r21,STK_REG(R21)(r1)
	73	ld r20,STK_REG(R20)(r1)
	74	ld r19,STK_REG(R19)(r1)
	75	ld r18,STK_REG(R18)(r1)
	76	ld r17,STK_REG(R17)(r1)
	77	ld r16,STK_REG(R16)(r1)
	78	ld r15,STK_REG(R15)(r1)
	79	ld r14,STK_REG(R14)(r1)
a66086b8 AB	80	.Lexit:
	81	addi r1,r1,STACKFRAMESIZE
	82	.Ldo_err1:
	83	ld r3,48(r1)
	84	ld r4,56(r1)
	85	ld r5,64(r1)
	86	b __copy_tofrom_user_base
	87
	88
	89	_GLOBAL(__copy_tofrom_user_power7)
	90	#ifdef CONFIG_ALTIVEC
	91	cmpldi r5,16
	92	cmpldi cr1,r5,4096
	93
	94	std r3,48(r1)
	95	std r4,56(r1)
	96	std r5,64(r1)
	97
	98	blt .Lshort_copy
	99	bgt cr1,.Lvmx_copy
	100	#else
	101	cmpldi r5,16
	102
	103	std r3,48(r1)
	104	std r4,56(r1)
	105	std r5,64(r1)
	106
	107	blt .Lshort_copy
	108	#endif
	109
	110	.Lnonvmx_copy:
	111	/* Get the source 8B aligned */
	112	neg r6,r4
	113	mtocrf 0x01,r6
	114	clrldi r6,r6,(64-3)
	115
	116	bf cr7*4+3,1f
	117	err1; lbz r0,0(r4)
	118	addi r4,r4,1
	119	err1; stb r0,0(r3)
	120	addi r3,r3,1
	121
	122	1: bf cr7*4+2,2f
	123	err1; lhz r0,0(r4)
	124	addi r4,r4,2
	125	err1; sth r0,0(r3)
	126	addi r3,r3,2
	127
	128	2: bf cr7*4+1,3f
	129	err1; lwz r0,0(r4)
	130	addi r4,r4,4
	131	err1; stw r0,0(r3)
	132	addi r3,r3,4
	133
	134	3: sub r5,r5,r6
	135	cmpldi r5,128
	136	blt 5f
	137
	138	mflr r0
	139	stdu r1,-STACKFRAMESIZE(r1)
c75df6f9 MN	140	std r14,STK_REG(R14)(r1)
	141	std r15,STK_REG(R15)(r1)
	142	std r16,STK_REG(R16)(r1)
	143	std r17,STK_REG(R17)(r1)
	144	std r18,STK_REG(R18)(r1)
	145	std r19,STK_REG(R19)(r1)
	146	std r20,STK_REG(R20)(r1)
	147	std r21,STK_REG(R21)(r1)
	148	std r22,STK_REG(R22)(r1)
a66086b8 AB	149	std r0,STACKFRAMESIZE+16(r1)
	150
	151	srdi r6,r5,7
	152	mtctr r6
	153
	154	/* Now do cacheline (128B) sized loads and stores. */
	155	.align 5
	156	4:
	157	err2; ld r0,0(r4)
	158	err2; ld r6,8(r4)
	159	err2; ld r7,16(r4)
	160	err2; ld r8,24(r4)
	161	err2; ld r9,32(r4)
	162	err2; ld r10,40(r4)
	163	err2; ld r11,48(r4)
	164	err2; ld r12,56(r4)
	165	err2; ld r14,64(r4)
	166	err2; ld r15,72(r4)
	167	err2; ld r16,80(r4)
	168	err2; ld r17,88(r4)
	169	err2; ld r18,96(r4)
	170	err2; ld r19,104(r4)
	171	err2; ld r20,112(r4)
	172	err2; ld r21,120(r4)
	173	addi r4,r4,128
	174	err2; std r0,0(r3)
	175	err2; std r6,8(r3)
	176	err2; std r7,16(r3)
	177	err2; std r8,24(r3)
	178	err2; std r9,32(r3)
	179	err2; std r10,40(r3)
	180	err2; std r11,48(r3)
	181	err2; std r12,56(r3)
	182	err2; std r14,64(r3)
	183	err2; std r15,72(r3)
	184	err2; std r16,80(r3)
	185	err2; std r17,88(r3)
	186	err2; std r18,96(r3)
	187	err2; std r19,104(r3)
	188	err2; std r20,112(r3)
	189	err2; std r21,120(r3)
	190	addi r3,r3,128
	191	bdnz 4b
	192
	193	clrldi r5,r5,(64-7)
	194
c75df6f9 MN	195	ld r14,STK_REG(R14)(r1)
	196	ld r15,STK_REG(R15)(r1)
	197	ld r16,STK_REG(R16)(r1)
	198	ld r17,STK_REG(R17)(r1)
	199	ld r18,STK_REG(R18)(r1)
	200	ld r19,STK_REG(R19)(r1)
	201	ld r20,STK_REG(R20)(r1)
	202	ld r21,STK_REG(R21)(r1)
	203	ld r22,STK_REG(R22)(r1)
a66086b8 AB	204	addi r1,r1,STACKFRAMESIZE
	205
	206	/* Up to 127B to go */
	207	5: srdi r6,r5,4
	208	mtocrf 0x01,r6
	209
	210	6: bf cr7*4+1,7f
	211	err1; ld r0,0(r4)
	212	err1; ld r6,8(r4)
	213	err1; ld r7,16(r4)
	214	err1; ld r8,24(r4)
	215	err1; ld r9,32(r4)
	216	err1; ld r10,40(r4)
	217	err1; ld r11,48(r4)
	218	err1; ld r12,56(r4)
	219	addi r4,r4,64
	220	err1; std r0,0(r3)
	221	err1; std r6,8(r3)
	222	err1; std r7,16(r3)
	223	err1; std r8,24(r3)
	224	err1; std r9,32(r3)
	225	err1; std r10,40(r3)
	226	err1; std r11,48(r3)
	227	err1; std r12,56(r3)
	228	addi r3,r3,64
	229
	230	/* Up to 63B to go */
	231	7: bf cr7*4+2,8f
	232	err1; ld r0,0(r4)
	233	err1; ld r6,8(r4)
	234	err1; ld r7,16(r4)
	235	err1; ld r8,24(r4)
	236	addi r4,r4,32
	237	err1; std r0,0(r3)
	238	err1; std r6,8(r3)
	239	err1; std r7,16(r3)
	240	err1; std r8,24(r3)
	241	addi r3,r3,32
	242
	243	/* Up to 31B to go */
	244	8: bf cr7*4+3,9f
	245	err1; ld r0,0(r4)
	246	err1; ld r6,8(r4)
	247	addi r4,r4,16
	248	err1; std r0,0(r3)
	249	err1; std r6,8(r3)
	250	addi r3,r3,16
	251
	252	9: clrldi r5,r5,(64-4)
	253
	254	/* Up to 15B to go */
	255	.Lshort_copy:
	256	mtocrf 0x01,r5
	257	bf cr7*4+0,12f
	258	err1; lwz r0,0(r4) /* Less chance of a reject with word ops */
	259	err1; lwz r6,4(r4)
	260	addi r4,r4,8
	261	err1; stw r0,0(r3)
	262	err1; stw r6,4(r3)
	263	addi r3,r3,8
	264
	265	12: bf cr7*4+1,13f
	266	err1; lwz r0,0(r4)
	267	addi r4,r4,4
268	err1; stw r0,0(r3)
269	addi r3,r3,4
270
271	13: bf cr7*4+2,14f
272	err1; lhz r0,0(r4)
273	addi r4,r4,2
274	err1; sth r0,0(r3)
275	addi r3,r3,2
276
277	14: bf cr7*4+3,15f
278	err1; lbz r0,0(r4)
279	err1; stb r0,0(r3)
280
281	15: li r3,0
282	blr
283
284	.Lunwind_stack_nonvmx_copy:
285	addi r1,r1,STACKFRAMESIZE
286	b .Lnonvmx_copy
287
288	#ifdef CONFIG_ALTIVEC
289	.Lvmx_copy:
290	mflr r0
291	std r0,16(r1)
292	stdu r1,-STACKFRAMESIZE(r1)
6f7839e5	293	bl .enter_vmx_usercopy
a66086b8 AB	294	cmpwi r3,0
	295	ld r0,STACKFRAMESIZE+16(r1)
	296	ld r3,STACKFRAMESIZE+48(r1)
	297	ld r4,STACKFRAMESIZE+56(r1)
	298	ld r5,STACKFRAMESIZE+64(r1)
	299	mtlr r0
	300
a9514dc6 AB	301	/*
	302	* We prefetch both the source and destination using enhanced touch
	303	* instructions. We use a stream ID of 0 for the load side and
	304	* 1 for the store side.
	305	*/
	306	clrrdi r6,r4,7
	307	clrrdi r9,r3,7
	308	ori r9,r9,1 /* stream=1 */
	309
	310	srdi r7,r5,7 /* length in cachelines, capped at 0x3FF */
	311	cmpldi r7,0x3FF
	312	ble 1f
	313	li r7,0x3FF
	314	1: lis r0,0x0E00 /* depth=7 */
	315	sldi r7,r7,7
	316	or r7,r7,r0
	317	ori r10,r7,1 /* stream=1 */
	318
	319	lis r8,0x8000 /* GO=1 */
	320	clrldi r8,r8,32
	321
bce4b4bd AB	322	.machine push
	323	.machine "power4"
	324	dcbt r0,r6,0b01000
	325	dcbt r0,r7,0b01010
	326	dcbtst r0,r9,0b01000
	327	dcbtst r0,r10,0b01010
	328	eieio
	329	dcbt r0,r8,0b01010 /* GO */
	330	.machine pop
	331
	332	/*
	333	* We prefetch both the source and destination using enhanced touch
	334	* instructions. We use a stream ID of 0 for the load side and
	335	* 1 for the store side.
	336	*/
	337	clrrdi r6,r4,7
	338	clrrdi r9,r3,7
	339	ori r9,r9,1 /* stream=1 */
	340
	341	srdi r7,r5,7 /* length in cachelines, capped at 0x3FF */
	342	cmpldi cr1,r7,0x3FF
	343	ble cr1,1f
	344	li r7,0x3FF
	345	1: lis r0,0x0E00 /* depth=7 */
	346	sldi r7,r7,7
	347	or r7,r7,r0
	348	ori r10,r7,1 /* stream=1 */
	349
	350	lis r8,0x8000 /* GO=1 */
	351	clrldi r8,r8,32
	352
a9514dc6 AB	353	.machine push
	354	.machine "power4"
	355	dcbt r0,r6,0b01000
	356	dcbt r0,r7,0b01010
	357	dcbtst r0,r9,0b01000
	358	dcbtst r0,r10,0b01010
	359	eieio
	360	dcbt r0,r8,0b01010 /* GO */
	361	.machine pop
	362
a66086b8 AB	363	beq .Lunwind_stack_nonvmx_copy
	364
	365	/*
	366	* If source and destination are not relatively aligned we use a
	367	* slower permute loop.
	368	*/
	369	xor r6,r4,r3
	370	rldicl. r6,r6,0,(64-4)
	371	bne .Lvmx_unaligned_copy
	372
	373	/* Get the destination 16B aligned */
	374	neg r6,r3
	375	mtocrf 0x01,r6
	376	clrldi r6,r6,(64-4)
	377
	378	bf cr7*4+3,1f
	379	err3; lbz r0,0(r4)
	380	addi r4,r4,1
	381	err3; stb r0,0(r3)
	382	addi r3,r3,1
	383
	384	1: bf cr7*4+2,2f
	385	err3; lhz r0,0(r4)
	386	addi r4,r4,2
	387	err3; sth r0,0(r3)
	388	addi r3,r3,2
	389
	390	2: bf cr7*4+1,3f
	391	err3; lwz r0,0(r4)
	392	addi r4,r4,4
	393	err3; stw r0,0(r3)
	394	addi r3,r3,4
	395
	396	3: bf cr7*4+0,4f
	397	err3; ld r0,0(r4)
	398	addi r4,r4,8
	399	err3; std r0,0(r3)
	400	addi r3,r3,8
	401
	402	4: sub r5,r5,r6
	403
	404	/* Get the desination 128B aligned */
	405	neg r6,r3
	406	srdi r7,r6,4
	407	mtocrf 0x01,r7
	408	clrldi r6,r6,(64-7)
	409
	410	li r9,16
	411	li r10,32
	412	li r11,48
	413
	414	bf cr7*4+3,5f
	415	err3; lvx vr1,r0,r4
	416	addi r4,r4,16
	417	err3; stvx vr1,r0,r3
	418	addi r3,r3,16
	419
	420	5: bf cr7*4+2,6f
	421	err3; lvx vr1,r0,r4
	422	err3; lvx vr0,r4,r9
	423	addi r4,r4,32
	424	err3; stvx vr1,r0,r3
	425	err3; stvx vr0,r3,r9
	426	addi r3,r3,32
427
428	6: bf cr7*4+1,7f
429	err3; lvx vr3,r0,r4
430	err3; lvx vr2,r4,r9
431	err3; lvx vr1,r4,r10
432	err3; lvx vr0,r4,r11
433	addi r4,r4,64
434	err3; stvx vr3,r0,r3
435	err3; stvx vr2,r3,r9
436	err3; stvx vr1,r3,r10
437	err3; stvx vr0,r3,r11
438	addi r3,r3,64
439
440	7: sub r5,r5,r6
441	srdi r6,r5,7
442
c75df6f9 MN	443	std r14,STK_REG(R14)(r1)
	444	std r15,STK_REG(R15)(r1)
	445	std r16,STK_REG(R16)(r1)
a66086b8 AB	446
	447	li r12,64
	448	li r14,80
	449	li r15,96
	450	li r16,112
	451
	452	mtctr r6
	453
	454	/*
	455	* Now do cacheline sized loads and stores. By this stage the
	456	* cacheline stores are also cacheline aligned.
	457	*/
	458	.align 5
	459	8:
	460	err4; lvx vr7,r0,r4
	461	err4; lvx vr6,r4,r9
	462	err4; lvx vr5,r4,r10
	463	err4; lvx vr4,r4,r11
	464	err4; lvx vr3,r4,r12
	465	err4; lvx vr2,r4,r14
	466	err4; lvx vr1,r4,r15
	467	err4; lvx vr0,r4,r16
	468	addi r4,r4,128
	469	err4; stvx vr7,r0,r3
	470	err4; stvx vr6,r3,r9
	471	err4; stvx vr5,r3,r10
	472	err4; stvx vr4,r3,r11
	473	err4; stvx vr3,r3,r12
	474	err4; stvx vr2,r3,r14
	475	err4; stvx vr1,r3,r15
	476	err4; stvx vr0,r3,r16
	477	addi r3,r3,128
	478	bdnz 8b
	479
c75df6f9 MN	480	ld r14,STK_REG(R14)(r1)
	481	ld r15,STK_REG(R15)(r1)
	482	ld r16,STK_REG(R16)(r1)
a66086b8 AB	483
	484	/* Up to 127B to go */
	485	clrldi r5,r5,(64-7)
	486	srdi r6,r5,4
	487	mtocrf 0x01,r6
	488
	489	bf cr7*4+1,9f
	490	err3; lvx vr3,r0,r4
	491	err3; lvx vr2,r4,r9
	492	err3; lvx vr1,r4,r10
	493	err3; lvx vr0,r4,r11
	494	addi r4,r4,64
	495	err3; stvx vr3,r0,r3
	496	err3; stvx vr2,r3,r9
	497	err3; stvx vr1,r3,r10
	498	err3; stvx vr0,r3,r11
	499	addi r3,r3,64
	500
	501	9: bf cr7*4+2,10f
	502	err3; lvx vr1,r0,r4
	503	err3; lvx vr0,r4,r9
	504	addi r4,r4,32
	505	err3; stvx vr1,r0,r3
	506	err3; stvx vr0,r3,r9
	507	addi r3,r3,32
	508
	509	10: bf cr7*4+3,11f
	510	err3; lvx vr1,r0,r4
	511	addi r4,r4,16
	512	err3; stvx vr1,r0,r3
	513	addi r3,r3,16
	514
	515	/* Up to 15B to go */
	516	11: clrldi r5,r5,(64-4)
	517	mtocrf 0x01,r5
	518	bf cr7*4+0,12f
	519	err3; ld r0,0(r4)
	520	addi r4,r4,8
	521	err3; std r0,0(r3)
	522	addi r3,r3,8
	523
	524	12: bf cr7*4+1,13f
	525	err3; lwz r0,0(r4)
	526	addi r4,r4,4
	527	err3; stw r0,0(r3)
	528	addi r3,r3,4
	529
	530	13: bf cr7*4+2,14f
	531	err3; lhz r0,0(r4)
	532	addi r4,r4,2
	533	err3; sth r0,0(r3)
	534	addi r3,r3,2
	535
	536	14: bf cr7*4+3,15f
	537	err3; lbz r0,0(r4)
	538	err3; stb r0,0(r3)
	539
	540	15: addi r1,r1,STACKFRAMESIZE
6f7839e5	541	b .exit_vmx_usercopy /* tail call optimise */
a66086b8 AB	542
	543	.Lvmx_unaligned_copy:
	544	/* Get the destination 16B aligned */
	545	neg r6,r3
	546	mtocrf 0x01,r6
	547	clrldi r6,r6,(64-4)
	548
	549	bf cr7*4+3,1f
	550	err3; lbz r0,0(r4)
	551	addi r4,r4,1
	552	err3; stb r0,0(r3)
	553	addi r3,r3,1
	554
	555	1: bf cr7*4+2,2f
	556	err3; lhz r0,0(r4)
	557	addi r4,r4,2
	558	err3; sth r0,0(r3)
	559	addi r3,r3,2
	560
	561	2: bf cr7*4+1,3f
	562	err3; lwz r0,0(r4)
	563	addi r4,r4,4
	564	err3; stw r0,0(r3)
	565	addi r3,r3,4
	566
	567	3: bf cr7*4+0,4f
	568	err3; lwz r0,0(r4) /* Less chance of a reject with word ops */
	569	err3; lwz r7,4(r4)
	570	addi r4,r4,8
	571	err3; stw r0,0(r3)
	572	err3; stw r7,4(r3)
	573	addi r3,r3,8
	574
	575	4: sub r5,r5,r6
	576
	577	/* Get the desination 128B aligned */
	578	neg r6,r3
	579	srdi r7,r6,4
	580	mtocrf 0x01,r7
	581	clrldi r6,r6,(64-7)
	582
	583	li r9,16
	584	li r10,32
	585	li r11,48
	586
	587	lvsl vr16,0,r4 /* Setup permute control vector */
	588	err3; lvx vr0,0,r4
	589	addi r4,r4,16
	590
	591	bf cr7*4+3,5f
	592	err3; lvx vr1,r0,r4
	593	vperm vr8,vr0,vr1,vr16
	594	addi r4,r4,16
	595	err3; stvx vr8,r0,r3
	596	addi r3,r3,16
	597	vor vr0,vr1,vr1
	598
	599	5: bf cr7*4+2,6f
	600	err3; lvx vr1,r0,r4
	601	vperm vr8,vr0,vr1,vr16
	602	err3; lvx vr0,r4,r9
	603	vperm vr9,vr1,vr0,vr16
	604	addi r4,r4,32
	605	err3; stvx vr8,r0,r3
606	err3; stvx vr9,r3,r9
607	addi r3,r3,32
608
609	6: bf cr7*4+1,7f
610	err3; lvx vr3,r0,r4
611	vperm vr8,vr0,vr3,vr16
612	err3; lvx vr2,r4,r9
613	vperm vr9,vr3,vr2,vr16
614	err3; lvx vr1,r4,r10
615	vperm vr10,vr2,vr1,vr16
616	err3; lvx vr0,r4,r11
617	vperm vr11,vr1,vr0,vr16
618	addi r4,r4,64
619	err3; stvx vr8,r0,r3
620	err3; stvx vr9,r3,r9
621	err3; stvx vr10,r3,r10
622	err3; stvx vr11,r3,r11
623	addi r3,r3,64
624
625	7: sub r5,r5,r6
626	srdi r6,r5,7
627
c75df6f9 MN	628	std r14,STK_REG(R14)(r1)
	629	std r15,STK_REG(R15)(r1)
	630	std r16,STK_REG(R16)(r1)
a66086b8 AB	631
	632	li r12,64
	633	li r14,80
	634	li r15,96
	635	li r16,112
	636
	637	mtctr r6
	638
	639	/*
	640	* Now do cacheline sized loads and stores. By this stage the
	641	* cacheline stores are also cacheline aligned.
	642	*/
	643	.align 5
	644	8:
	645	err4; lvx vr7,r0,r4
	646	vperm vr8,vr0,vr7,vr16
	647	err4; lvx vr6,r4,r9
	648	vperm vr9,vr7,vr6,vr16
	649	err4; lvx vr5,r4,r10
	650	vperm vr10,vr6,vr5,vr16
	651	err4; lvx vr4,r4,r11
	652	vperm vr11,vr5,vr4,vr16
	653	err4; lvx vr3,r4,r12
	654	vperm vr12,vr4,vr3,vr16
	655	err4; lvx vr2,r4,r14
	656	vperm vr13,vr3,vr2,vr16
	657	err4; lvx vr1,r4,r15
	658	vperm vr14,vr2,vr1,vr16
	659	err4; lvx vr0,r4,r16
	660	vperm vr15,vr1,vr0,vr16
	661	addi r4,r4,128
	662	err4; stvx vr8,r0,r3
	663	err4; stvx vr9,r3,r9
	664	err4; stvx vr10,r3,r10
	665	err4; stvx vr11,r3,r11
	666	err4; stvx vr12,r3,r12
	667	err4; stvx vr13,r3,r14
	668	err4; stvx vr14,r3,r15
	669	err4; stvx vr15,r3,r16
	670	addi r3,r3,128
	671	bdnz 8b
	672
c75df6f9 MN	673	ld r14,STK_REG(R14)(r1)
	674	ld r15,STK_REG(R15)(r1)
	675	ld r16,STK_REG(R16)(r1)
a66086b8 AB	676
	677	/* Up to 127B to go */
	678	clrldi r5,r5,(64-7)
	679	srdi r6,r5,4
	680	mtocrf 0x01,r6
	681
	682	bf cr7*4+1,9f
	683	err3; lvx vr3,r0,r4
	684	vperm vr8,vr0,vr3,vr16
	685	err3; lvx vr2,r4,r9
	686	vperm vr9,vr3,vr2,vr16
	687	err3; lvx vr1,r4,r10
	688	vperm vr10,vr2,vr1,vr16
	689	err3; lvx vr0,r4,r11
	690	vperm vr11,vr1,vr0,vr16
	691	addi r4,r4,64
	692	err3; stvx vr8,r0,r3
	693	err3; stvx vr9,r3,r9
	694	err3; stvx vr10,r3,r10
	695	err3; stvx vr11,r3,r11
	696	addi r3,r3,64
	697
	698	9: bf cr7*4+2,10f
	699	err3; lvx vr1,r0,r4
	700	vperm vr8,vr0,vr1,vr16
	701	err3; lvx vr0,r4,r9
	702	vperm vr9,vr1,vr0,vr16
	703	addi r4,r4,32
	704	err3; stvx vr8,r0,r3
	705	err3; stvx vr9,r3,r9
	706	addi r3,r3,32
	707
	708	10: bf cr7*4+3,11f
	709	err3; lvx vr1,r0,r4
	710	vperm vr8,vr0,vr1,vr16
	711	addi r4,r4,16
	712	err3; stvx vr8,r0,r3
	713	addi r3,r3,16
	714
	715	/* Up to 15B to go */
	716	11: clrldi r5,r5,(64-4)
	717	addi r4,r4,-16 /* Unwind the +16 load offset */
	718	mtocrf 0x01,r5
	719	bf cr7*4+0,12f
	720	err3; lwz r0,0(r4) /* Less chance of a reject with word ops */
	721	err3; lwz r6,4(r4)
	722	addi r4,r4,8
	723	err3; stw r0,0(r3)
	724	err3; stw r6,4(r3)
	725	addi r3,r3,8
	726
	727	12: bf cr7*4+1,13f
	728	err3; lwz r0,0(r4)
	729	addi r4,r4,4
	730	err3; stw r0,0(r3)
	731	addi r3,r3,4
	732
	733	13: bf cr7*4+2,14f
	734	err3; lhz r0,0(r4)
	735	addi r4,r4,2
	736	err3; sth r0,0(r3)
	737	addi r3,r3,2
	738
	739	14: bf cr7*4+3,15f
740	err3; lbz r0,0(r4)
741	err3; stb r0,0(r3)
742
743	15: addi r1,r1,STACKFRAMESIZE
6f7839e5	744	b .exit_vmx_usercopy /* tail call optimise */
a66086b8	745	#endif /* CONFiG_ALTIVEC */