arch/arm/lib/uaccess.S

   1 /*
   2  *  linux/arch/arm/lib/uaccess.S
   3  *
   4  *  Copyright (C) 1995, 1996,1997,1998 Russell King
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 as
   8  * published by the Free Software Foundation.
   9  *
  10  *  Routines to block copy data to/from user memory
  11  *   These are highly optimised both for the 4k page size
  12  *   and for various alignments.
  13  */
  14 #include <linux/linkage.h>
  15 #include <asm/assembler.h>
  16 #include <asm/errno.h>
  17
  18                 .text
  19
  20 #define PAGE_SHIFT 12
  21
  22 /* Prototype: int __copy_to_user(void *to, const char *from, size_t n)
  23  * Purpose  : copy a block to user memory from kernel memory
  24  * Params   : to   - user memory
  25  *          : from - kernel memory
  26  *          : n    - number of bytes to copy
  27  * Returns  : Number of bytes NOT copied.
  28  */
  29
  30 .Lc2u_dest_not_aligned:
  31                 rsb     ip, ip, #4
  32                 cmp     ip, #2
  33                 ldrb    r3, [r1], #1
  34 USER(           strbt   r3, [r0], #1)                   @ May fault
  35                 ldrgeb  r3, [r1], #1
  36 USER(           strgebt r3, [r0], #1)                   @ May fault
  37                 ldrgtb  r3, [r1], #1
  38 USER(           strgtbt r3, [r0], #1)                   @ May fault
  39                 sub     r2, r2, ip
  40                 b       .Lc2u_dest_aligned
  41
  42 ENTRY(__copy_to_user)
  43                 stmfd   sp!, {r2, r4 - r7, lr}
  44                 cmp     r2, #4
  45                 blt     .Lc2u_not_enough
  46                 ands    ip, r0, #3
  47                 bne     .Lc2u_dest_not_aligned
  48 .Lc2u_dest_aligned:
  49
  50                 ands    ip, r1, #3
  51                 bne     .Lc2u_src_not_aligned
  52 /*
  53  * Seeing as there has to be at least 8 bytes to copy, we can
  54  * copy one word, and force a user-mode page fault...
  55  */
  56
  57 .Lc2u_0fupi:    subs    r2, r2, #4
  58                 addmi   ip, r2, #4
  59                 bmi     .Lc2u_0nowords
  60                 ldr     r3, [r1], #4
  61 USER(           strt    r3, [r0], #4)                   @ May fault
  62                 mov     ip, r0, lsl #32 - PAGE_SHIFT    @ On each page, use a ld/st??t instruction
  63                 rsb     ip, ip, #0
  64                 movs    ip, ip, lsr #32 - PAGE_SHIFT
  65                 beq     .Lc2u_0fupi
  66 /*
  67  * ip = max no. of bytes to copy before needing another "strt" insn
  68  */
  69                 cmp     r2, ip
  70                 movlt   ip, r2
  71                 sub     r2, r2, ip
  72                 subs    ip, ip, #32
  73                 blt     .Lc2u_0rem8lp
  74
  75 .Lc2u_0cpy8lp:  ldmia   r1!, {r3 - r6}
  76                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
  77                 ldmia   r1!, {r3 - r6}
  78                 subs    ip, ip, #32
  79                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
  80                 bpl     .Lc2u_0cpy8lp
  81
  82 .Lc2u_0rem8lp:  cmn     ip, #16
  83                 ldmgeia r1!, {r3 - r6}
  84                 stmgeia r0!, {r3 - r6}                  @ Shouldnt fault
  85                 tst     ip, #8
  86                 ldmneia r1!, {r3 - r4}
  87                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
  88                 tst     ip, #4
  89                 ldrne   r3, [r1], #4
  90                 strnet  r3, [r0], #4                    @ Shouldnt fault
  91                 ands    ip, ip, #3
  92                 beq     .Lc2u_0fupi
  93 .Lc2u_0nowords: teq     ip, #0
  94                 beq     .Lc2u_finished
  95 .Lc2u_nowords:  cmp     ip, #2
  96                 ldrb    r3, [r1], #1
  97 USER(           strbt   r3, [r0], #1)                   @ May fault
  98                 ldrgeb  r3, [r1], #1
  99 USER(           strgebt r3, [r0], #1)                   @ May fault
 100                 ldrgtb  r3, [r1], #1
 101 USER(           strgtbt r3, [r0], #1)                   @ May fault
 102                 b       .Lc2u_finished
 103
 104 .Lc2u_not_enough:
 105                 movs    ip, r2
 106                 bne     .Lc2u_nowords
 107 .Lc2u_finished: mov     r0, #0
 108                 ldmfd   sp!, {r2, r4 - r7, pc}
 109
 110 .Lc2u_src_not_aligned:
 111                 bic     r1, r1, #3
 112                 ldr     r7, [r1], #4
 113                 cmp     ip, #2
 114                 bgt     .Lc2u_3fupi
 115                 beq     .Lc2u_2fupi
 116 .Lc2u_1fupi:    subs    r2, r2, #4
 117                 addmi   ip, r2, #4
 118                 bmi     .Lc2u_1nowords
 119                 mov     r3, r7, pull #8
 120                 ldr     r7, [r1], #4
 121                 orr     r3, r3, r7, push #24
 122 USER(           strt    r3, [r0], #4)                   @ May fault
 123                 mov     ip, r0, lsl #32 - PAGE_SHIFT
 124                 rsb     ip, ip, #0
 125                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 126                 beq     .Lc2u_1fupi
 127                 cmp     r2, ip
 128                 movlt   ip, r2
 129                 sub     r2, r2, ip
 130                 subs    ip, ip, #16
 131                 blt     .Lc2u_1rem8lp
 132
 133 .Lc2u_1cpy8lp:  mov     r3, r7, pull #8
 134                 ldmia   r1!, {r4 - r7}
 135                 subs    ip, ip, #16
 136                 orr     r3, r3, r4, push #24
 137                 mov     r4, r4, pull #8
 138                 orr     r4, r4, r5, push #24
 139                 mov     r5, r5, pull #8
 140                 orr     r5, r5, r6, push #24
 141                 mov     r6, r6, pull #8
 142                 orr     r6, r6, r7, push #24
 143                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
 144                 bpl     .Lc2u_1cpy8lp
 145
 146 .Lc2u_1rem8lp:  tst     ip, #8
 147                 movne   r3, r7, pull #8
 148                 ldmneia r1!, {r4, r7}
 149                 orrne   r3, r3, r4, push #24
 150                 movne   r4, r4, pull #8
 151                 orrne   r4, r4, r7, push #24
 152                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 153                 tst     ip, #4
 154                 movne   r3, r7, pull #8
 155                 ldrne   r7, [r1], #4
 156                 orrne   r3, r3, r7, push #24
 157                 strnet  r3, [r0], #4                    @ Shouldnt fault
 158                 ands    ip, ip, #3
 159                 beq     .Lc2u_1fupi
 160 .Lc2u_1nowords: mov     r3, r7, get_byte_1
 161                 teq     ip, #0
 162                 beq     .Lc2u_finished
 163                 cmp     ip, #2
 164 USER(           strbt   r3, [r0], #1)                   @ May fault
 165                 movge   r3, r7, get_byte_2
 166 USER(           strgebt r3, [r0], #1)                   @ May fault
 167                 movgt   r3, r7, get_byte_3
 168 USER(           strgtbt r3, [r0], #1)                   @ May fault
 169                 b       .Lc2u_finished
 170
 171 .Lc2u_2fupi:    subs    r2, r2, #4
 172                 addmi   ip, r2, #4
 173                 bmi     .Lc2u_2nowords
 174                 mov     r3, r7, pull #16
 175                 ldr     r7, [r1], #4
 176                 orr     r3, r3, r7, push #16
 177 USER(           strt    r3, [r0], #4)                   @ May fault
 178                 mov     ip, r0, lsl #32 - PAGE_SHIFT
 179                 rsb     ip, ip, #0
 180                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 181                 beq     .Lc2u_2fupi
 182                 cmp     r2, ip
 183                 movlt   ip, r2
 184                 sub     r2, r2, ip
 185                 subs    ip, ip, #16
 186                 blt     .Lc2u_2rem8lp
 187
 188 .Lc2u_2cpy8lp:  mov     r3, r7, pull #16
 189                 ldmia   r1!, {r4 - r7}
 190                 subs    ip, ip, #16
 191                 orr     r3, r3, r4, push #16
 192                 mov     r4, r4, pull #16
 193                 orr     r4, r4, r5, push #16
 194                 mov     r5, r5, pull #16
 195                 orr     r5, r5, r6, push #16
 196                 mov     r6, r6, pull #16
 197                 orr     r6, r6, r7, push #16
 198                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
 199                 bpl     .Lc2u_2cpy8lp
 200
 201 .Lc2u_2rem8lp:  tst     ip, #8
 202                 movne   r3, r7, pull #16
 203                 ldmneia r1!, {r4, r7}
 204                 orrne   r3, r3, r4, push #16
 205                 movne   r4, r4, pull #16
 206                 orrne   r4, r4, r7, push #16
 207                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 208                 tst     ip, #4
 209                 movne   r3, r7, pull #16
 210                 ldrne   r7, [r1], #4
 211                 orrne   r3, r3, r7, push #16
 212                 strnet  r3, [r0], #4                    @ Shouldnt fault
 213                 ands    ip, ip, #3
 214                 beq     .Lc2u_2fupi
 215 .Lc2u_2nowords: mov     r3, r7, get_byte_2
 216                 teq     ip, #0
 217                 beq     .Lc2u_finished
 218                 cmp     ip, #2
 219 USER(           strbt   r3, [r0], #1)                   @ May fault
 220                 movge   r3, r7, get_byte_3
 221 USER(           strgebt r3, [r0], #1)                   @ May fault
 222                 ldrgtb  r3, [r1], #0
 223 USER(           strgtbt r3, [r0], #1)                   @ May fault
 224                 b       .Lc2u_finished
 225
 226 .Lc2u_3fupi:    subs    r2, r2, #4
 227                 addmi   ip, r2, #4
 228                 bmi     .Lc2u_3nowords
 229                 mov     r3, r7, pull #24
 230                 ldr     r7, [r1], #4
 231                 orr     r3, r3, r7, push #8
 232 USER(           strt    r3, [r0], #4)                   @ May fault
 233                 mov     ip, r0, lsl #32 - PAGE_SHIFT
 234                 rsb     ip, ip, #0
 235                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 236                 beq     .Lc2u_3fupi
 237                 cmp     r2, ip
 238                 movlt   ip, r2
 239                 sub     r2, r2, ip
 240                 subs    ip, ip, #16
 241                 blt     .Lc2u_3rem8lp
 242
 243 .Lc2u_3cpy8lp:  mov     r3, r7, pull #24
 244                 ldmia   r1!, {r4 - r7}
 245                 subs    ip, ip, #16
 246                 orr     r3, r3, r4, push #8
 247                 mov     r4, r4, pull #24
 248                 orr     r4, r4, r5, push #8
 249                 mov     r5, r5, pull #24
 250                 orr     r5, r5, r6, push #8
 251                 mov     r6, r6, pull #24
 252                 orr     r6, r6, r7, push #8
 253                 stmia   r0!, {r3 - r6}                  @ Shouldnt fault
 254                 bpl     .Lc2u_3cpy8lp
 255
 256 .Lc2u_3rem8lp:  tst     ip, #8
 257                 movne   r3, r7, pull #24
 258                 ldmneia r1!, {r4, r7}
 259                 orrne   r3, r3, r4, push #8
 260                 movne   r4, r4, pull #24
 261                 orrne   r4, r4, r7, push #8
 262                 stmneia r0!, {r3 - r4}                  @ Shouldnt fault
 263                 tst     ip, #4
 264                 movne   r3, r7, pull #24
 265                 ldrne   r7, [r1], #4
 266                 orrne   r3, r3, r7, push #8
 267                 strnet  r3, [r0], #4                    @ Shouldnt fault
 268                 ands    ip, ip, #3
 269                 beq     .Lc2u_3fupi
 270 .Lc2u_3nowords: mov     r3, r7, get_byte_3
 271                 teq     ip, #0
 272                 beq     .Lc2u_finished
 273                 cmp     ip, #2
 274 USER(           strbt   r3, [r0], #1)                   @ May fault
 275                 ldrgeb  r3, [r1], #1
 276 USER(           strgebt r3, [r0], #1)                   @ May fault
 277                 ldrgtb  r3, [r1], #0
 278 USER(           strgtbt r3, [r0], #1)                   @ May fault
 279                 b       .Lc2u_finished
 280 ENDPROC(__copy_to_user)
 281
 282                 .pushsection .fixup,"ax"
 283                 .align  0
 284 9001:           ldmfd   sp!, {r0, r4 - r7, pc}
 285                 .popsection
 286
 287 /* Prototype: unsigned long __copy_from_user(void *to,const void *from,unsigned long n);
 288  * Purpose  : copy a block from user memory to kernel memory
 289  * Params   : to   - kernel memory
 290  *          : from - user memory
 291  *          : n    - number of bytes to copy
 292  * Returns  : Number of bytes NOT copied.
 293  */
 294 .Lcfu_dest_not_aligned:
 295                 rsb     ip, ip, #4
 296                 cmp     ip, #2
 297 USER(           ldrbt   r3, [r1], #1)                   @ May fault
 298                 strb    r3, [r0], #1
 299 USER(           ldrgebt r3, [r1], #1)                   @ May fault
 300                 strgeb  r3, [r0], #1
 301 USER(           ldrgtbt r3, [r1], #1)                   @ May fault
 302                 strgtb  r3, [r0], #1
 303                 sub     r2, r2, ip
 304                 b       .Lcfu_dest_aligned
 305
 306 ENTRY(__copy_from_user)
 307                 stmfd   sp!, {r0, r2, r4 - r7, lr}
 308                 cmp     r2, #4
 309                 blt     .Lcfu_not_enough
 310                 ands    ip, r0, #3
 311                 bne     .Lcfu_dest_not_aligned
 312 .Lcfu_dest_aligned:
 313                 ands    ip, r1, #3
 314                 bne     .Lcfu_src_not_aligned
 315
 316 /*
 317  * Seeing as there has to be at least 8 bytes to copy, we can
 318  * copy one word, and force a user-mode page fault...
 319  */
 320
 321 .Lcfu_0fupi:    subs    r2, r2, #4
 322                 addmi   ip, r2, #4
 323                 bmi     .Lcfu_0nowords
 324 USER(           ldrt    r3, [r1], #4)
 325                 str     r3, [r0], #4
 326                 mov     ip, r1, lsl #32 - PAGE_SHIFT    @ On each page, use a ld/st??t instruction
 327                 rsb     ip, ip, #0
 328                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 329                 beq     .Lcfu_0fupi
 330 /*
 331  * ip = max no. of bytes to copy before needing another "strt" insn
 332  */
 333                 cmp     r2, ip
 334                 movlt   ip, r2
 335                 sub     r2, r2, ip
 336                 subs    ip, ip, #32
 337                 blt     .Lcfu_0rem8lp
 338
 339 .Lcfu_0cpy8lp:  ldmia   r1!, {r3 - r6}                  @ Shouldnt fault
 340                 stmia   r0!, {r3 - r6}
 341                 ldmia   r1!, {r3 - r6}                  @ Shouldnt fault
 342                 subs    ip, ip, #32
 343                 stmia   r0!, {r3 - r6}
 344                 bpl     .Lcfu_0cpy8lp
 345
 346 .Lcfu_0rem8lp:  cmn     ip, #16
 347                 ldmgeia r1!, {r3 - r6}                  @ Shouldnt fault
 348                 stmgeia r0!, {r3 - r6}
 349                 tst     ip, #8
 350                 ldmneia r1!, {r3 - r4}                  @ Shouldnt fault
 351                 stmneia r0!, {r3 - r4}
 352                 tst     ip, #4
 353                 ldrnet  r3, [r1], #4                    @ Shouldnt fault
 354                 strne   r3, [r0], #4
 355                 ands    ip, ip, #3
 356                 beq     .Lcfu_0fupi
 357 .Lcfu_0nowords: teq     ip, #0
 358                 beq     .Lcfu_finished
 359 .Lcfu_nowords:  cmp     ip, #2
 360 USER(           ldrbt   r3, [r1], #1)                   @ May fault
 361                 strb    r3, [r0], #1
 362 USER(           ldrgebt r3, [r1], #1)                   @ May fault
 363                 strgeb  r3, [r0], #1
 364 USER(           ldrgtbt r3, [r1], #1)                   @ May fault
 365                 strgtb  r3, [r0], #1
 366                 b       .Lcfu_finished
 367
 368 .Lcfu_not_enough:
 369                 movs    ip, r2
 370                 bne     .Lcfu_nowords
 371 .Lcfu_finished: mov     r0, #0
 372                 add     sp, sp, #8
 373                 ldmfd   sp!, {r4 - r7, pc}
 374
 375 .Lcfu_src_not_aligned:
 376                 bic     r1, r1, #3
 377 USER(           ldrt    r7, [r1], #4)                   @ May fault
 378                 cmp     ip, #2
 379                 bgt     .Lcfu_3fupi
 380                 beq     .Lcfu_2fupi
 381 .Lcfu_1fupi:    subs    r2, r2, #4
 382                 addmi   ip, r2, #4
 383                 bmi     .Lcfu_1nowords
 384                 mov     r3, r7, pull #8
 385 USER(           ldrt    r7, [r1], #4)                   @ May fault
 386                 orr     r3, r3, r7, push #24
 387                 str     r3, [r0], #4
 388                 mov     ip, r1, lsl #32 - PAGE_SHIFT
 389                 rsb     ip, ip, #0
 390                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 391                 beq     .Lcfu_1fupi
 392                 cmp     r2, ip
 393                 movlt   ip, r2
 394                 sub     r2, r2, ip
 395                 subs    ip, ip, #16
 396                 blt     .Lcfu_1rem8lp
 397
 398 .Lcfu_1cpy8lp:  mov     r3, r7, pull #8
 399                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
 400                 subs    ip, ip, #16
 401                 orr     r3, r3, r4, push #24
 402                 mov     r4, r4, pull #8
 403                 orr     r4, r4, r5, push #24
 404                 mov     r5, r5, pull #8
 405                 orr     r5, r5, r6, push #24
 406                 mov     r6, r6, pull #8
 407                 orr     r6, r6, r7, push #24
 408                 stmia   r0!, {r3 - r6}
 409                 bpl     .Lcfu_1cpy8lp
 410
 411 .Lcfu_1rem8lp:  tst     ip, #8
 412                 movne   r3, r7, pull #8
 413                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
 414                 orrne   r3, r3, r4, push #24
 415                 movne   r4, r4, pull #8
 416                 orrne   r4, r4, r7, push #24
 417                 stmneia r0!, {r3 - r4}
 418                 tst     ip, #4
 419                 movne   r3, r7, pull #8
 420 USER(           ldrnet  r7, [r1], #4)                   @ May fault
 421                 orrne   r3, r3, r7, push #24
 422                 strne   r3, [r0], #4
 423                 ands    ip, ip, #3
 424                 beq     .Lcfu_1fupi
 425 .Lcfu_1nowords: mov     r3, r7, get_byte_1
 426                 teq     ip, #0
 427                 beq     .Lcfu_finished
 428                 cmp     ip, #2
 429                 strb    r3, [r0], #1
 430                 movge   r3, r7, get_byte_2
 431                 strgeb  r3, [r0], #1
 432                 movgt   r3, r7, get_byte_3
 433                 strgtb  r3, [r0], #1
 434                 b       .Lcfu_finished
 435
 436 .Lcfu_2fupi:    subs    r2, r2, #4
 437                 addmi   ip, r2, #4
 438                 bmi     .Lcfu_2nowords
 439                 mov     r3, r7, pull #16
 440 USER(           ldrt    r7, [r1], #4)                   @ May fault
 441                 orr     r3, r3, r7, push #16
 442                 str     r3, [r0], #4
 443                 mov     ip, r1, lsl #32 - PAGE_SHIFT
 444                 rsb     ip, ip, #0
 445                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 446                 beq     .Lcfu_2fupi
 447                 cmp     r2, ip
 448                 movlt   ip, r2
 449                 sub     r2, r2, ip
 450                 subs    ip, ip, #16
 451                 blt     .Lcfu_2rem8lp
 452
 453
 454 .Lcfu_2cpy8lp:  mov     r3, r7, pull #16
 455                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
 456                 subs    ip, ip, #16
 457                 orr     r3, r3, r4, push #16
 458                 mov     r4, r4, pull #16
 459                 orr     r4, r4, r5, push #16
 460                 mov     r5, r5, pull #16
 461                 orr     r5, r5, r6, push #16
 462                 mov     r6, r6, pull #16
 463                 orr     r6, r6, r7, push #16
 464                 stmia   r0!, {r3 - r6}
 465                 bpl     .Lcfu_2cpy8lp
 466
 467 .Lcfu_2rem8lp:  tst     ip, #8
 468                 movne   r3, r7, pull #16
 469                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
 470                 orrne   r3, r3, r4, push #16
 471                 movne   r4, r4, pull #16
 472                 orrne   r4, r4, r7, push #16
 473                 stmneia r0!, {r3 - r4}
 474                 tst     ip, #4
 475                 movne   r3, r7, pull #16
 476 USER(           ldrnet  r7, [r1], #4)                   @ May fault
 477                 orrne   r3, r3, r7, push #16
 478                 strne   r3, [r0], #4
 479                 ands    ip, ip, #3
 480                 beq     .Lcfu_2fupi
 481 .Lcfu_2nowords: mov     r3, r7, get_byte_2
 482                 teq     ip, #0
 483                 beq     .Lcfu_finished
 484                 cmp     ip, #2
 485                 strb    r3, [r0], #1
 486                 movge   r3, r7, get_byte_3
 487                 strgeb  r3, [r0], #1
 488 USER(           ldrgtbt r3, [r1], #0)                   @ May fault
 489                 strgtb  r3, [r0], #1
 490                 b       .Lcfu_finished
 491
 492 .Lcfu_3fupi:    subs    r2, r2, #4
 493                 addmi   ip, r2, #4
 494                 bmi     .Lcfu_3nowords
 495                 mov     r3, r7, pull #24
 496 USER(           ldrt    r7, [r1], #4)                   @ May fault
 497                 orr     r3, r3, r7, push #8
 498                 str     r3, [r0], #4
 499                 mov     ip, r1, lsl #32 - PAGE_SHIFT
 500                 rsb     ip, ip, #0
 501                 movs    ip, ip, lsr #32 - PAGE_SHIFT
 502                 beq     .Lcfu_3fupi
 503                 cmp     r2, ip
 504                 movlt   ip, r2
 505                 sub     r2, r2, ip
 506                 subs    ip, ip, #16
 507                 blt     .Lcfu_3rem8lp
 508
 509 .Lcfu_3cpy8lp:  mov     r3, r7, pull #24
 510                 ldmia   r1!, {r4 - r7}                  @ Shouldnt fault
 511                 orr     r3, r3, r4, push #8
 512                 mov     r4, r4, pull #24
 513                 orr     r4, r4, r5, push #8
 514                 mov     r5, r5, pull #24
 515                 orr     r5, r5, r6, push #8
 516                 mov     r6, r6, pull #24
 517                 orr     r6, r6, r7, push #8
 518                 stmia   r0!, {r3 - r6}
 519                 subs    ip, ip, #16
 520                 bpl     .Lcfu_3cpy8lp
 521
 522 .Lcfu_3rem8lp:  tst     ip, #8
 523                 movne   r3, r7, pull #24
 524                 ldmneia r1!, {r4, r7}                   @ Shouldnt fault
 525                 orrne   r3, r3, r4, push #8
 526                 movne   r4, r4, pull #24
 527                 orrne   r4, r4, r7, push #8
 528                 stmneia r0!, {r3 - r4}
 529                 tst     ip, #4
 530                 movne   r3, r7, pull #24
 531 USER(           ldrnet  r7, [r1], #4)                   @ May fault
 532                 orrne   r3, r3, r7, push #8
 533                 strne   r3, [r0], #4
 534                 ands    ip, ip, #3
 535                 beq     .Lcfu_3fupi
 536 .Lcfu_3nowords: mov     r3, r7, get_byte_3
 537                 teq     ip, #0
 538                 beq     .Lcfu_finished
 539                 cmp     ip, #2
 540                 strb    r3, [r0], #1
 541 USER(           ldrgebt r3, [r1], #1)                   @ May fault
 542                 strgeb  r3, [r0], #1
 543 USER(           ldrgtbt r3, [r1], #1)                   @ May fault
 544                 strgtb  r3, [r0], #1
 545                 b       .Lcfu_finished
 546 ENDPROC(__copy_from_user)
 547
 548                 .pushsection .fixup,"ax"
 549                 .align  0
 550                 /*
 551                  * We took an exception.  r0 contains a pointer to
 552                  * the byte not copied.
 553                  */
 554 9001:           ldr     r2, [sp], #4                    @ void *to
 555                 sub     r2, r0, r2                      @ bytes copied
 556                 ldr     r1, [sp], #4                    @ unsigned long count
 557                 subs    r4, r1, r2                      @ bytes left to copy
 558                 movne   r1, r4
 559                 blne    __memzero
 560                 mov     r0, r4
 561                 ldmfd   sp!, {r4 - r7, pc}
 562                 .popsection
 563