2 * This file is subject to the terms and conditions of the GNU General Public
3 * License. See the file "COPYING" in the main directory of this archive
6 * Quick'n'dirty IP checksum ...
8 * Copyright (C) 1998, 1999 Ralf Baechle
9 * Copyright (C) 1999 Silicon Graphics, Inc.
10 * Copyright (C) 2007 Maciej W. Rozycki
11 * Copyright (C) 2014 Imagination Technologies Ltd.
13 #include <linux/errno.h>
15 #include <asm/asm-offsets.h>
16 #include <asm/regdef.h>
20 * As we are sharing code base with the mips32 tree (which use the o32 ABI
21 * register definitions). We need to redefine the register definitions from
22 * the n64 ABI register naming to the o32 ABI register naming.
54 #endif /* USE_DOUBLE */
56 #define UNIT(unit) ((unit)*NBYTES)
58 #define ADDC(sum,reg) \
63 #define ADDC32(sum,reg) \
68 #define CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3) \
69 LOAD _t0, (offset + UNIT(0))(src); \
70 LOAD _t1, (offset + UNIT(1))(src); \
71 LOAD _t2, (offset + UNIT(2))(src); \
72 LOAD _t3, (offset + UNIT(3))(src); \
79 #define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
80 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3)
82 #define CSUM_BIGCHUNK(src, offset, sum, _t0, _t1, _t2, _t3) \
83 CSUM_BIGCHUNK1(src, offset, sum, _t0, _t1, _t2, _t3); \
84 CSUM_BIGCHUNK1(src, offset + 0x10, sum, _t0, _t1, _t2, _t3)
89 * a1: length of the area to checksum
90 * a2: partial checksum
104 bnez t8, .Lsmall_csumcpy /* < 8 bytes to copy */
107 andi t7, src, 0x1 /* odd buffer? */
110 beqz t7, .Lword_align
114 LONG_SUBU a1, a1, 0x1
119 PTR_ADDU src, src, 0x1
123 beqz t8, .Ldword_align
127 LONG_SUBU a1, a1, 0x2
130 PTR_ADDU src, src, 0x2
133 bnez t8, .Ldo_end_words
137 beqz t8, .Lqword_align
141 LONG_SUBU a1, a1, 0x4
143 PTR_ADDU src, src, 0x4
147 beqz t8, .Loword_align
152 LONG_SUBU a1, a1, 0x8
157 LONG_SUBU a1, a1, 0x8
161 PTR_ADDU src, src, 0x8
165 beqz t8, .Lbegin_movement
174 CSUM_BIGCHUNK1(src, 0x00, sum, t0, t1, t3, t4)
176 LONG_SUBU a1, a1, 0x10
177 PTR_ADDU src, src, 0x10
185 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
186 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
187 CSUM_BIGCHUNK(src, 0x40, sum, t0, t1, t3, t4)
188 CSUM_BIGCHUNK(src, 0x60, sum, t0, t1, t3, t4)
189 LONG_SUBU t8, t8, 0x01
190 .set reorder /* DADDI_WAR */
191 PTR_ADDU src, src, 0x80
192 bnez t8, .Lmove_128bytes
200 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
201 CSUM_BIGCHUNK(src, 0x20, sum, t0, t1, t3, t4)
202 PTR_ADDU src, src, 0x40
205 beqz t2, .Ldo_end_words
209 CSUM_BIGCHUNK(src, 0x00, sum, t0, t1, t3, t4)
211 PTR_ADDU src, src, 0x20
214 beqz t8, .Lsmall_csumcpy
220 LONG_SUBU t8, t8, 0x1
222 .set reorder /* DADDI_WAR */
223 PTR_ADDU src, src, 0x4
227 /* unknown src alignment and < 8 bytes to go */
235 /* Still a full word to go */
239 dsll t1, t1, 32 /* clear lower 32bit */
247 /* Still a halfword to go */
273 /* odd buffer alignment? */
274 #ifdef CONFIG_CPU_MIPSR2
278 beqz t7, 1f /* odd buffer alignment? */
289 /* Add the passed partial csum. */
297 * checksum and copy routines based on memcpy.S
299 * csum_partial_copy_nocheck(src, dst, len, sum)
300 * __csum_partial_copy_kernel(src, dst, len, sum, errp)
302 * See "Spec" in memcpy.S for details. Unlike __copy_user, all
303 * function in this file use the standard calling convention.
315 * The exception handler for loads requires that:
316 * 1- AT contain the address of the byte just past the end of the source
318 * 2- src_entry <= src < AT, and
319 * 3- (dst - src) == (dst_entry - src_entry),
320 * The _entry suffix denotes values when __copy_user was called.
322 * (1) is set up up by __csum_partial_copy_from_user and maintained by
323 * not writing AT in __csum_partial_copy
324 * (2) is met by incrementing src by the number of bytes copied
325 * (3) is met by not doing loads between a pair of increments of dst and src
327 * The exception handlers for stores stores -EFAULT to errptr and return.
328 * These handlers do not need to overwrite any data.
331 /* Instruction type */
334 #define LEGACY_MODE 1
340 * Wrapper to add an entry in the exception table
341 * in case the insn causes a memory exception.
343 * insn : Load/store instruction
344 * type : Instruction type
347 * handler : Exception handler
349 #define EXC(insn, type, reg, addr, handler) \
350 .if \mode == LEGACY_MODE; \
352 .section __ex_table,"a"; \
361 #define LOADK ld /* No exception */
362 #define LOAD(reg, addr, handler) EXC(ld, LD_INSN, reg, addr, handler)
363 #define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
364 #define LOADL(reg, addr, handler) EXC(ldl, LD_INSN, reg, addr, handler)
365 #define LOADR(reg, addr, handler) EXC(ldr, LD_INSN, reg, addr, handler)
366 #define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
367 #define STOREL(reg, addr, handler) EXC(sdl, ST_INSN, reg, addr, handler)
368 #define STORER(reg, addr, handler) EXC(sdr, ST_INSN, reg, addr, handler)
369 #define STORE(reg, addr, handler) EXC(sd, ST_INSN, reg, addr, handler)
381 #define LOADK lw /* No exception */
382 #define LOAD(reg, addr, handler) EXC(lw, LD_INSN, reg, addr, handler)
383 #define LOADBU(reg, addr, handler) EXC(lbu, LD_INSN, reg, addr, handler)
384 #define LOADL(reg, addr, handler) EXC(lwl, LD_INSN, reg, addr, handler)
385 #define LOADR(reg, addr, handler) EXC(lwr, LD_INSN, reg, addr, handler)
386 #define STOREB(reg, addr, handler) EXC(sb, ST_INSN, reg, addr, handler)
387 #define STOREL(reg, addr, handler) EXC(swl, ST_INSN, reg, addr, handler)
388 #define STORER(reg, addr, handler) EXC(swr, ST_INSN, reg, addr, handler)
389 #define STORE(reg, addr, handler) EXC(sw, ST_INSN, reg, addr, handler)
399 #endif /* USE_DOUBLE */
401 #ifdef CONFIG_CPU_LITTLE_ENDIAN
402 #define LDFIRST LOADR
404 #define STFIRST STORER
405 #define STREST STOREL
406 #define SHIFT_DISCARD SLLV
407 #define SHIFT_DISCARD_REVERT SRLV
409 #define LDFIRST LOADL
411 #define STFIRST STOREL
412 #define STREST STORER
413 #define SHIFT_DISCARD SRLV
414 #define SHIFT_DISCARD_REVERT SLLV
417 #define FIRST(unit) ((unit)*NBYTES)
418 #define REST(unit) (FIRST(unit)+NBYTES-1)
420 #define ADDRMASK (NBYTES-1)
422 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
428 .macro __BUILD_CSUM_PARTIAL_COPY_USER mode, from, to, __nocheck
430 PTR_ADDU AT, src, len /* See (1) above. */
431 /* initialize __nocheck if this the first time we execute this
440 FEXPORT(csum_partial_copy_nocheck)
445 * Note: dst & src may be unaligned, len may be 0
449 * The "issue break"s below are very approximate.
450 * Issue delays for dcache fills will perturb the schedule, as will
451 * load queue full replay traps, etc.
453 * If len < NBYTES use byte operations.
456 and t1, dst, ADDRMASK
457 bnez t2, .Lcopy_bytes_checklen\@
458 and t0, src, ADDRMASK
459 andi odd, dst, 0x1 /* odd buffer? */
460 bnez t1, .Ldst_unaligned\@
462 bnez t0, .Lsrc_unaligned_dst_aligned\@
464 * use delay slot for fall-through
465 * src and dst are aligned; need to compute rem
468 SRL t0, len, LOG_NBYTES+3 # +3 for 8 units/iter
469 beqz t0, .Lcleanup_both_aligned\@ # len < 8*NBYTES
471 SUB len, 8*NBYTES # subtract here for bgez loop
474 LOAD(t0, UNIT(0)(src), .Ll_exc\@)
475 LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
476 LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
477 LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
478 LOAD(t4, UNIT(4)(src), .Ll_exc_copy\@)
479 LOAD(t5, UNIT(5)(src), .Ll_exc_copy\@)
480 LOAD(t6, UNIT(6)(src), .Ll_exc_copy\@)
481 LOAD(t7, UNIT(7)(src), .Ll_exc_copy\@)
482 SUB len, len, 8*NBYTES
483 ADD src, src, 8*NBYTES
484 STORE(t0, UNIT(0)(dst), .Ls_exc\@)
486 STORE(t1, UNIT(1)(dst), .Ls_exc\@)
488 STORE(t2, UNIT(2)(dst), .Ls_exc\@)
490 STORE(t3, UNIT(3)(dst), .Ls_exc\@)
492 STORE(t4, UNIT(4)(dst), .Ls_exc\@)
494 STORE(t5, UNIT(5)(dst), .Ls_exc\@)
496 STORE(t6, UNIT(6)(dst), .Ls_exc\@)
498 STORE(t7, UNIT(7)(dst), .Ls_exc\@)
500 .set reorder /* DADDI_WAR */
501 ADD dst, dst, 8*NBYTES
504 ADD len, 8*NBYTES # revert len (see above)
507 * len == the number of bytes left to copy < 8*NBYTES
509 .Lcleanup_both_aligned\@:
512 sltu t0, len, 4*NBYTES
513 bnez t0, .Lless_than_4units\@
514 and rem, len, (NBYTES-1) # rem = len % NBYTES
518 LOAD(t0, UNIT(0)(src), .Ll_exc\@)
519 LOAD(t1, UNIT(1)(src), .Ll_exc_copy\@)
520 LOAD(t2, UNIT(2)(src), .Ll_exc_copy\@)
521 LOAD(t3, UNIT(3)(src), .Ll_exc_copy\@)
522 SUB len, len, 4*NBYTES
523 ADD src, src, 4*NBYTES
524 STORE(t0, UNIT(0)(dst), .Ls_exc\@)
526 STORE(t1, UNIT(1)(dst), .Ls_exc\@)
528 STORE(t2, UNIT(2)(dst), .Ls_exc\@)
530 STORE(t3, UNIT(3)(dst), .Ls_exc\@)
532 .set reorder /* DADDI_WAR */
533 ADD dst, dst, 4*NBYTES
536 .Lless_than_4units\@:
540 beq rem, len, .Lcopy_bytes\@
543 LOAD(t0, 0(src), .Ll_exc\@)
546 STORE(t0, 0(dst), .Ls_exc\@)
548 .set reorder /* DADDI_WAR */
554 * src and dst are aligned, need to copy rem bytes (rem < NBYTES)
555 * A loop would do only a byte at a time with possible branch
556 * mispredicts. Can't do an explicit LOAD dst,mask,or,STORE
557 * because can't assume read-access to dst. Instead, use
558 * STREST dst, which doesn't require read access to dst.
560 * This code should perform better than a simple loop on modern,
561 * wide-issue mips processors because the code has fewer branches and
562 * more instruction-level parallelism.
566 ADD t1, dst, len # t1 is just past last byte of dst
568 SLL rem, len, 3 # rem = number of bits to keep
569 LOAD(t0, 0(src), .Ll_exc\@)
570 SUB bits, bits, rem # bits = number of bits to discard
571 SHIFT_DISCARD t0, t0, bits
572 STREST(t0, -1(t1), .Ls_exc\@)
573 SHIFT_DISCARD_REVERT t0, t0, bits
581 * t0 = src & ADDRMASK
582 * t1 = dst & ADDRMASK; T1 > 0
585 * Copy enough bytes to align dst
586 * Set match = (src and dst have same alignment)
589 LDFIRST(t3, FIRST(0)(src), .Ll_exc\@)
591 LDREST(t3, REST(0)(src), .Ll_exc_copy\@)
592 SUB t2, t2, t1 # t2 = number of bytes copied
594 STFIRST(t3, FIRST(0)(dst), .Ls_exc\@)
595 SLL t4, t1, 3 # t4 = number of bits to discard
596 SHIFT_DISCARD t3, t3, t4
597 /* no SHIFT_DISCARD_REVERT to handle odd buffer properly */
599 beq len, t2, .Ldone\@
602 beqz match, .Lboth_aligned\@
605 .Lsrc_unaligned_dst_aligned\@:
606 SRL t0, len, LOG_NBYTES+2 # +2 for 4 units/iter
607 beqz t0, .Lcleanup_src_unaligned\@
608 and rem, len, (4*NBYTES-1) # rem = len % 4*NBYTES
611 * Avoid consecutive LD*'s to the same register since some mips
612 * implementations can't issue them in the same cycle.
613 * It's OK to load FIRST(N+1) before REST(N) because the two addresses
614 * are to the same unit (unless src is aligned, but it's not).
616 LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
617 LDFIRST(t1, FIRST(1)(src), .Ll_exc_copy\@)
618 SUB len, len, 4*NBYTES
619 LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
620 LDREST(t1, REST(1)(src), .Ll_exc_copy\@)
621 LDFIRST(t2, FIRST(2)(src), .Ll_exc_copy\@)
622 LDFIRST(t3, FIRST(3)(src), .Ll_exc_copy\@)
623 LDREST(t2, REST(2)(src), .Ll_exc_copy\@)
624 LDREST(t3, REST(3)(src), .Ll_exc_copy\@)
625 ADD src, src, 4*NBYTES
626 #ifdef CONFIG_CPU_SB1
627 nop # improves slotting
629 STORE(t0, UNIT(0)(dst), .Ls_exc\@)
631 STORE(t1, UNIT(1)(dst), .Ls_exc\@)
633 STORE(t2, UNIT(2)(dst), .Ls_exc\@)
635 STORE(t3, UNIT(3)(dst), .Ls_exc\@)
637 .set reorder /* DADDI_WAR */
638 ADD dst, dst, 4*NBYTES
642 .Lcleanup_src_unaligned\@:
644 and rem, len, NBYTES-1 # rem = len % NBYTES
645 beq rem, len, .Lcopy_bytes\@
648 LDFIRST(t0, FIRST(0)(src), .Ll_exc\@)
649 LDREST(t0, REST(0)(src), .Ll_exc_copy\@)
652 STORE(t0, 0(dst), .Ls_exc\@)
654 .set reorder /* DADDI_WAR */
659 .Lcopy_bytes_checklen\@:
663 /* 0 < len < NBYTES */
664 #ifdef CONFIG_CPU_LITTLE_ENDIAN
665 #define SHIFT_START 0
668 #define SHIFT_START 8*(NBYTES-1)
671 move t2, zero # partial word
672 li t3, SHIFT_START # shift
673 /* use .Ll_exc_copy here to return correct sum on fault */
674 #define COPY_BYTE(N) \
675 LOADBU(t0, N(src), .Ll_exc_copy\@); \
677 STOREB(t0, N(dst), .Ls_exc\@); \
679 addu t3, SHIFT_INC; \
680 beqz len, .Lcopy_bytes_done\@; \
691 LOADBU(t0, NBYTES-2(src), .Ll_exc_copy\@)
693 STOREB(t0, NBYTES-2(dst), .Ls_exc\@)
708 #ifdef CONFIG_CPU_MIPSR2
712 beqz odd, 1f /* odd buffer alignment? */
729 * Copy bytes from src until faulting load address (or until a
732 * When reached by a faulting LDFIRST/LDREST, THREAD_BUADDR($28)
733 * may be more than a byte beyond the last address.
734 * Hence, the lb below may get an exception.
736 * Assumes src < THREAD_BUADDR($28)
738 LOADK t0, TI_TASK($28)
740 LOADK t0, THREAD_BUADDR(t0)
742 LOADBU(t1, 0(src), .Ll_exc\@)
744 sb t1, 0(dst) # can't fault -- we're copy_from_user
748 .set reorder /* DADDI_WAR */
753 LOADK t0, TI_TASK($28)
755 LOADK t0, THREAD_BUADDR(t0) # t0 is just past last good address
757 SUB len, AT, t0 # len number of uncopied bytes
759 * Here's where we rely on src and dst being incremented in tandem,
761 * dst += (fault addr - src) to put dst at first byte to clear
763 ADD dst, t0 # compute start address in a1
766 * Clear len bytes starting at dst. Can't call __bzero because it
767 * might modify len. An inefficient loop for these rare times...
769 .set reorder /* DADDI_WAR */
777 #ifndef CONFIG_CPU_DADDI_WORKAROUNDS
790 li v0, -1 /* invalid checksum */
797 LEAF(__csum_partial_copy_kernel)
798 FEXPORT(__csum_partial_copy_to_user)
799 FEXPORT(__csum_partial_copy_from_user)
800 __BUILD_CSUM_PARTIAL_COPY_USER LEGACY_MODE USEROP USEROP 1
801 END(__csum_partial_copy_kernel)