projects / deliverable / linux.git / blame
| Commit | Line | Data |
|---|---|---|
| fdd23894 AB |
1 | /* |
| 2 | * Accelerated GHASH implementation with ARMv8 PMULL instructions. | |
| 3 | * | |
| 4 | * Copyright (C) 2014 Linaro Ltd. <ard.biesheuvel@linaro.org> | |
| 5 | * | |
| fdd23894 AB |
6 | * This program is free software; you can redistribute it and/or modify it |
| 7 | * under the terms of the GNU General Public License version 2 as published | |
| 8 | * by the Free Software Foundation. | |
| 9 | */ | |
| 10 | ||
| 11 | #include <linux/linkage.h> | |
| 12 | #include <asm/assembler.h> | |
| 13 | ||
| b913a640 AB |
14 | SHASH .req v0 |
| 15 | SHASH2 .req v1 | |
| fdd23894 AB |
16 | T1 .req v2 |
| 17 | T2 .req v3 | |
| b913a640 AB |
18 | MASK .req v4 |
| 19 | XL .req v5 | |
| 20 | XM .req v6 | |
| 21 | XH .req v7 | |
| 22 | IN1 .req v7 | |
| fdd23894 AB |
23 | |
| 24 | .text | |
| 25 | .arch armv8-a+crypto | |
| 26 | ||
| 27 | /* | |
| 28 | * void pmull_ghash_update(int blocks, u64 dg[], const char *src, | |
| 29 | * struct ghash_key const *k, const char *head) | |
| 30 | */ | |
| 31 | ENTRY(pmull_ghash_update) | |
| fdd23894 | 32 | ld1 {SHASH.16b}, [x3] |
| b913a640 AB |
33 | ld1 {XL.16b}, [x1] |
| 34 | movi MASK.16b, #0xe1 | |
| 35 | ext SHASH2.16b, SHASH.16b, SHASH.16b, #8 | |
| 36 | shl MASK.2d, MASK.2d, #57 | |
| 37 | eor SHASH2.16b, SHASH2.16b, SHASH.16b | |
| fdd23894 AB |
38 | |
| 39 | /* do the head block first, if supplied */ | |
| 40 | cbz x4, 0f | |
| b913a640 | 41 | ld1 {T1.2d}, [x4] |
| fdd23894 AB |
42 | b 1f |
| 43 | ||
| b913a640 | 44 | 0: ld1 {T1.2d}, [x2], #16 |
| fdd23894 | 45 | sub w0, w0, #1 |
| fdd23894 | 46 | |
| b913a640 AB |
47 | 1: /* multiply XL by SHASH in GF(2^128) */ |
| 48 | CPU_LE( rev64 T1.16b, T1.16b ) | |
| fdd23894 | 49 | |
| b913a640 AB |
50 | ext T2.16b, XL.16b, XL.16b, #8 |
| 51 | ext IN1.16b, T1.16b, T1.16b, #8 | |
| 52 | eor T1.16b, T1.16b, T2.16b | |
| 53 | eor XL.16b, XL.16b, IN1.16b | |
| fdd23894 | 54 | |
| b913a640 AB |
55 | pmull2 XH.1q, SHASH.2d, XL.2d // a1 * b1 |
| 56 | eor T1.16b, T1.16b, XL.16b | |
| 57 | pmull XL.1q, SHASH.1d, XL.1d // a0 * b0 | |
| 58 | pmull XM.1q, SHASH2.1d, T1.1d // (a1 + a0)(b1 + b0) | |
| fdd23894 | 59 | |
| b913a640 AB |
60 | ext T1.16b, XL.16b, XH.16b, #8 |
| 61 | eor T2.16b, XL.16b, XH.16b | |
| 62 | eor XM.16b, XM.16b, T1.16b | |
| 63 | eor XM.16b, XM.16b, T2.16b | |
| 64 | pmull T2.1q, XL.1d, MASK.1d | |
| fdd23894 | 65 | |
| b913a640 AB |
66 | mov XH.d[0], XM.d[1] |
| 67 | mov XM.d[1], XL.d[0] | |
| 68 | ||
| 69 | eor XL.16b, XM.16b, T2.16b | |
| 70 | ext T2.16b, XL.16b, XL.16b, #8 | |
| 71 | pmull XL.1q, XL.1d, MASK.1d | |
| 72 | eor T2.16b, T2.16b, XH.16b | |
| 73 | eor XL.16b, XL.16b, T2.16b | |
| fdd23894 AB |
74 | |
| 75 | cbnz w0, 0b | |
| 76 | ||
| b913a640 | 77 | st1 {XL.16b}, [x1] |
| fdd23894 AB |
78 | ret |
| 79 | ENDPROC(pmull_ghash_update) |