Commit | Line | Data |
---|---|---|
0e1227d3 HY |
1 | /* |
2 | * Accelerated GHASH implementation with Intel PCLMULQDQ-NI | |
3 | * instructions. This file contains accelerated part of ghash | |
4 | * implementation. More information about PCLMULQDQ can be found at: | |
5 | * | |
6 | * http://software.intel.com/en-us/articles/carry-less-multiplication-and-its-usage-for-computing-the-gcm-mode/ | |
7 | * | |
8 | * Copyright (c) 2009 Intel Corp. | |
9 | * Author: Huang Ying <ying.huang@intel.com> | |
10 | * Vinodh Gopal | |
11 | * Erdinc Ozturk | |
12 | * Deniz Karakoyunlu | |
13 | * | |
14 | * This program is free software; you can redistribute it and/or modify it | |
15 | * under the terms of the GNU General Public License version 2 as published | |
16 | * by the Free Software Foundation. | |
17 | */ | |
18 | ||
19 | #include <linux/linkage.h> | |
564ec0ec | 20 | #include <asm/inst.h> |
8691ccd7 | 21 | #include <asm/frame.h> |
0e1227d3 | 22 | |
68ee8716 JK |
23 | .data |
24 | ||
0e1227d3 HY |
25 | .align 16 |
26 | .Lbswap_mask: | |
27 | .octa 0x000102030405060708090a0b0c0d0e0f | |
0e1227d3 HY |
28 | |
29 | #define DATA %xmm0 | |
30 | #define SHASH %xmm1 | |
31 | #define T1 %xmm2 | |
32 | #define T2 %xmm3 | |
33 | #define T3 %xmm4 | |
34 | #define BSWAP %xmm5 | |
35 | #define IN1 %xmm6 | |
36 | ||
37 | .text | |
38 | ||
39 | /* | |
40 | * __clmul_gf128mul_ble: internal ABI | |
41 | * input: | |
42 | * DATA: operand1 | |
43 | * SHASH: operand2, hash_key << 1 mod poly | |
44 | * output: | |
45 | * DATA: operand1 * operand2 mod poly | |
46 | * changed: | |
47 | * T1 | |
48 | * T2 | |
49 | * T3 | |
50 | */ | |
51 | __clmul_gf128mul_ble: | |
52 | movaps DATA, T1 | |
53 | pshufd $0b01001110, DATA, T2 | |
54 | pshufd $0b01001110, SHASH, T3 | |
55 | pxor DATA, T2 | |
56 | pxor SHASH, T3 | |
57 | ||
564ec0ec HY |
58 | PCLMULQDQ 0x00 SHASH DATA # DATA = a0 * b0 |
59 | PCLMULQDQ 0x11 SHASH T1 # T1 = a1 * b1 | |
60 | PCLMULQDQ 0x00 T3 T2 # T2 = (a1 + a0) * (b1 + b0) | |
0e1227d3 HY |
61 | pxor DATA, T2 |
62 | pxor T1, T2 # T2 = a0 * b1 + a1 * b0 | |
63 | ||
64 | movaps T2, T3 | |
65 | pslldq $8, T3 | |
66 | psrldq $8, T2 | |
67 | pxor T3, DATA | |
68 | pxor T2, T1 # <T1:DATA> is result of | |
69 | # carry-less multiplication | |
70 | ||
71 | # first phase of the reduction | |
72 | movaps DATA, T3 | |
73 | psllq $1, T3 | |
74 | pxor DATA, T3 | |
75 | psllq $5, T3 | |
76 | pxor DATA, T3 | |
77 | psllq $57, T3 | |
78 | movaps T3, T2 | |
79 | pslldq $8, T2 | |
80 | psrldq $8, T3 | |
81 | pxor T2, DATA | |
82 | pxor T3, T1 | |
83 | ||
84 | # second phase of the reduction | |
85 | movaps DATA, T2 | |
86 | psrlq $5, T2 | |
87 | pxor DATA, T2 | |
88 | psrlq $1, T2 | |
89 | pxor DATA, T2 | |
90 | psrlq $1, T2 | |
91 | pxor T2, T1 | |
92 | pxor T1, DATA | |
93 | ret | |
b05d3f37 | 94 | ENDPROC(__clmul_gf128mul_ble) |
0e1227d3 | 95 | |
0ea48146 | 96 | /* void clmul_ghash_mul(char *dst, const u128 *shash) */ |
0e1227d3 | 97 | ENTRY(clmul_ghash_mul) |
8691ccd7 | 98 | FRAME_BEGIN |
0e1227d3 HY |
99 | movups (%rdi), DATA |
100 | movups (%rsi), SHASH | |
101 | movaps .Lbswap_mask, BSWAP | |
564ec0ec | 102 | PSHUFB_XMM BSWAP DATA |
0e1227d3 | 103 | call __clmul_gf128mul_ble |
564ec0ec | 104 | PSHUFB_XMM BSWAP DATA |
0e1227d3 | 105 | movups DATA, (%rdi) |
8691ccd7 | 106 | FRAME_END |
0e1227d3 | 107 | ret |
b05d3f37 | 108 | ENDPROC(clmul_ghash_mul) |
0e1227d3 HY |
109 | |
110 | /* | |
111 | * void clmul_ghash_update(char *dst, const char *src, unsigned int srclen, | |
0ea48146 | 112 | * const u128 *shash); |
0e1227d3 HY |
113 | */ |
114 | ENTRY(clmul_ghash_update) | |
8691ccd7 | 115 | FRAME_BEGIN |
0e1227d3 HY |
116 | cmp $16, %rdx |
117 | jb .Lupdate_just_ret # check length | |
118 | movaps .Lbswap_mask, BSWAP | |
119 | movups (%rdi), DATA | |
120 | movups (%rcx), SHASH | |
564ec0ec | 121 | PSHUFB_XMM BSWAP DATA |
0e1227d3 HY |
122 | .align 4 |
123 | .Lupdate_loop: | |
124 | movups (%rsi), IN1 | |
564ec0ec | 125 | PSHUFB_XMM BSWAP IN1 |
0e1227d3 HY |
126 | pxor IN1, DATA |
127 | call __clmul_gf128mul_ble | |
128 | sub $16, %rdx | |
129 | add $16, %rsi | |
130 | cmp $16, %rdx | |
131 | jge .Lupdate_loop | |
564ec0ec | 132 | PSHUFB_XMM BSWAP DATA |
0e1227d3 HY |
133 | movups DATA, (%rdi) |
134 | .Lupdate_just_ret: | |
8691ccd7 | 135 | FRAME_END |
0e1227d3 | 136 | ret |
b05d3f37 | 137 | ENDPROC(clmul_ghash_update) |