Commit | Line | Data |
---|---|---|
87de4579 TC |
1 | /* |
2 | * Cryptographic API. | |
3 | * | |
4 | * Glue code for the SHA512 Secure Hash Algorithm assembler | |
5 | * implementation using supplemental SSE3 / AVX / AVX2 instructions. | |
6 | * | |
7 | * This file is based on sha512_generic.c | |
8 | * | |
9 | * Copyright (C) 2013 Intel Corporation | |
10 | * Author: Tim Chen <tim.c.chen@linux.intel.com> | |
11 | * | |
12 | * This program is free software; you can redistribute it and/or modify it | |
13 | * under the terms of the GNU General Public License as published by the Free | |
14 | * Software Foundation; either version 2 of the License, or (at your option) | |
15 | * any later version. | |
16 | * | |
17 | * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, | |
18 | * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF | |
19 | * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND | |
20 | * NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS | |
21 | * BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN | |
22 | * ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN | |
23 | * CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE | |
24 | * SOFTWARE. | |
25 | * | |
26 | */ | |
27 | ||
28 | #define pr_fmt(fmt) KBUILD_MODNAME ": " fmt | |
29 | ||
30 | #include <crypto/internal/hash.h> | |
31 | #include <linux/init.h> | |
32 | #include <linux/module.h> | |
33 | #include <linux/mm.h> | |
34 | #include <linux/cryptohash.h> | |
35 | #include <linux/types.h> | |
36 | #include <crypto/sha.h> | |
e68410eb | 37 | #include <crypto/sha512_base.h> |
df6b35f4 | 38 | #include <asm/fpu/api.h> |
87de4579 TC |
39 | |
40 | #include <linux/string.h> | |
41 | ||
e68410eb AB |
42 | asmlinkage void sha512_transform_ssse3(u64 *digest, const char *data, |
43 | u64 rounds); | |
87de4579 | 44 | #ifdef CONFIG_AS_AVX |
e68410eb | 45 | asmlinkage void sha512_transform_avx(u64 *digest, const char *data, |
87de4579 TC |
46 | u64 rounds); |
47 | #endif | |
48 | #ifdef CONFIG_AS_AVX2 | |
e68410eb AB |
49 | asmlinkage void sha512_transform_rorx(u64 *digest, const char *data, |
50 | u64 rounds); | |
87de4579 TC |
51 | #endif |
52 | ||
e68410eb | 53 | static void (*sha512_transform_asm)(u64 *, const char *, u64); |
87de4579 | 54 | |
e68410eb AB |
55 | static int sha512_ssse3_update(struct shash_desc *desc, const u8 *data, |
56 | unsigned int len) | |
87de4579 TC |
57 | { |
58 | struct sha512_state *sctx = shash_desc_ctx(desc); | |
87de4579 | 59 | |
e68410eb AB |
60 | if (!irq_fpu_usable() || |
61 | (sctx->count[0] % SHA512_BLOCK_SIZE) + len < SHA512_BLOCK_SIZE) | |
62 | return crypto_sha512_update(desc, data, len); | |
87de4579 | 63 | |
e68410eb AB |
64 | /* make sure casting to sha512_block_fn() is safe */ |
65 | BUILD_BUG_ON(offsetof(struct sha512_state, state) != 0); | |
87de4579 | 66 | |
e68410eb AB |
67 | kernel_fpu_begin(); |
68 | sha512_base_do_update(desc, data, len, | |
69 | (sha512_block_fn *)sha512_transform_asm); | |
70 | kernel_fpu_end(); | |
87de4579 TC |
71 | |
72 | return 0; | |
73 | } | |
74 | ||
e68410eb AB |
75 | static int sha512_ssse3_finup(struct shash_desc *desc, const u8 *data, |
76 | unsigned int len, u8 *out) | |
87de4579 | 77 | { |
e68410eb AB |
78 | if (!irq_fpu_usable()) |
79 | return crypto_sha512_finup(desc, data, len, out); | |
87de4579 | 80 | |
e68410eb AB |
81 | kernel_fpu_begin(); |
82 | if (len) | |
83 | sha512_base_do_update(desc, data, len, | |
84 | (sha512_block_fn *)sha512_transform_asm); | |
85 | sha512_base_do_finalize(desc, (sha512_block_fn *)sha512_transform_asm); | |
86 | kernel_fpu_end(); | |
87de4579 | 87 | |
e68410eb | 88 | return sha512_base_finish(desc, out); |
87de4579 TC |
89 | } |
90 | ||
87de4579 TC |
91 | /* Add padding and return the message digest. */ |
92 | static int sha512_ssse3_final(struct shash_desc *desc, u8 *out) | |
93 | { | |
e68410eb | 94 | return sha512_ssse3_finup(desc, NULL, 0, out); |
340991e3 JK |
95 | } |
96 | ||
97 | static struct shash_alg algs[] = { { | |
87de4579 | 98 | .digestsize = SHA512_DIGEST_SIZE, |
e68410eb | 99 | .init = sha512_base_init, |
87de4579 TC |
100 | .update = sha512_ssse3_update, |
101 | .final = sha512_ssse3_final, | |
e68410eb | 102 | .finup = sha512_ssse3_finup, |
87de4579 | 103 | .descsize = sizeof(struct sha512_state), |
87de4579 TC |
104 | .base = { |
105 | .cra_name = "sha512", | |
106 | .cra_driver_name = "sha512-ssse3", | |
107 | .cra_priority = 150, | |
108 | .cra_flags = CRYPTO_ALG_TYPE_SHASH, | |
109 | .cra_blocksize = SHA512_BLOCK_SIZE, | |
110 | .cra_module = THIS_MODULE, | |
111 | } | |
340991e3 JK |
112 | }, { |
113 | .digestsize = SHA384_DIGEST_SIZE, | |
e68410eb | 114 | .init = sha384_base_init, |
340991e3 | 115 | .update = sha512_ssse3_update, |
e68410eb AB |
116 | .final = sha512_ssse3_final, |
117 | .finup = sha512_ssse3_finup, | |
340991e3 | 118 | .descsize = sizeof(struct sha512_state), |
340991e3 JK |
119 | .base = { |
120 | .cra_name = "sha384", | |
121 | .cra_driver_name = "sha384-ssse3", | |
122 | .cra_priority = 150, | |
123 | .cra_flags = CRYPTO_ALG_TYPE_SHASH, | |
124 | .cra_blocksize = SHA384_BLOCK_SIZE, | |
125 | .cra_module = THIS_MODULE, | |
126 | } | |
127 | } }; | |
87de4579 TC |
128 | |
129 | #ifdef CONFIG_AS_AVX | |
130 | static bool __init avx_usable(void) | |
131 | { | |
c93b8a39 IM |
132 | if (!cpu_has_xfeatures(XSTATE_SSE | XSTATE_YMM, NULL)) { |
133 | if (cpu_has_avx) | |
134 | pr_info("AVX detected but unusable.\n"); | |
87de4579 TC |
135 | return false; |
136 | } | |
137 | ||
138 | return true; | |
139 | } | |
140 | #endif | |
141 | ||
142 | static int __init sha512_ssse3_mod_init(void) | |
143 | { | |
340991e3 | 144 | /* test for SSSE3 first */ |
87de4579 TC |
145 | if (cpu_has_ssse3) |
146 | sha512_transform_asm = sha512_transform_ssse3; | |
147 | ||
148 | #ifdef CONFIG_AS_AVX | |
149 | /* allow AVX to override SSSE3, it's a little faster */ | |
150 | if (avx_usable()) { | |
151 | #ifdef CONFIG_AS_AVX2 | |
152 | if (boot_cpu_has(X86_FEATURE_AVX2)) | |
153 | sha512_transform_asm = sha512_transform_rorx; | |
154 | else | |
155 | #endif | |
156 | sha512_transform_asm = sha512_transform_avx; | |
157 | } | |
158 | #endif | |
159 | ||
160 | if (sha512_transform_asm) { | |
161 | #ifdef CONFIG_AS_AVX | |
162 | if (sha512_transform_asm == sha512_transform_avx) | |
163 | pr_info("Using AVX optimized SHA-512 implementation\n"); | |
164 | #ifdef CONFIG_AS_AVX2 | |
165 | else if (sha512_transform_asm == sha512_transform_rorx) | |
166 | pr_info("Using AVX2 optimized SHA-512 implementation\n"); | |
167 | #endif | |
168 | else | |
169 | #endif | |
170 | pr_info("Using SSSE3 optimized SHA-512 implementation\n"); | |
340991e3 | 171 | return crypto_register_shashes(algs, ARRAY_SIZE(algs)); |
87de4579 TC |
172 | } |
173 | pr_info("Neither AVX nor SSSE3 is available/usable.\n"); | |
174 | ||
175 | return -ENODEV; | |
176 | } | |
177 | ||
178 | static void __exit sha512_ssse3_mod_fini(void) | |
179 | { | |
340991e3 | 180 | crypto_unregister_shashes(algs, ARRAY_SIZE(algs)); |
87de4579 TC |
181 | } |
182 | ||
183 | module_init(sha512_ssse3_mod_init); | |
184 | module_exit(sha512_ssse3_mod_fini); | |
185 | ||
186 | MODULE_LICENSE("GPL"); | |
187 | MODULE_DESCRIPTION("SHA512 Secure Hash Algorithm, Supplemental SSE3 accelerated"); | |
188 | ||
5d26a105 KC |
189 | MODULE_ALIAS_CRYPTO("sha512"); |
190 | MODULE_ALIAS_CRYPTO("sha384"); |