2 * Using hardware provided CRC32 instruction to accelerate the CRC32 disposal.
3 * CRC32C polynomial:0x1EDC6F41(BE)/0x82F63B78(LE)
4 * CRC32 is a new instruction in Intel SSE4.2, the reference can be found at:
5 * http://www.intel.com/products/processor/manuals/
6 * Intel(R) 64 and IA-32 Architectures Software Developer's Manual
7 * Volume 2A: Instruction Set Reference, A-M
9 * Copyright (c) 2008 Austin Zhang <austin_zhang@linux.intel.com>
10 * Copyright (c) 2008 Kent Liu <kent.liu@intel.com>
12 * This program is free software; you can redistribute it and/or modify it
13 * under the terms of the GNU General Public License as published by the Free
14 * Software Foundation; either version 2 of the License, or (at your option)
18 #include <linux/init.h>
19 #include <linux/module.h>
20 #include <linux/string.h>
21 #include <linux/kernel.h>
22 #include <crypto/internal/hash.h>
24 #include <asm/cpufeature.h>
26 #define CHKSUM_BLOCK_SIZE 1
27 #define CHKSUM_DIGEST_SIZE 4
29 #define SCALE_F sizeof(unsigned long)
32 #define REX_PRE "0x48, "
37 static u32
crc32c_intel_le_hw_byte(u32 crc
, unsigned char const *data
, size_t length
)
41 ".byte 0xf2, 0xf, 0x38, 0xf0, 0xf1"
51 static u32 __pure
crc32c_intel_le_hw(u32 crc
, unsigned char const *p
, size_t len
)
53 unsigned int iquotient
= len
/ SCALE_F
;
54 unsigned int iremainder
= len
% SCALE_F
;
55 unsigned long *ptmp
= (unsigned long *)p
;
59 ".byte 0xf2, " REX_PRE
"0xf, 0x38, 0xf1, 0xf1;"
67 crc
= crc32c_intel_le_hw_byte(crc
, (unsigned char *)ptmp
,
74 * Setting the seed allows arbitrary accumulators and flexible XOR policy
75 * If your algorithm starts with ~0, then XOR with ~0 before you set
78 static int crc32c_intel_setkey(struct crypto_ahash
*hash
, const u8
*key
,
81 u32
*mctx
= crypto_ahash_ctx(hash
);
83 if (keylen
!= sizeof(u32
)) {
84 crypto_ahash_set_flags(hash
, CRYPTO_TFM_RES_BAD_KEY_LEN
);
87 *mctx
= le32_to_cpup((__le32
*)key
);
91 static int crc32c_intel_init(struct ahash_request
*req
)
93 u32
*mctx
= crypto_ahash_ctx(crypto_ahash_reqtfm(req
));
94 u32
*crcp
= ahash_request_ctx(req
);
101 static int crc32c_intel_update(struct ahash_request
*req
)
103 struct crypto_hash_walk walk
;
104 u32
*crcp
= ahash_request_ctx(req
);
108 for (nbytes
= crypto_hash_walk_first(req
, &walk
); nbytes
;
109 nbytes
= crypto_hash_walk_done(&walk
, 0))
110 crc
= crc32c_intel_le_hw(crc
, walk
.data
, nbytes
);
116 static int crc32c_intel_final(struct ahash_request
*req
)
118 u32
*crcp
= ahash_request_ctx(req
);
120 *(__le32
*)req
->result
= ~cpu_to_le32p(crcp
);
124 static int crc32c_intel_digest(struct ahash_request
*req
)
126 struct crypto_hash_walk walk
;
127 u32
*mctx
= crypto_ahash_ctx(crypto_ahash_reqtfm(req
));
131 for (nbytes
= crypto_hash_walk_first(req
, &walk
); nbytes
;
132 nbytes
= crypto_hash_walk_done(&walk
, 0))
133 crc
= crc32c_intel_le_hw(crc
, walk
.data
, nbytes
);
135 *(__le32
*)req
->result
= ~cpu_to_le32(crc
);
139 static int crc32c_intel_cra_init(struct crypto_tfm
*tfm
)
141 u32
*key
= crypto_tfm_ctx(tfm
);
145 tfm
->crt_ahash
.reqsize
= sizeof(u32
);
150 static struct crypto_alg alg
= {
151 .cra_name
= "crc32c",
152 .cra_driver_name
= "crc32c-intel",
154 .cra_flags
= CRYPTO_ALG_TYPE_AHASH
,
155 .cra_blocksize
= CHKSUM_BLOCK_SIZE
,
157 .cra_ctxsize
= sizeof(u32
),
158 .cra_module
= THIS_MODULE
,
159 .cra_list
= LIST_HEAD_INIT(alg
.cra_list
),
160 .cra_init
= crc32c_intel_cra_init
,
161 .cra_type
= &crypto_ahash_type
,
164 .digestsize
= CHKSUM_DIGEST_SIZE
,
165 .setkey
= crc32c_intel_setkey
,
166 .init
= crc32c_intel_init
,
167 .update
= crc32c_intel_update
,
168 .final
= crc32c_intel_final
,
169 .digest
= crc32c_intel_digest
,
175 static int __init
crc32c_intel_mod_init(void)
178 return crypto_register_alg(&alg
);
183 static void __exit
crc32c_intel_mod_fini(void)
185 crypto_unregister_alg(&alg
);
188 module_init(crc32c_intel_mod_init
);
189 module_exit(crc32c_intel_mod_fini
);
191 MODULE_AUTHOR("Austin Zhang <austin.zhang@intel.com>, Kent Liu <kent.liu@intel.com>");
192 MODULE_DESCRIPTION("CRC32c (Castagnoli) optimization using Intel Hardware.");
193 MODULE_LICENSE("GPL");
195 MODULE_ALIAS("crc32c");
196 MODULE_ALIAS("crc32c-intel");